iokit/Kernel/IOMapper.cpp

   1 /*
   2  * Copyright (c) 1998-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #define IOKIT_ENABLE_SHARED_PTR
  30
  31 #include <IOKit/IOLib.h>
  32 #include <IOKit/IOMapper.h>
  33 #include <IOKit/IODMACommand.h>
  34 #include <libkern/c++/OSData.h>
  35 #include <libkern/OSDebug.h>
  36 #include <mach_debug/zone_info.h>
  37 #include "IOKitKernelInternal.h"
  38
  39 __BEGIN_DECLS
  40 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
  41 __END_DECLS
  42
  43 #define super IOService
  44 OSDefineMetaClassAndAbstractStructors(IOMapper, IOService);
  45
  46 OSMetaClassDefineReservedUnused(IOMapper, 0);
  47 OSMetaClassDefineReservedUnused(IOMapper, 1);
  48 OSMetaClassDefineReservedUnused(IOMapper, 2);
  49 OSMetaClassDefineReservedUnused(IOMapper, 3);
  50 OSMetaClassDefineReservedUnused(IOMapper, 4);
  51 OSMetaClassDefineReservedUnused(IOMapper, 5);
  52 OSMetaClassDefineReservedUnused(IOMapper, 6);
  53 OSMetaClassDefineReservedUnused(IOMapper, 7);
  54 OSMetaClassDefineReservedUnused(IOMapper, 8);
  55 OSMetaClassDefineReservedUnused(IOMapper, 9);
  56 OSMetaClassDefineReservedUnused(IOMapper, 10);
  57 OSMetaClassDefineReservedUnused(IOMapper, 11);
  58 OSMetaClassDefineReservedUnused(IOMapper, 12);
  59 OSMetaClassDefineReservedUnused(IOMapper, 13);
  60 OSMetaClassDefineReservedUnused(IOMapper, 14);
  61 OSMetaClassDefineReservedUnused(IOMapper, 15);
  62
  63 IOMapper * IOMapper::gSystem = (IOMapper *) IOMapper::kUnknown;
  64
  65 class IOMapperLock {
  66         IOLock *fWaitLock;
  67 public:
  68         IOMapperLock()
  69         {
  70                 fWaitLock = IOLockAlloc();
  71         }
  72         ~IOMapperLock()
  73         {
  74                 IOLockFree(fWaitLock);
  75         }
  76
  77         void
  78         lock()
  79         {
  80                 IOLockLock(fWaitLock);
  81         }
  82         void
  83         unlock()
  84         {
  85                 IOLockUnlock(fWaitLock);
  86         }
  87         void
  88         sleep(void *event)
  89         {
  90                 IOLockSleep(fWaitLock, event, THREAD_UNINT);
  91         }
  92         void
  93         wakeup(void *event)
  94         {
  95                 IOLockWakeup(fWaitLock, event, false);
  96         }
  97 };
  98
  99 static IOMapperLock sMapperLock;
 100
 101 bool
 102 IOMapper::start(IOService *provider)
 103 {
 104         OSSharedPtr<OSObject> obj;
 105         if (!super::start(provider)) {
 106                 return false;
 107         }
 108
 109         if (!initHardware(provider)) {
 110                 return false;
 111         }
 112
 113         uint64_t pageSize = getPageSize();
 114         assert(pageSize <= UINT_MAX);
 115         fPageSize = (uint32_t) pageSize;
 116
 117         if (fIsSystem) {
 118                 sMapperLock.lock();
 119                 IOMapper::gSystem = this;
 120                 sMapperLock.wakeup(&IOMapper::gSystem);
 121                 sMapperLock.unlock();
 122         }
 123
 124         if (provider) {
 125                 obj = provider->copyProperty("iommu-id");
 126                 if (!obj) {
 127                         obj = provider->copyProperty("AAPL,phandle");
 128                 }
 129                 if (obj) {
 130                         setProperty(gIOMapperIDKey, obj.get());
 131                 }
 132         }
 133         return true;
 134 }
 135
 136 void
 137 IOMapper::free()
 138 {
 139         super::free();
 140 }
 141
 142 void
 143 IOMapper::setMapperRequired(bool hasMapper)
 144 {
 145         if (hasMapper) {
 146                 IOMapper::gSystem = (IOMapper *) kHasMapper;
 147         } else {
 148                 sMapperLock.lock();
 149                 IOMapper::gSystem = (IOMapper *) kNoMapper;
 150                 sMapperLock.unlock();
 151                 sMapperLock.wakeup(&IOMapper::gSystem);
 152         }
 153 }
 154
 155 void
 156 IOMapper::waitForSystemMapper()
 157 {
 158         sMapperLock.lock();
 159         while ((uintptr_t) IOMapper::gSystem & kWaitMask) {
 160                 OSReportWithBacktrace("waitForSystemMapper");
 161                 sMapperLock.sleep(&IOMapper::gSystem);
 162         }
 163         sMapperLock.unlock();
 164 }
 165
 166 OSSharedPtr<IOMapper>
 167 IOMapper::copyMapperForDevice(IOService * device)
 168 {
 169         return copyMapperForDeviceWithIndex(device, 0);
 170 }
 171
 172 OSSharedPtr<IOMapper>
 173 IOMapper::copyMapperForDeviceWithIndex(IOService * device, unsigned int index)
 174 {
 175         OSSharedPtr<OSData> data;
 176         OSSharedPtr<OSObject> obj;
 177         OSSharedPtr<IOMapper> mapper;
 178         OSSharedPtr<OSDictionary> matching;
 179
 180         obj = device->copyProperty("iommu-parent");
 181         if (!obj) {
 182                 return NULL;
 183         }
 184
 185         if ((mapper = OSDynamicPtrCast<IOMapper>(obj))) {
 186                 goto found;
 187         }
 188
 189         if ((data = OSDynamicPtrCast<OSData>(obj))) {
 190                 if (index >= data->getLength() / sizeof(UInt32)) {
 191                         goto found;
 192                 }
 193
 194                 data = OSData::withBytesNoCopy((UInt32 *)data->getBytesNoCopy() + index, sizeof(UInt32));
 195                 if (!data) {
 196                         goto found;
 197                 }
 198
 199                 matching = IOService::propertyMatching(gIOMapperIDKey, data.get());
 200         } else {
 201                 matching = IOService::propertyMatching(gIOMapperIDKey, obj.get());
 202         }
 203
 204         if (matching) {
 205                 mapper = OSDynamicPtrCast<IOMapper>(IOService::waitForMatchingService(matching.get()));
 206         }
 207
 208 found:
 209         if (mapper) {
 210                 if (!mapper->fAllocName) {
 211                         char name[MACH_ZONE_NAME_MAX_LEN];
 212                         char kmodname[KMOD_MAX_NAME];
 213                         vm_tag_t tag;
 214                         uint32_t kmodid;
 215
 216                         tag = IOMemoryTag(kernel_map);
 217                         if (!(kmodid = vm_tag_get_kext(tag, &kmodname[0], KMOD_MAX_NAME))) {
 218                                 snprintf(kmodname, sizeof(kmodname), "%d", tag);
 219                         }
 220                         snprintf(name, sizeof(name), "%s.DMA.%s", kmodname, device->getName());
 221                         mapper->fAllocName = kern_allocation_name_allocate(name, 16);
 222                 }
 223         }
 224
 225         return mapper;
 226 }
 227
 228 __BEGIN_DECLS
 229
 230 // These are C accessors to the system mapper for non-IOKit clients
 231 ppnum_t
 232 IOMapperIOVMAlloc(unsigned pages)
 233 {
 234         IOReturn ret;
 235         uint64_t dmaAddress, dmaLength;
 236
 237         IOMapper::checkForSystemMapper();
 238
 239         ret = kIOReturnUnsupported;
 240         if (IOMapper::gSystem) {
 241                 ret = IOMapper::gSystem->iovmMapMemory(
 242                         NULL, 0, ptoa_64(pages),
 243                         (kIODMAMapReadAccess | kIODMAMapWriteAccess),
 244                         NULL, NULL, NULL,
 245                         &dmaAddress, &dmaLength);
 246         }
 247
 248         if (kIOReturnSuccess == ret) {
 249                 uint64_t dmaAddressPage64;
 250                 dmaAddressPage64 = atop_64(dmaAddress);
 251                 if (dmaAddressPage64 > UINT_MAX) {
 252                         return 0;
 253                 }
 254                 return (ppnum_t) atop_64(dmaAddress);
 255         }
 256         return 0;
 257 }
 258
 259 void
 260 IOMapperIOVMFree(ppnum_t addr, unsigned pages)
 261 {
 262         if (IOMapper::gSystem) {
 263                 IOMapper::gSystem->iovmUnmapMemory(NULL, NULL, ptoa_64(addr), ptoa_64(pages));
 264         }
 265 }
 266
 267 ppnum_t
 268 IOMapperInsertPage(ppnum_t addr, unsigned offset, ppnum_t page)
 269 {
 270         if (!IOMapper::gSystem) {
 271                 return page;
 272         }
 273         if (!addr) {
 274                 panic("!addr");
 275         }
 276         IOMapper::gSystem->iovmInsert((kIODMAMapReadAccess | kIODMAMapWriteAccess),
 277             ptoa_64(addr), ptoa_64(offset), ptoa_64(page), ptoa_64(1));
 278         return addr + offset;
 279 }
 280
 281 /////////////////////////////////////////////////////////////////////////////
 282 //
 283 //
 284 //      IOLib.h APIs
 285 //
 286 //
 287 /////////////////////////////////////////////////////////////////////////////
 288
 289 #include <machine/machine_routines.h>
 290
 291 UInt8
 292 IOMappedRead8(IOPhysicalAddress address)
 293 {
 294         IOMapper::checkForSystemMapper();
 295
 296         if (IOMapper::gSystem) {
 297                 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
 298                 return (UInt8) ml_phys_read_byte_64(addr);
 299         } else {
 300                 return (UInt8) ml_phys_read_byte((vm_offset_t) address);
 301         }
 302 }
 303
 304 UInt16
 305 IOMappedRead16(IOPhysicalAddress address)
 306 {
 307         IOMapper::checkForSystemMapper();
 308
 309         if (IOMapper::gSystem) {
 310                 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
 311                 return (UInt16) ml_phys_read_half_64(addr);
 312         } else {
 313                 return (UInt16) ml_phys_read_half((vm_offset_t) address);
 314         }
 315 }
 316
 317 UInt32
 318 IOMappedRead32(IOPhysicalAddress address)
 319 {
 320         IOMapper::checkForSystemMapper();
 321
 322         if (IOMapper::gSystem) {
 323                 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
 324                 return (UInt32) ml_phys_read_word_64(addr);
 325         } else {
 326                 return (UInt32) ml_phys_read_word((vm_offset_t) address);
 327         }
 328 }
 329
 330 UInt64
 331 IOMappedRead64(IOPhysicalAddress address)
 332 {
 333         IOMapper::checkForSystemMapper();
 334
 335         if (IOMapper::gSystem) {
 336                 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
 337                 return (UInt64) ml_phys_read_double_64(addr);
 338         } else {
 339                 return (UInt64) ml_phys_read_double((vm_offset_t) address);
 340         }
 341 }
 342
 343 void
 344 IOMappedWrite8(IOPhysicalAddress address, UInt8 value)
 345 {
 346         IOMapper::checkForSystemMapper();
 347
 348         if (IOMapper::gSystem) {
 349                 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
 350                 ml_phys_write_byte_64(addr, value);
 351         } else {
 352                 ml_phys_write_byte((vm_offset_t) address, value);
 353         }
 354 }
 355
 356 void
 357 IOMappedWrite16(IOPhysicalAddress address, UInt16 value)
 358 {
 359         IOMapper::checkForSystemMapper();
 360
 361         if (IOMapper::gSystem) {
 362                 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
 363                 ml_phys_write_half_64(addr, value);
 364         } else {
 365                 ml_phys_write_half((vm_offset_t) address, value);
 366         }
 367 }
 368
 369 void
 370 IOMappedWrite32(IOPhysicalAddress address, UInt32 value)
 371 {
 372         IOMapper::checkForSystemMapper();
 373
 374         if (IOMapper::gSystem) {
 375                 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
 376                 ml_phys_write_word_64(addr, value);
 377         } else {
 378                 ml_phys_write_word((vm_offset_t) address, value);
 379         }
 380 }
 381
 382 void
 383 IOMappedWrite64(IOPhysicalAddress address, UInt64 value)
 384 {
 385         IOMapper::checkForSystemMapper();
 386
 387         if (IOMapper::gSystem) {
 388                 addr64_t addr = IOMapper::gSystem->mapToPhysicalAddress(address);
 389                 ml_phys_write_double_64(addr, value);
 390         } else {
 391                 ml_phys_write_double((vm_offset_t) address, value);
 392         }
 393 }
 394
 395 __END_DECLS