iokit/Kernel/IOKitDebug.cpp

   1 /*
   2  * Copyright (c) 1998-2010 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29
  30 #include <sys/sysctl.h>
  31 extern "C" {
  32 #include <vm/vm_kern.h>
  33 }
  34
  35 #include <libkern/c++/OSContainers.h>
  36 #include <libkern/OSDebug.h>
  37 #include <libkern/c++/OSCPPDebug.h>
  38
  39 #include <IOKit/IOKitDebug.h>
  40 #include <IOKit/IOLib.h>
  41 #include <IOKit/assert.h>
  42 #include <IOKit/IODeviceTreeSupport.h>
  43 #include <IOKit/IOService.h>
  44
  45 #ifdef IOKITDEBUG
  46 #define DEBUG_INIT_VALUE IOKITDEBUG
  47 #else
  48 #define DEBUG_INIT_VALUE 0
  49 #endif
  50
  51 SInt64          gIOKitDebug = DEBUG_INIT_VALUE;
  52 SInt64          gIOKitTrace = 0;
  53
  54 #if DEVELOPMENT || DEBUG
  55 #define IODEBUG_CTLFLAGS        CTLFLAG_RW
  56 #else
  57 #define IODEBUG_CTLFLAGS        CTLFLAG_RD
  58 #endif
  59
  60 SYSCTL_QUAD(_debug, OID_AUTO, iokit, IODEBUG_CTLFLAGS | CTLFLAG_LOCKED, &gIOKitDebug, "boot_arg io");
  61 SYSCTL_QUAD(_debug, OID_AUTO, iotrace, CTLFLAG_RW | CTLFLAG_LOCKED, &gIOKitTrace, "trace io");
  62
  63
  64 int             debug_malloc_size;
  65 int             debug_iomalloc_size;
  66
  67 vm_size_t       debug_iomallocpageable_size;
  68 int             debug_container_malloc_size;
  69 // int          debug_ivars_size; // in OSObject.cpp
  70
  71 extern "C" {
  72
  73 #if 0
  74 #define DEBG(fmt, args...)   { kprintf(fmt, ## args); }
  75 #else
  76 #define DEBG(fmt, args...)   { IOLog(fmt, ## args); }
  77 #endif
  78
  79 void IOPrintPlane( const IORegistryPlane * plane )
  80 {
  81     IORegistryEntry *           next;
  82     IORegistryIterator *        iter;
  83     OSOrderedSet *              all;
  84     char                        format[] = "%xxxs";
  85     IOService *                 service;
  86
  87     iter = IORegistryIterator::iterateOver( plane );
  88     assert( iter );
  89     all = iter->iterateAll();
  90     if( all) {
  91         DEBG("Count %d\n", all->getCount() );
  92         all->release();
  93     } else
  94         DEBG("Empty\n");
  95
  96     iter->reset();
  97     while( (next = iter->getNextObjectRecursive())) {
  98         snprintf(format + 1, sizeof(format) - 1, "%ds", 2 * next->getDepth( plane ));
  99         DEBG( format, "");
 100         DEBG( "\033[33m%s", next->getName( plane ));
 101         if( (next->getLocation( plane )))
 102             DEBG("@%s", next->getLocation( plane ));
 103         DEBG("\033[0m <class %s", next->getMetaClass()->getClassName());
 104         if( (service = OSDynamicCast(IOService, next)))
 105             DEBG(", busy %ld", (long) service->getBusyState());
 106         DEBG( ">\n");
 107 //      IOSleep(250);
 108     }
 109     iter->release();
 110 }
 111
 112 void db_piokjunk(void)
 113 {
 114 }
 115
 116 void db_dumpiojunk( const IORegistryPlane * plane __unused )
 117 {
 118 }
 119
 120 void IOPrintMemory( void )
 121 {
 122
 123 //    OSMetaClass::printInstanceCounts();
 124
 125     IOLog("\n"
 126             "ivar kalloc()       0x%08x\n"
 127             "malloc()            0x%08x\n"
 128             "containers kalloc() 0x%08x\n"
 129             "IOMalloc()          0x%08x\n"
 130             "----------------------------------------\n",
 131             debug_ivars_size,
 132             debug_malloc_size,
 133             debug_container_malloc_size,
 134             debug_iomalloc_size
 135             );
 136 }
 137
 138 } /* extern "C" */
 139
 140 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 141
 142 #define super OSObject
 143 OSDefineMetaClassAndStructors(IOKitDiagnostics, OSObject)
 144
 145 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 146
 147 OSObject * IOKitDiagnostics::diagnostics( void )
 148 {
 149     IOKitDiagnostics * diags;
 150
 151     diags = new IOKitDiagnostics;
 152     if( diags && !diags->init()) {
 153         diags->release();
 154         diags = 0;
 155     }
 156
 157     return( diags );
 158 }
 159
 160 void IOKitDiagnostics::updateOffset( OSDictionary * dict,
 161                         UInt64 value, const char * name )
 162 {
 163     OSNumber * off;
 164
 165     off = OSNumber::withNumber( value, 64 );
 166     if( !off)
 167         return;
 168
 169     dict->setObject( name, off );
 170     off->release();
 171 }
 172
 173 bool IOKitDiagnostics::serialize(OSSerialize *s) const
 174 {
 175     OSDictionary *      dict;
 176     bool                ok;
 177
 178     dict = OSDictionary::withCapacity( 5 );
 179     if( !dict)
 180         return( false );
 181
 182     updateOffset( dict, debug_ivars_size, "Instance allocation" );
 183     updateOffset( dict, debug_container_malloc_size, "Container allocation" );
 184     updateOffset( dict, debug_iomalloc_size, "IOMalloc allocation" );
 185     updateOffset( dict, debug_iomallocpageable_size, "Pageable allocation" );
 186
 187     OSMetaClass::serializeClassDictionary(dict);
 188
 189     ok = dict->serialize( s );
 190
 191     dict->release();
 192
 193     return( ok );
 194 }
 195
 196 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 197
 198 #if IOTRACKING
 199
 200 #include <libkern/c++/OSCPPDebug.h>
 201 #include <libkern/c++/OSKext.h>
 202 #include <kern/zalloc.h>
 203
 204 __private_extern__ "C" void qsort(
 205     void * array,
 206     size_t nmembers,
 207     size_t member_size,
 208     int (*)(const void *, const void *));
 209
 210 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
 211 extern "C" ppnum_t pmap_valid_page(ppnum_t pn);
 212
 213 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 214
 215 struct IOTRecursiveLock
 216 {
 217     lck_mtx_t * mutex;
 218     thread_t    thread;
 219     UInt32      count;
 220 };
 221
 222 struct IOTrackingQueue
 223 {
 224     queue_chain_t     link;
 225     IOTRecursiveLock  lock;
 226     queue_head_t      sites;
 227     const char *      name;
 228     size_t            allocSize;
 229     size_t            minCaptureSize;
 230     uint32_t          siteCount;
 231     uint8_t           captureOn;
 232     uint8_t           isAlloc;
 233 };
 234
 235 struct IOTrackingCallSite
 236 {
 237     queue_chain_t          link;
 238     IOTrackingQueue *      queue;
 239     uint32_t               crc;
 240     IOTrackingCallSiteInfo info;
 241     queue_chain_t          instances;
 242     IOTracking *           addresses;
 243 };
 244
 245 struct IOTrackingLeaksRef
 246 {
 247     uintptr_t * instances;
 248     uint32_t    count;
 249     uint32_t    found;
 250     size_t      bytes;
 251 };
 252
 253 enum
 254 {
 255     kInstanceFlagAddress    = 0x01UL,
 256     kInstanceFlagReferenced = 0x02UL,
 257     kInstanceFlags          = 0x03UL
 258 };
 259
 260 lck_mtx_t *  gIOTrackingLock;
 261 queue_head_t gIOTrackingQ;
 262
 263 enum
 264 {
 265     kTrackingAddressFlagAllocated    = 0x00000001
 266 };
 267
 268 #if defined(__LP64__)
 269 #define IOTrackingAddressFlags(ptr)     (ptr->flags)
 270 #else
 271 #define IOTrackingAddressFlags(ptr)     (ptr->tracking.flags)
 272 #endif
 273
 274 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 275
 276 static void
 277 IOTRecursiveLockLock(IOTRecursiveLock * lock)
 278 {
 279     if (lock->thread == current_thread()) lock->count++;
 280     else
 281     {
 282         lck_mtx_lock(lock->mutex);
 283         assert(lock->thread == 0);
 284         assert(lock->count == 0);
 285         lock->thread = current_thread();
 286         lock->count = 1;
 287     }
 288 }
 289
 290 static void
 291 IOTRecursiveLockUnlock(IOTRecursiveLock * lock)
 292 {
 293     assert(lock->thread == current_thread());
 294     if (0 == (--lock->count))
 295     {
 296         lock->thread = 0;
 297         lck_mtx_unlock(lock->mutex);
 298     }
 299 }
 300
 301 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 302
 303 void
 304 IOTrackingInit(void)
 305 {
 306     queue_init(&gIOTrackingQ);
 307     gIOTrackingLock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL);
 308 }
 309
 310 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 311
 312 IOTrackingQueue *
 313 IOTrackingQueueAlloc(const char * name, size_t allocSize, size_t minCaptureSize, bool isAlloc)
 314 {
 315     IOTrackingQueue * queue;
 316     queue = (typeof(queue)) kalloc(sizeof(IOTrackingQueue));
 317     bzero(queue, sizeof(IOTrackingQueue));
 318
 319     queue->name           = name;
 320     queue->allocSize      = allocSize;
 321     queue->minCaptureSize = minCaptureSize;
 322     queue->lock.mutex     = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL);
 323     queue_init(&queue->sites);
 324
 325     queue->captureOn = (0 != (kIOTrackingBoot & gIOKitDebug));
 326     queue->isAlloc   = isAlloc;
 327
 328     lck_mtx_lock(gIOTrackingLock);
 329     queue_enter(&gIOTrackingQ, queue, IOTrackingQueue *, link);
 330     lck_mtx_unlock(gIOTrackingLock);
 331
 332     return (queue);
 333 };
 334
 335 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 336
 337 void
 338 IOTrackingQueueFree(IOTrackingQueue * queue)
 339 {
 340     lck_mtx_lock(gIOTrackingLock);
 341     IOTrackingReset(queue);
 342     remque(&queue->link);
 343     lck_mtx_unlock(gIOTrackingLock);
 344
 345     lck_mtx_free(queue->lock.mutex, IOLockGroup);
 346
 347     kfree(queue, sizeof(IOTrackingQueue));
 348 };
 349
 350 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 351
 352 /* fasthash
 353    The MIT License
 354
 355    Copyright (C) 2012 Zilong Tan (eric.zltan@gmail.com)
 356
 357    Permission is hereby granted, free of charge, to any person
 358    obtaining a copy of this software and associated documentation
 359    files (the "Software"), to deal in the Software without
 360    restriction, including without limitation the rights to use, copy,
 361    modify, merge, publish, distribute, sublicense, and/or sell copies
 362    of the Software, and to permit persons to whom the Software is
 363    furnished to do so, subject to the following conditions:
 364
 365    The above copyright notice and this permission notice shall be
 366    included in all copies or substantial portions of the Software.
 367
 368    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 369    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 370    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 371    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 372    BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 373    ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 374    CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 375    SOFTWARE.
 376 */
 377
 378
 379 // Compression function for Merkle-Damgard construction.
 380 // This function is generated using the framework provided.
 381 #define mix(h) ({                               \
 382                   (h) ^= (h) >> 23;             \
 383                   (h) *= 0x2127599bf4325c37ULL; \
 384                   (h) ^= (h) >> 47; })
 385
 386 static uint64_t
 387 fasthash64(const void *buf, size_t len, uint64_t seed)
 388 {
 389     const uint64_t    m = 0x880355f21e6d1965ULL;
 390     const uint64_t *pos = (const uint64_t *)buf;
 391     const uint64_t *end = pos + (len / 8);
 392     const unsigned char *pos2;
 393     uint64_t h = seed ^ (len * m);
 394     uint64_t v;
 395
 396     while (pos != end) {
 397         v  = *pos++;
 398         h ^= mix(v);
 399         h *= m;
 400     }
 401
 402     pos2 = (const unsigned char*)pos;
 403     v = 0;
 404
 405     switch (len & 7) {
 406     case 7: v ^= (uint64_t)pos2[6] << 48;
 407     case 6: v ^= (uint64_t)pos2[5] << 40;
 408     case 5: v ^= (uint64_t)pos2[4] << 32;
 409     case 4: v ^= (uint64_t)pos2[3] << 24;
 410     case 3: v ^= (uint64_t)pos2[2] << 16;
 411     case 2: v ^= (uint64_t)pos2[1] << 8;
 412     case 1: v ^= (uint64_t)pos2[0];
 413             h ^= mix(v);
 414             h *= m;
 415     }
 416
 417     return mix(h);
 418 }
 419
 420 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 421
 422 static uint32_t
 423 fasthash32(const void *buf, size_t len, uint32_t seed)
 424 {
 425     // the following trick converts the 64-bit hashcode to Fermat
 426     // residue, which shall retain information from both the higher
 427     // and lower parts of hashcode.
 428     uint64_t h = fasthash64(buf, len, seed);
 429     return h - (h >> 32);
 430 }
 431
 432 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 433
 434 void
 435 IOTrackingAdd(IOTrackingQueue * queue, IOTracking * mem, size_t size, bool address)
 436 {
 437     IOTrackingCallSite * site;
 438     uint32_t             crc, num;
 439     uintptr_t            bt[kIOTrackingCallSiteBTs + 1];
 440
 441     if (mem->site)                    return;
 442     if (!queue->captureOn)            return;
 443     if (size < queue->minCaptureSize) return;
 444
 445     assert(!mem->link.next);
 446
 447     num  = fastbacktrace(&bt[0], kIOTrackingCallSiteBTs + 1);
 448     num--;
 449     crc = fasthash32(&bt[1], num * sizeof(bt[0]), 0x04C11DB7);
 450
 451     IOTRecursiveLockLock(&queue->lock);
 452     queue_iterate(&queue->sites, site, IOTrackingCallSite *, link)
 453     {
 454         if (crc == site->crc) break;
 455     }
 456
 457     if (queue_end(&queue->sites, (queue_entry_t) site))
 458     {
 459         site = (typeof(site)) kalloc(sizeof(IOTrackingCallSite));
 460
 461         queue_init(&site->instances);
 462         site->addresses  = (IOTracking *) &site->instances;
 463         site->queue      = queue;
 464         site->crc        = crc;
 465         site->info.count = 0;
 466         memset(&site->info.size[0], 0, sizeof(site->info.size));
 467         bcopy(&bt[1], &site->info.bt[0], num * sizeof(site->info.bt[0]));
 468         assert(num <= kIOTrackingCallSiteBTs);
 469         bzero(&site->info.bt[num], (kIOTrackingCallSiteBTs - num) * sizeof(site->info.bt[0]));
 470
 471         queue_enter_first(&queue->sites, site, IOTrackingCallSite *, link);
 472         queue->siteCount++;
 473     }
 474
 475     if (address)
 476     {
 477         queue_enter/*last*/(&site->instances, mem, IOTrackingCallSite *, link);
 478         if (queue_end(&site->instances, (queue_entry_t)site->addresses)) site->addresses = mem;
 479     }
 480     else queue_enter_first(&site->instances, mem, IOTrackingCallSite *, link);
 481
 482     mem->site = site;
 483     site->info.size[0] += size;
 484     site->info.count++;
 485
 486     IOTRecursiveLockUnlock(&queue->lock);
 487 }
 488
 489 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 490
 491 void
 492 IOTrackingRemove(IOTrackingQueue * queue, IOTracking * mem, size_t size)
 493 {
 494     if (!mem->link.next) return;
 495
 496     IOTRecursiveLockLock(&queue->lock);
 497
 498     assert(mem->site);
 499
 500     if (mem == mem->site->addresses) mem->site->addresses = (IOTracking *) queue_next(&mem->link);
 501     remque(&mem->link);
 502
 503     assert(mem->site->info.count);
 504     mem->site->info.count--;
 505     assert(mem->site->info.size[0] >= size);
 506     mem->site->info.size[0] -= size;
 507     if (!mem->site->info.count)
 508     {
 509         assert(queue_empty(&mem->site->instances));
 510         assert(!mem->site->info.size[0]);
 511         assert(!mem->site->info.size[1]);
 512
 513         remque(&mem->site->link);
 514         assert(queue->siteCount);
 515         queue->siteCount--;
 516         kfree(mem->site, sizeof(IOTrackingCallSite));
 517     }
 518     IOTRecursiveLockUnlock(&queue->lock);
 519 }
 520
 521 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 522
 523 void
 524 IOTrackingAlloc(IOTrackingQueue * queue, uintptr_t address, size_t size)
 525 {
 526     IOTrackingAddress * tracking;
 527
 528     if (!queue->captureOn)            return;
 529     if (size < queue->minCaptureSize) return;
 530
 531     address = ~address;
 532     tracking = (typeof(tracking)) kalloc(sizeof(IOTrackingAddress));
 533     bzero(tracking, sizeof(IOTrackingAddress));
 534     IOTrackingAddressFlags(tracking) |= kTrackingAddressFlagAllocated;
 535     tracking->address = address;
 536     tracking->size    = size;
 537
 538     IOTrackingAdd(queue, &tracking->tracking, size, true);
 539 }
 540
 541 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 542
 543 void
 544 IOTrackingFree(IOTrackingQueue * queue, uintptr_t address, size_t size)
 545 {
 546     IOTrackingCallSite * site;
 547     IOTrackingAddress  * tracking;
 548     bool                 done;
 549
 550     address = ~address;
 551     IOTRecursiveLockLock(&queue->lock);
 552     done = false;
 553     queue_iterate(&queue->sites, site, IOTrackingCallSite *, link)
 554     {
 555         for (tracking = (IOTrackingAddress *) site->addresses;
 556                 !done && !queue_end(&site->instances, (queue_entry_t) tracking);
 557                 tracking = (IOTrackingAddress *) queue_next(&tracking->tracking.link))
 558         {
 559             if ((done = (address == tracking->address)))
 560             {
 561                 IOTrackingRemove(queue, &tracking->tracking, size);
 562                 kfree(tracking, sizeof(IOTrackingAddress));
 563             }
 564         }
 565         if (done) break;
 566     }
 567
 568     IOTRecursiveLockUnlock(&queue->lock);
 569 }
 570
 571 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 572
 573 void
 574 IOTrackingAccumSize(IOTrackingQueue * queue, IOTracking * mem, size_t size)
 575 {
 576     IOTRecursiveLockLock(&queue->lock);
 577     if (mem->link.next)
 578     {
 579         assert(mem->site);
 580         assert((size > 0) || (mem->site->info.size[1] >= -size));
 581         mem->site->info.size[1] += size;
 582     };
 583     IOTRecursiveLockUnlock(&queue->lock);
 584 }
 585
 586 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 587
 588 void
 589 IOTrackingReset(IOTrackingQueue * queue)
 590 {
 591     IOTrackingCallSite * site;
 592     IOTracking         * tracking;
 593     IOTrackingAddress  * trackingAddress;
 594     bool                 addresses;
 595
 596     IOTRecursiveLockLock(&queue->lock);
 597     while (!queue_empty(&queue->sites))
 598     {
 599         queue_remove_first(&queue->sites, site, IOTrackingCallSite *, link);
 600         addresses = false;
 601         while (!queue_empty(&site->instances))
 602         {
 603             queue_remove_first(&site->instances, tracking, IOTracking *, link);
 604             tracking->link.next = 0;
 605             if (tracking == site->addresses) addresses = true;
 606             if (addresses)
 607             {
 608                 trackingAddress = (typeof(trackingAddress)) tracking;
 609                 if (kTrackingAddressFlagAllocated & IOTrackingAddressFlags(trackingAddress))
 610                 {
 611                     kfree(tracking, sizeof(IOTrackingAddress));
 612                 }
 613             }
 614         }
 615         kfree(site, sizeof(IOTrackingCallSite));
 616     }
 617     queue->siteCount = 0;
 618     IOTRecursiveLockUnlock(&queue->lock);
 619 }
 620
 621 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 622
 623 static int
 624 IOTrackingCallSiteInfoCompare(const void * left, const void * right)
 625 {
 626     IOTrackingCallSiteInfo * l = (typeof(l)) left;
 627     IOTrackingCallSiteInfo * r = (typeof(r)) right;
 628     size_t                   lsize, rsize;
 629
 630     rsize = r->size[0] + r->size[1];
 631     lsize = l->size[0] + l->size[1];
 632
 633     return ((rsize > lsize) ? 1 : ((rsize == lsize) ? 0 : -1));
 634 }
 635
 636 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 637
 638 static int
 639 IOTrackingAddressCompare(const void * left, const void * right)
 640 {
 641     IOTracking * instance;
 642     uintptr_t    inst, laddr, raddr;
 643
 644     inst = ((typeof(inst) *) left)[0];
 645     instance = (typeof(instance)) (inst & ~kInstanceFlags);
 646     if (kInstanceFlagAddress & inst) laddr = ~((IOTrackingAddress *)instance)->address;
 647     else                             laddr = (uintptr_t) (instance + 1);
 648
 649     inst = ((typeof(inst) *) right)[0];
 650     instance = (typeof(instance)) (inst & ~kInstanceFlags);
 651     if (kInstanceFlagAddress & inst) raddr = ~((IOTrackingAddress *)instance)->address;
 652     else                             raddr = (uintptr_t) (instance + 1);
 653
 654     return ((laddr > raddr) ? 1 : ((laddr == raddr) ? 0 : -1));
 655 }
 656
 657 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 658
 659 static void
 660 IOTrackingLeakScan(void * refcon)
 661 {
 662     IOTrackingLeaksRef * ref = (typeof(ref)) refcon;
 663     uintptr_t          * instances;
 664     IOTracking         * instance;
 665     uint64_t             vaddr, vincr;
 666     ppnum_t              ppn;
 667     uintptr_t            ptr, addr, inst;
 668     size_t               size;
 669     uint32_t             baseIdx, lim, ptrIdx, count;
 670     boolean_t            is;
 671
 672 //    if (cpu_number()) return;
 673
 674     instances = ref->instances;
 675     count     = ref->count;
 676
 677     for (vaddr = VM_MIN_KERNEL_AND_KEXT_ADDRESS;
 678          vaddr < VM_MAX_KERNEL_ADDRESS;
 679          ml_set_interrupts_enabled(is), vaddr += vincr)
 680     {
 681 #if !defined(__LP64__)
 682         thread_block(NULL);
 683 #endif
 684         is = ml_set_interrupts_enabled(false);
 685
 686         ppn = kernel_pmap_present_mapping(vaddr, &vincr);
 687         // check noencrypt to avoid VM structs (map entries) with pointers
 688         if (ppn && (!pmap_valid_page(ppn) || pmap_is_noencrypt(ppn))) ppn = 0;
 689         if (!ppn) continue;
 690
 691         for (ptrIdx = 0; ptrIdx < (page_size / sizeof(uintptr_t)); ptrIdx++)
 692         {
 693             ptr = ((uintptr_t *)vaddr)[ptrIdx];
 694
 695             for (lim = count, baseIdx = 0; lim; lim >>= 1)
 696             {
 697                 inst = instances[baseIdx + (lim >> 1)];
 698                 instance = (typeof(instance)) (inst & ~kInstanceFlags);
 699                 if (kInstanceFlagAddress & inst)
 700                 {
 701                     addr = ~((IOTrackingAddress *)instance)->address;
 702                     size = ((IOTrackingAddress *)instance)->size;
 703                 }
 704                 else
 705                 {
 706                     addr = (uintptr_t) (instance + 1);
 707                     size = instance->site->queue->allocSize;
 708                 }
 709                 if ((ptr >= addr) && (ptr < (addr + size)))
 710                 {
 711                     if (!(kInstanceFlagReferenced & inst))
 712                     {
 713                         inst |= kInstanceFlagReferenced;
 714                         instances[baseIdx + (lim >> 1)] = inst;
 715                         ref->found++;
 716                     }
 717                     break;
 718                 }
 719                 if (ptr > addr)
 720                 {
 721                     // move right
 722                     baseIdx += (lim >> 1) + 1;
 723                     lim--;
 724                 }
 725                 // else move left
 726             }
 727         }
 728         ref->bytes += page_size;
 729     }
 730 }
 731
 732 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 733
 734 static OSData *
 735 IOTrackingLeaks(OSData * data)
 736 {
 737     IOTrackingLeaksRef       ref;
 738     IOTrackingCallSiteInfo   unslideInfo;
 739     IOTrackingCallSite     * site;
 740     OSData                 * leakData;
 741     uintptr_t              * instances;
 742     IOTracking             * instance;
 743     uintptr_t                inst;
 744     uint32_t                 count, idx, numSites, dups, siteCount;
 745
 746     instances = (typeof(instances)) data->getBytesNoCopy();
 747     count = (data->getLength() / sizeof(*instances));
 748     qsort(instances, count, sizeof(*instances), &IOTrackingAddressCompare);
 749
 750     bzero(&ref, sizeof(ref));
 751     ref.instances = instances;
 752     ref.count = count;
 753
 754     IOTrackingLeakScan(&ref);
 755
 756     IOLog("leaks scanned %ld MB, instance count %d, found %d\n", ref.bytes / 1024 / 1024, count, ref.found);
 757
 758     leakData = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo));
 759
 760     for (numSites = 0, idx = 0; idx < count; idx++)
 761     {
 762         inst = instances[idx];
 763         if (kInstanceFlagReferenced & inst) continue;
 764         instance = (typeof(instance)) (inst & ~kInstanceFlags);
 765         site = instance->site;
 766         instances[numSites] = (uintptr_t) site;
 767         numSites++;
 768     }
 769
 770     for (idx = 0; idx < numSites; idx++)
 771     {
 772         inst = instances[idx];
 773         if (!inst) continue;
 774         site = (typeof(site)) inst;
 775         for (siteCount = 1, dups = (idx + 1); dups < numSites; dups++)
 776         {
 777             if (instances[dups] == (uintptr_t) site)
 778             {
 779                 siteCount++;
 780                 instances[dups] = 0;
 781             }
 782         }
 783         unslideInfo.count   = siteCount;
 784         unslideInfo.size[0] = (site->info.size[0] * site->info.count) / siteCount;
 785         unslideInfo.size[1] = (site->info.size[1] * site->info.count) / siteCount;;
 786         for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++)
 787         {
 788             unslideInfo.bt[j] = VM_KERNEL_UNSLIDE(site->info.bt[j]);
 789         }
 790         leakData->appendBytes(&unslideInfo, sizeof(unslideInfo));
 791     }
 792     data->release();
 793
 794     return (leakData);
 795 }
 796
 797 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 798
 799 static bool
 800 SkipName(uint32_t options, const char * name, size_t namesLen, const char * names)
 801 {
 802     const char * scan;
 803     const char * next;
 804     bool         exclude, found;
 805     size_t       qLen, sLen;
 806
 807     if (!namesLen || !names) return (false);
 808     // <len><name>...<len><name><0>
 809     exclude = (0 != (kIOTrackingExcludeNames & options));
 810     qLen    = strlen(name);
 811     scan    = names;
 812     found   = false;
 813     do
 814     {
 815         sLen = scan[0];
 816         scan++;
 817         next = scan + sLen;
 818         if (next >= (names + namesLen)) break;
 819         found = ((sLen == qLen) && !strncmp(scan, name, sLen));
 820         scan = next;
 821     }
 822     while (!found && (scan < (names + namesLen)));
 823
 824     return (!(exclude ^ found));
 825 }
 826
 827 #endif /* IOTRACKING */
 828
 829 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 830
 831 kern_return_t
 832 IOTrackingDebug(uint32_t selector, uint32_t options,
 833                 const char * names, size_t namesLen,
 834                 size_t size, OSObject ** result)
 835 {
 836     kern_return_t            ret;
 837     OSData                 * data;
 838
 839     if (result) *result = 0;
 840     data = 0;
 841     ret = kIOReturnNotReady;
 842
 843 #if IOTRACKING
 844
 845     IOTrackingQueue        * queue;
 846     IOTracking             * instance;
 847     IOTrackingCallSite     * site;
 848     IOTrackingCallSiteInfo * siteInfos;
 849     IOTrackingCallSiteInfo * siteInfo;
 850     bool                     addresses;
 851     uint32_t                 num, idx;
 852     uintptr_t                instFlags;
 853
 854     if (!(kIOTracking & gIOKitDebug)) return (kIOReturnNotReady);
 855     ret = kIOReturnNotFound;
 856
 857     lck_mtx_lock(gIOTrackingLock);
 858     queue_iterate(&gIOTrackingQ, queue, IOTrackingQueue *, link)
 859     {
 860         if (SkipName(options, queue->name, namesLen, names)) continue;
 861
 862         switch (selector)
 863         {
 864             case kIOTrackingResetTracking:
 865             {
 866                 IOTrackingReset(queue);
 867                 ret = kIOReturnSuccess;
 868                 break;
 869             }
 870
 871             case kIOTrackingStartCapture:
 872             case kIOTrackingStopCapture:
 873             {
 874                 queue->captureOn = (kIOTrackingStartCapture == selector);
 875                 ret = kIOReturnSuccess;
 876                 break;
 877             }
 878
 879             case kIOTrackingSetMinCaptureSize:
 880             {
 881                 queue->minCaptureSize = size;
 882                 ret = kIOReturnSuccess;
 883                 break;
 884             }
 885
 886             case kIOTrackingLeaks:
 887             {
 888                 if (!queue->isAlloc) break;
 889
 890                 if (!data) data = OSData::withCapacity(1024 * sizeof(uintptr_t));
 891
 892                 IOTRecursiveLockLock(&queue->lock);
 893                 queue_iterate(&queue->sites, site, IOTrackingCallSite *, link)
 894                 {
 895                     addresses = false;
 896                     queue_iterate(&site->instances, instance, IOTracking *, link)
 897                     {
 898                         if (instance == site->addresses) addresses = true;
 899                         instFlags = (typeof(instFlags)) instance;
 900                         if (addresses) instFlags |= kInstanceFlagAddress;
 901                         data->appendBytes(&instFlags, sizeof(instFlags));
 902                     }
 903                 }
 904                 // queue is locked
 905                 ret = kIOReturnSuccess;
 906                 break;
 907             }
 908
 909             case kIOTrackingGetTracking:
 910             case kIOTrackingPrintTracking:
 911             {
 912                 if (!data) data = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo));
 913
 914                 IOTRecursiveLockLock(&queue->lock);
 915                 num = queue->siteCount;
 916                 idx = 0;
 917                 queue_iterate(&queue->sites, site, IOTrackingCallSite *, link)
 918                 {
 919                     assert(idx < num);
 920                     idx++;
 921
 922                     if (size && ((site->info.size[0] + site->info.size[1]) < size)) continue;
 923
 924                     IOTrackingCallSiteInfo unslideInfo;
 925                     unslideInfo.count = site->info.count;
 926                     memcpy(&unslideInfo.size[0], &site->info.size[0], sizeof(unslideInfo.size));
 927
 928                     for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++)
 929                     {
 930                         unslideInfo.bt[j] = VM_KERNEL_UNSLIDE(site->info.bt[j]);
 931                     }
 932                     data->appendBytes(&unslideInfo, sizeof(unslideInfo));
 933                 }
 934                 assert(idx == num);
 935                 IOTRecursiveLockUnlock(&queue->lock);
 936                 ret = kIOReturnSuccess;
 937                 break;
 938             }
 939             default:
 940                 ret = kIOReturnUnsupported;
 941                 break;
 942         }
 943     }
 944
 945     if ((kIOTrackingLeaks == selector) && data)
 946     {
 947         data = IOTrackingLeaks(data);
 948         queue_iterate(&gIOTrackingQ, queue, IOTrackingQueue *, link)
 949         {
 950             if (SkipName(options, queue->name, namesLen, names)) continue;
 951             if (!queue->isAlloc)                                 continue;
 952             IOTRecursiveLockUnlock(&queue->lock);
 953         }
 954     }
 955
 956     lck_mtx_unlock(gIOTrackingLock);
 957
 958     if (data)
 959     {
 960         siteInfos = (typeof(siteInfos)) data->getBytesNoCopy();
 961         num = (data->getLength() / sizeof(IOTrackingCallSiteInfo));
 962         qsort(siteInfos, num, sizeof(*siteInfos), &IOTrackingCallSiteInfoCompare);
 963
 964         if (kIOTrackingPrintTracking == selector)
 965         {
 966             for (idx = 0; idx < num; idx++)
 967             {
 968                 siteInfo = &siteInfos[idx];
 969                 printf("\n0x%lx bytes (0x%lx + 0x%lx), %d call%s, [%d]\n",
 970                     siteInfo->size[0] + siteInfo->size[1],
 971                     siteInfo->size[0], siteInfo->size[1],
 972                     siteInfo->count, (siteInfo->count != 1) ? "s" : "", idx);
 973                 uintptr_t * bt = &siteInfo->bt[0];
 974                 printf("      Backtrace 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx\n",
 975                         bt[0], bt[1], bt[2], bt[3], bt[4], bt[5], bt[6], bt[7],
 976                         bt[8], bt[9], bt[10], bt[11], bt[12], bt[13], bt[14], bt[15]);
 977                 kmod_dump_log((vm_offset_t *) &bt[0], kIOTrackingCallSiteBTs, FALSE);
 978             }
 979             data->release();
 980             data = 0;
 981         }
 982     }
 983
 984     *result = data;
 985
 986 #endif /* IOTRACKING */
 987
 988     return (ret);
 989 }
 990
 991 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 992
 993 #include <IOKit/IOKitDiagnosticsUserClient.h>
 994
 995 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 996
 997 #undef super
 998 #define super IOUserClient
 999
1000 OSDefineMetaClassAndStructors(IOKitDiagnosticsClient, IOUserClient)
1001
1002 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1003
1004 IOUserClient * IOKitDiagnosticsClient::withTask(task_t owningTask)
1005 {
1006     IOKitDiagnosticsClient * inst;
1007
1008     inst = new IOKitDiagnosticsClient;
1009     if (inst && !inst->init())
1010     {
1011         inst->release();
1012         inst = 0;
1013     }
1014
1015     return (inst);
1016 }
1017
1018 IOReturn IOKitDiagnosticsClient::clientClose(void)
1019 {
1020     terminate();
1021     return (kIOReturnSuccess);
1022 }
1023
1024 IOReturn IOKitDiagnosticsClient::setProperties(OSObject * properties)
1025 {
1026     IOReturn kr = kIOReturnUnsupported;
1027     return (kr);
1028 }
1029
1030 IOReturn IOKitDiagnosticsClient::externalMethod(uint32_t selector, IOExternalMethodArguments * args,
1031                                                 IOExternalMethodDispatch * dispatch, OSObject * target, void * reference)
1032 {
1033     IOReturn                           ret = kIOReturnBadArgument;
1034     const IOKitDiagnosticsParameters * params;
1035     const char * names;
1036     size_t       namesLen;
1037     OSObject   * result;
1038
1039     if (args->structureInputSize < sizeof(IOKitDiagnosticsParameters)) return (kIOReturnBadArgument);
1040     params = (typeof(params)) args->structureInput;
1041     if (!params) return (kIOReturnBadArgument);
1042
1043     names = 0;
1044     namesLen = args->structureInputSize - sizeof(IOKitDiagnosticsParameters);
1045     if (namesLen) names = (typeof(names))(params + 1);
1046
1047     ret = IOTrackingDebug(selector, params->options, names, namesLen, params->size, &result);
1048
1049     if ((kIOReturnSuccess == ret) && args->structureVariableOutputData) *args->structureVariableOutputData = result;
1050     else if (result) result->release();
1051
1052     return (ret);
1053 }
1054
1055 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */