iokit/Kernel/IOStatistics.cpp

   1 /*
   2  * Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/sysctl.h>
  30 #include <kern/host.h>
  31
  32 #include <IOKit/system.h>
  33 #include <libkern/c++/OSKext.h>
  34 #include <libkern/OSAtomic.h>
  35
  36 #include <IOKit/IOStatisticsPrivate.h>
  37 #include <IOKit/IOUserClient.h>
  38 #include <IOKit/IOEventSource.h>
  39 #include <IOKit/IOKitDebug.h>
  40
  41 #if IOKITSTATS
  42
  43 bool IOStatistics::enabled = false;
  44
  45 uint32_t IOStatistics::sequenceID = 0;
  46
  47 uint32_t IOStatistics::lastClassIndex = 0;
  48 uint32_t IOStatistics::lastKextIndex = 0;
  49
  50 uint32_t IOStatistics::loadedKexts = 0;
  51 uint32_t IOStatistics::registeredClasses = 0;
  52 uint32_t IOStatistics::registeredCounters = 0;
  53 uint32_t IOStatistics::registeredWorkloops = 0;
  54
  55 uint32_t IOStatistics::attachedEventSources = 0;
  56
  57 IOWorkLoopDependency *IOStatistics::nextWorkLoopDependency = NULL;
  58
  59 /* Logging */
  60
  61 #define LOG_LEVEL 0
  62
  63 #define LOG(level, format, ...) \
  64 do { \
  65         if (level <= LOG_LEVEL) \
  66                 printf(format, ##__VA_ARGS__); \
  67 } while (0)
  68
  69 /* Locks */
  70
  71 IORWLock *IOStatistics::lock = NULL;
  72
  73 /* Kext tree */
  74
  75 KextNode *IOStatistics::kextHint = NULL;
  76
  77 IOStatistics::KextTreeHead IOStatistics::kextHead = RB_INITIALIZER(&IOStatistics::kextHead);
  78
  79 int IOStatistics::kextNodeCompare(KextNode *e1, KextNode *e2)
  80 {
  81     if (e1->kext < e2->kext)
  82         return -1;
  83     else if (e1->kext > e2->kext)
  84         return 1;
  85     else
  86         return 0;
  87 }
  88
  89 RB_GENERATE(IOStatistics::KextTree, KextNode, link, kextNodeCompare);
  90
  91 /* Kext tree ordered by address */
  92
  93 IOStatistics::KextAddressTreeHead IOStatistics::kextAddressHead = RB_INITIALIZER(&IOStatistics::kextAddressHead);
  94
  95 int IOStatistics::kextAddressNodeCompare(KextNode *e1, KextNode *e2)
  96 {
  97     if (e1->address < e2->address)
  98         return -1;
  99     else if (e1->address > e2->address)
 100         return 1;
 101     else
 102         return 0;
 103 }
 104
 105 RB_GENERATE(IOStatistics::KextAddressTree, KextNode, addressLink, kextAddressNodeCompare);
 106
 107 /* Class tree */
 108
 109 IOStatistics::ClassTreeHead IOStatistics::classHead = RB_INITIALIZER(&IOStatistics::classHead);
 110
 111 int IOStatistics::classNodeCompare(ClassNode *e1, ClassNode *e2) {
 112     if (e1->metaClass < e2->metaClass)
 113         return -1;
 114     else if (e1->metaClass > e2->metaClass)
 115         return 1;
 116     else
 117         return 0;
 118 }
 119
 120 RB_GENERATE(IOStatistics::ClassTree, ClassNode, tLink, classNodeCompare);
 121
 122 /* Workloop dependencies */
 123
 124 int IOWorkLoopCounter::loadTagCompare(IOWorkLoopDependency *e1, IOWorkLoopDependency *e2) {
 125     if (e1->loadTag < e2->loadTag)
 126         return -1;
 127     else if (e1->loadTag > e2->loadTag)
 128         return 1;
 129     else
 130         return 0;
 131 }
 132
 133 RB_GENERATE(IOWorkLoopCounter::DependencyTree, IOWorkLoopDependency, link, IOWorkLoopCounter::loadTagCompare);
 134
 135 /* sysctl stuff */
 136
 137 static int
 138 oid_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1, int arg2, struct sysctl_req *req)
 139 {
 140         int error = EINVAL;
 141         uint32_t request = arg2;
 142
 143         switch (request)
 144         {
 145                 case kIOStatisticsGeneral:
 146                         error = IOStatistics::getStatistics(req);
 147                         break;
 148                 case kIOStatisticsWorkLoop:
 149                         error = IOStatistics::getWorkLoopStatistics(req);
 150                         break;
 151                 case kIOStatisticsUserClient:
 152                         error = IOStatistics::getUserClientStatistics(req);
 153                         break;
 154                 default:
 155                         break;
 156         }
 157
 158         return error;
 159 }
 160
 161 SYSCTL_NODE(_debug, OID_AUTO, iokit_statistics, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "IOStatistics");
 162
 163 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, general,
 164             CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 165             0, kIOStatisticsGeneral, oid_sysctl, "S", "");
 166
 167 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, workloop,
 168             CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 169             0, kIOStatisticsWorkLoop, oid_sysctl, "S", "");
 170
 171 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, userclient,
 172             CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 173             0, kIOStatisticsUserClient, oid_sysctl, "S", "");
 174
 175 void IOStatistics::initialize()
 176 {
 177         if (enabled) {
 178                 return;
 179         }
 180
 181 #if DEVELOPMENT || DEBUG
 182         /* Always enabled in development and debug builds. */
 183 #else
 184         /* Only enabled in release builds if the boot argument is set. */
 185         if (!(kIOStatistics & gIOKitDebug)) {
 186                 return;
 187         }
 188 #endif
 189
 190         sysctl_register_oid(&sysctl__debug_iokit_statistics_general);
 191         sysctl_register_oid(&sysctl__debug_iokit_statistics_workloop);
 192         sysctl_register_oid(&sysctl__debug_iokit_statistics_userclient);
 193
 194         lock = IORWLockAlloc();
 195         if (!lock) {
 196                 return;
 197         }
 198
 199         nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
 200         if (!nextWorkLoopDependency) {
 201                 return;
 202         }
 203
 204         enabled = true;
 205 }
 206
 207 void IOStatistics::onKextLoad(OSKext *kext, kmod_info_t *kmod_info)
 208 {
 209         KextNode *ke;
 210
 211         assert(kext && kmod_info);
 212
 213         if (!enabled) {
 214                 return;
 215         }
 216
 217         LOG(1, "IOStatistics::onKextLoad: %s, tag %d, address 0x%llx, address end 0x%llx\n",
 218                 kext->getIdentifierCString(), kmod_info->id, (uint64_t)kmod_info->address, (uint64_t)(kmod_info->address + kmod_info->size));
 219
 220         ke = (KextNode *)kalloc(sizeof(KextNode));
 221         if (!ke) {
 222                 return;
 223         }
 224
 225         memset(ke, 0, sizeof(KextNode));
 226
 227         ke->kext = kext;
 228         ke->loadTag = kmod_info->id;
 229         ke->address = kmod_info->address;
 230         ke->address_end = kmod_info->address + kmod_info->size;
 231
 232         SLIST_INIT(&ke->classList);
 233         TAILQ_INIT(&ke->userClientCallList);
 234
 235         IORWLockWrite(lock);
 236
 237         RB_INSERT(KextTree, &kextHead, ke);
 238         RB_INSERT(KextAddressTree, &kextAddressHead, ke);
 239
 240         sequenceID++;
 241         loadedKexts++;
 242         lastKextIndex++;
 243
 244         IORWLockUnlock(lock);
 245 }
 246
 247 void IOStatistics::onKextUnload(OSKext *kext)
 248 {
 249         KextNode sought, *found;
 250
 251         assert(kext);
 252
 253         if (!enabled) {
 254                 return;
 255         }
 256
 257         LOG(1, "IOStatistics::onKextUnload: %s\n", kext->getIdentifierCString());
 258
 259         IORWLockWrite(lock);
 260
 261         sought.kext = kext;
 262         found = RB_FIND(KextTree, &kextHead, &sought);
 263         if (found) {
 264                 IOWorkLoopCounter *wlc;
 265                 IOUserClientProcessEntry *uce;
 266
 267                 /* Free up the list of counters */
 268                 while ((wlc = SLIST_FIRST(&found->workLoopList))) {
 269                         SLIST_REMOVE_HEAD(&found->workLoopList, link);
 270                         kfree(wlc, sizeof(IOWorkLoopCounter));
 271                 }
 272
 273                 /* Free up the user client list */
 274                 while ((uce = TAILQ_FIRST(&found->userClientCallList))) {
 275                         TAILQ_REMOVE(&found->userClientCallList, uce, link);
 276                         kfree(uce, sizeof(IOUserClientProcessEntry));
 277                 }
 278
 279                 /* Remove from kext trees */
 280                 RB_REMOVE(KextTree, &kextHead, found);
 281                 RB_REMOVE(KextAddressTree, &kextAddressHead, found);
 282
 283                 /*
 284                  * Clear a matching kextHint to avoid use after free in
 285                  * onClassAdded() for a class add after a KEXT unload.
 286                  */
 287                 if (found == kextHint) {
 288                         kextHint = NULL;
 289                 }
 290
 291                 /* Finally, free the class node */
 292                 kfree(found, sizeof(KextNode));
 293
 294                 sequenceID++;
 295                 loadedKexts--;
 296         }
 297         else {
 298                 panic("IOStatistics::onKextUnload: cannot find kext: %s", kext->getIdentifierCString());
 299         }
 300
 301         IORWLockUnlock(lock);
 302 }
 303
 304 void IOStatistics::onClassAdded(OSKext *parentKext, OSMetaClass *metaClass)
 305 {
 306         ClassNode *ce;
 307         KextNode soughtKext, *foundKext = NULL;
 308
 309         assert(parentKext && metaClass);
 310
 311         if (!enabled) {
 312                 return;
 313         }
 314
 315         LOG(1, "IOStatistics::onClassAdded: %s\n", metaClass->getClassName());
 316
 317         ce = (ClassNode *)kalloc(sizeof(ClassNode));
 318         if (!ce) {
 319                 return;
 320         }
 321
 322         memset(ce, 0, sizeof(ClassNode));
 323
 324         IORWLockWrite(lock);
 325
 326         /* Hinted? */
 327         if (kextHint && kextHint->kext == parentKext) {
 328                 foundKext = kextHint;
 329         }
 330         else {
 331                 soughtKext.kext = parentKext;
 332                 foundKext = RB_FIND(KextTree, &kextHead, &soughtKext);
 333         }
 334
 335         if (foundKext) {
 336                 ClassNode soughtClass, *foundClass = NULL;
 337                 const OSMetaClass *superClass;
 338
 339                 ce->metaClass = metaClass;
 340                 ce->classID = lastClassIndex++;
 341                 ce->parentKext = foundKext;
 342
 343                 /* Has superclass? */
 344                 superClass = ce->metaClass->getSuperClass();
 345                 if (superClass) {
 346                         soughtClass.metaClass = superClass;
 347                         foundClass = RB_FIND(ClassTree, &classHead, &soughtClass);
 348                 }
 349                 ce->superClassID = foundClass ? foundClass->classID : (uint32_t)(-1);
 350
 351                 SLIST_INIT(&ce->counterList);
 352                 SLIST_INIT(&ce->userClientList);
 353
 354                 RB_INSERT(ClassTree, &classHead, ce);
 355                 SLIST_INSERT_HEAD(&foundKext->classList, ce, lLink);
 356
 357                 foundKext->classes++;
 358
 359                 kextHint = foundKext;
 360
 361                 sequenceID++;
 362                 registeredClasses++;
 363         }
 364         else {
 365                 panic("IOStatistics::onClassAdded: cannot find parent kext: %s", parentKext->getIdentifierCString());
 366         }
 367
 368         IORWLockUnlock(lock);
 369 }
 370
 371 void IOStatistics::onClassRemoved(OSKext *parentKext, OSMetaClass *metaClass)
 372 {
 373         ClassNode sought, *found;
 374
 375         assert(parentKext && metaClass);
 376
 377         if (!enabled) {
 378                 return;
 379         }
 380
 381         LOG(1, "IOStatistics::onClassRemoved: %s\n", metaClass->getClassName());
 382
 383         IORWLockWrite(lock);
 384
 385         sought.metaClass = metaClass;
 386         found = RB_FIND(ClassTree, &classHead, &sought);
 387         if (found) {
 388                 IOEventSourceCounter *esc;
 389                 IOUserClientCounter *ucc;
 390
 391                 /* Free up the list of counters */
 392                 while ((esc = SLIST_FIRST(&found->counterList))) {
 393                         SLIST_REMOVE_HEAD(&found->counterList, link);
 394                         kfree(esc, sizeof(IOEventSourceCounter));
 395                 }
 396
 397                 /* Free up the user client list */
 398                 while ((ucc = SLIST_FIRST(&found->userClientList))) {
 399                         SLIST_REMOVE_HEAD(&found->userClientList, link);
 400                         kfree(ucc, sizeof(IOUserClientCounter));
 401                 }
 402
 403                 /* Remove from class tree */
 404                 RB_REMOVE(ClassTree, &classHead, found);
 405
 406                 /* Remove from parent */
 407                 SLIST_REMOVE(&found->parentKext->classList, found, ClassNode, lLink);
 408
 409                 /* Finally, free the class node */
 410                 kfree(found, sizeof(ClassNode));
 411
 412                 sequenceID++;
 413                 registeredClasses--;
 414         }
 415         else {
 416                 panic("IOStatistics::onClassRemoved: cannot find class: %s", metaClass->getClassName());
 417         }
 418
 419         IORWLockUnlock(lock);
 420 }
 421
 422 IOEventSourceCounter *IOStatistics::registerEventSource(OSObject *inOwner)
 423 {
 424         IOEventSourceCounter *counter = NULL;
 425         ClassNode sought, *found = NULL;
 426         boolean_t createDummyCounter = FALSE;
 427
 428         assert(inOwner);
 429
 430         if (!enabled) {
 431                 return NULL;
 432         }
 433
 434         counter = (IOEventSourceCounter*)kalloc(sizeof(IOEventSourceCounter));
 435         if (!counter) {
 436                 return NULL;
 437         }
 438
 439         memset(counter, 0, sizeof(IOEventSourceCounter));
 440
 441         IORWLockWrite(lock);
 442
 443         /* Workaround for <rdar://problem/7158117> - create a dummy counter when inOwner is bad.
 444          * We use retainCount here as our best indication that the pointer is awry.
 445          */
 446         if (inOwner->retainCount > 0xFFFFFF) {
 447                 kprintf("IOStatistics::registerEventSource - bad metaclass %p\n", inOwner);
 448                 createDummyCounter = TRUE;
 449         }
 450         else {
 451                 sought.metaClass = inOwner->getMetaClass();
 452                 found = RB_FIND(ClassTree, &classHead, &sought);
 453         }
 454
 455         if (found) {
 456                 counter->parentClass = found;
 457                 SLIST_INSERT_HEAD(&found->counterList, counter, link);
 458                 registeredCounters++;
 459         }
 460
 461         if (!(createDummyCounter || found)) {
 462                 panic("IOStatistics::registerEventSource: cannot find parent class: %s", inOwner->getMetaClass()->getClassName());
 463         }
 464
 465         IORWLockUnlock(lock);
 466
 467         return counter;
 468 }
 469
 470 void IOStatistics::unregisterEventSource(IOEventSourceCounter *counter)
 471 {
 472         if (!counter) {
 473                 return;
 474         }
 475
 476         IORWLockWrite(lock);
 477
 478         if (counter->parentClass) {
 479                 SLIST_REMOVE(&counter->parentClass->counterList, counter, IOEventSourceCounter, link);
 480                 registeredCounters--;
 481         }
 482         kfree(counter, sizeof(IOEventSourceCounter));
 483
 484         IORWLockUnlock(lock);
 485 }
 486
 487 IOWorkLoopCounter* IOStatistics::registerWorkLoop(IOWorkLoop *workLoop)
 488 {
 489         IOWorkLoopCounter *counter = NULL;
 490         KextNode *found;
 491
 492         assert(workLoop);
 493
 494         if (!enabled) {
 495                 return NULL;
 496         }
 497
 498         counter = (IOWorkLoopCounter*)kalloc(sizeof(IOWorkLoopCounter));
 499         if (!counter) {
 500                 return NULL;
 501         }
 502
 503         memset(counter, 0, sizeof(IOWorkLoopCounter));
 504
 505         found = getKextNodeFromBacktrace(TRUE);
 506         if (!found) {
 507                 panic("IOStatistics::registerWorkLoop: cannot find parent kext");
 508         }
 509
 510         counter->parentKext = found;
 511         counter->workLoop = workLoop;
 512         RB_INIT(&counter->dependencyHead);
 513         SLIST_INSERT_HEAD(&found->workLoopList, counter, link);
 514         registeredWorkloops++;
 515
 516         releaseKextNode(found);
 517
 518         return counter;
 519 }
 520
 521 void IOStatistics::unregisterWorkLoop(IOWorkLoopCounter *counter)
 522 {
 523         if (!counter) {
 524                 return;
 525         }
 526
 527         IORWLockWrite(lock);
 528
 529         SLIST_REMOVE(&counter->parentKext->workLoopList, counter, IOWorkLoopCounter, link);
 530         kfree(counter, sizeof(IOWorkLoopCounter));
 531         registeredWorkloops--;
 532
 533         IORWLockUnlock(lock);
 534 }
 535
 536 IOUserClientCounter *IOStatistics::registerUserClient(IOUserClient *userClient)
 537 {
 538         ClassNode sought, *found;
 539         IOUserClientCounter *counter = NULL;
 540
 541         assert(userClient);
 542
 543         if (!enabled) {
 544                 return NULL;
 545         }
 546
 547         counter = (IOUserClientCounter*)kalloc(sizeof(IOUserClientCounter));
 548         if (!counter) {
 549                 return NULL;
 550         }
 551
 552         memset(counter, 0, sizeof(IOUserClientCounter));
 553
 554         IORWLockWrite(lock);
 555
 556         sought.metaClass = userClient->getMetaClass();
 557
 558         found = RB_FIND(ClassTree, &classHead, &sought);
 559         if (found) {
 560                 counter->parentClass = found;
 561                 SLIST_INSERT_HEAD(&found->userClientList, counter, link);
 562         }
 563         else {
 564                 panic("IOStatistics::registerUserClient: cannot find parent class: %s", sought.metaClass->getClassName());
 565         }
 566
 567         IORWLockUnlock(lock);
 568
 569         return counter;
 570 }
 571
 572 void IOStatistics::unregisterUserClient(IOUserClientCounter *counter)
 573 {
 574         if (!counter) {
 575                 return;
 576         }
 577
 578         IORWLockWrite(lock);
 579
 580         SLIST_REMOVE(&counter->parentClass->userClientList, counter, IOUserClientCounter, link);
 581         kfree(counter, sizeof(IOUserClientCounter));
 582
 583         IORWLockUnlock(lock);
 584 }
 585
 586 void IOStatistics::attachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
 587 {
 588         if (!wlc) {
 589         return;
 590         }
 591
 592         IORWLockWrite(lock);
 593
 594         if (!nextWorkLoopDependency) {
 595                 return;
 596         }
 597
 598         attachedEventSources++;
 599         wlc->attachedEventSources++;
 600
 601         /* Track the kext dependency */
 602         nextWorkLoopDependency->loadTag = esc->parentClass->parentKext->loadTag;
 603         if (NULL == RB_INSERT(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, nextWorkLoopDependency)) {
 604                 nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
 605         }
 606
 607         IORWLockUnlock(lock);
 608 }
 609
 610 void IOStatistics::detachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
 611 {
 612         IOWorkLoopDependency sought, *found;
 613
 614         if (!wlc) {
 615                 return;
 616         }
 617
 618         IORWLockWrite(lock);
 619
 620         attachedEventSources--;
 621         wlc->attachedEventSources--;
 622
 623         sought.loadTag = esc->parentClass->parentKext->loadTag;
 624
 625         found = RB_FIND(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, &sought);
 626         if (found) {
 627                 RB_REMOVE(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, found);
 628                 kfree(found, sizeof(IOWorkLoopDependency));
 629         }
 630
 631         IORWLockUnlock(lock);
 632 }
 633
 634 int IOStatistics::getStatistics(sysctl_req *req)
 635 {
 636         int error;
 637         uint32_t calculatedSize, size;
 638         char *buffer, *ptr;
 639         IOStatisticsHeader *header;
 640
 641         assert(IOStatistics::enabled && req);
 642
 643         IORWLockRead(IOStatistics::lock);
 644
 645         /* Work out how much we need to allocate. IOStatisticsKext is of variable size. */
 646         calculatedSize = sizeof(IOStatisticsHeader) +
 647                                          sizeof(IOStatisticsGlobal) +
 648                                         (sizeof(IOStatisticsKext) * loadedKexts) + (sizeof(uint32_t) * registeredClasses) +
 649                                         (sizeof(IOStatisticsMemory) * loadedKexts) +
 650                                         (sizeof(IOStatisticsClass) * registeredClasses) +
 651                                         (sizeof(IOStatisticsCounter) * registeredClasses) +
 652                                         (sizeof(IOStatisticsKextIdentifier) * loadedKexts) +
 653                                         (sizeof(IOStatisticsClassName) * registeredClasses);
 654
 655         /* Size request? */
 656         if (req->oldptr == USER_ADDR_NULL) {
 657                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 658                 goto exit;
 659         }
 660
 661         /* Read only */
 662         if (req->newptr != USER_ADDR_NULL) {
 663                 error = EPERM;
 664                 goto exit;
 665         }
 666
 667         buffer = (char*)kalloc(calculatedSize);
 668         if (!buffer) {
 669                 error = ENOMEM;
 670                 goto exit;
 671         }
 672
 673         memset(buffer, 0, calculatedSize);
 674
 675         ptr = buffer;
 676
 677         header = (IOStatisticsHeader*)((void*)ptr);
 678
 679         header->sig = IOSTATISTICS_SIG;
 680         header->ver = IOSTATISTICS_VER;
 681
 682         header->seq = sequenceID;
 683
 684         ptr += sizeof(IOStatisticsHeader);
 685
 686         /* Global data - seq, timers, interrupts, etc) */
 687         header->globalStatsOffset = sizeof(IOStatisticsHeader);
 688         size = copyGlobalStatistics((IOStatisticsGlobal*)((void*)ptr));
 689         ptr += size;
 690
 691         /* Kext statistics */
 692         header->kextStatsOffset = header->globalStatsOffset + size;
 693         size = copyKextStatistics((IOStatisticsKext*)((void*)ptr));
 694         ptr += size;
 695
 696         /* Memory allocation info */
 697         header->memoryStatsOffset = header->kextStatsOffset + size;
 698         size = copyMemoryStatistics((IOStatisticsMemory*)((void*)ptr));
 699         ptr += size;
 700
 701         /* Class statistics */
 702         header->classStatsOffset = header->memoryStatsOffset + size;
 703         size = copyClassStatistics((IOStatisticsClass*)((void*)ptr));
 704         ptr += size;
 705
 706         /* Dynamic class counter data */
 707         header->counterStatsOffset = header->classStatsOffset + size;
 708         size = copyCounterStatistics((IOStatisticsCounter*)((void*)ptr));
 709         ptr += size;
 710
 711         /* Kext identifiers */
 712         header->kextIdentifiersOffset = header->counterStatsOffset + size;
 713         size = copyKextIdentifiers((IOStatisticsKextIdentifier*)((void*)ptr));
 714         ptr += size;
 715
 716         /* Class names */
 717         header->classNamesOffset = header->kextIdentifiersOffset + size;
 718         size = copyClassNames((IOStatisticsClassName*)ptr);
 719         ptr += size;
 720
 721         LOG(2, "IOStatistics::getStatistics - calculatedSize 0x%x, kexts 0x%x, classes 0x%x.\n",
 722                 calculatedSize, loadedKexts, registeredClasses);
 723
 724         assert( (uint32_t)(ptr - buffer) == calculatedSize );
 725
 726         error = SYSCTL_OUT(req, buffer, calculatedSize);
 727
 728         kfree(buffer, calculatedSize);
 729
 730 exit:
 731         IORWLockUnlock(IOStatistics::lock);
 732         return error;
 733 }
 734
 735 int IOStatistics::getWorkLoopStatistics(sysctl_req *req)
 736 {
 737         int error;
 738         uint32_t calculatedSize, size;
 739         char *buffer;
 740         IOStatisticsWorkLoopHeader *header;
 741
 742         assert(IOStatistics::enabled && req);
 743
 744         IORWLockRead(IOStatistics::lock);
 745
 746         /* Approximate how much we need to allocate (worse case estimate) */
 747         calculatedSize = sizeof(IOStatisticsWorkLoop) * registeredWorkloops +
 748                                          sizeof(uint32_t) * attachedEventSources;
 749
 750         /* Size request? */
 751         if (req->oldptr == USER_ADDR_NULL) {
 752                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 753                 goto exit;
 754         }
 755
 756         /* Read only */
 757         if (req->newptr != USER_ADDR_NULL) {
 758                 error = EPERM;
 759                 goto exit;
 760         }
 761
 762         buffer = (char*)kalloc(calculatedSize);
 763         if (!buffer) {
 764                 error = ENOMEM;
 765                 goto exit;
 766         }
 767
 768         header = (IOStatisticsWorkLoopHeader*)((void*)buffer);
 769
 770         header->sig = IOSTATISTICS_SIG_WORKLOOP;
 771         header->ver = IOSTATISTICS_VER;
 772
 773         header->seq = sequenceID;
 774
 775         header->workloopCount = registeredWorkloops;
 776
 777         size = copyWorkLoopStatistics(&header->workLoopStats);
 778
 779         LOG(2, "IOStatistics::getWorkLoopStatistics: calculatedSize %d, size %d\n", calculatedSize, size);
 780
 781         assert( size <= calculatedSize );
 782
 783         error = SYSCTL_OUT(req, buffer, size);
 784
 785         kfree(buffer, calculatedSize);
 786
 787 exit:
 788         IORWLockUnlock(IOStatistics::lock);
 789         return error;
 790 }
 791
 792 int IOStatistics::getUserClientStatistics(sysctl_req *req)
 793 {
 794         int error;
 795         uint32_t calculatedSize, size;
 796         char *buffer;
 797         uint32_t requestedLoadTag = 0;
 798         IOStatisticsUserClientHeader *header;
 799
 800         assert(IOStatistics::enabled && req);
 801
 802         IORWLockRead(IOStatistics::lock);
 803
 804         /* Work out how much we need to allocate */
 805         calculatedSize = sizeof(IOStatisticsUserClientHeader) +
 806                                          sizeof(IOStatisticsUserClientCall) * IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS * loadedKexts;
 807
 808         /* Size request? */
 809         if (req->oldptr == USER_ADDR_NULL) {
 810                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 811                 goto exit;
 812         }
 813
 814         /* Kext request (potentially) valid? */
 815         if (!req->newptr || req->newlen < sizeof(requestedLoadTag)) {
 816                 error = EINVAL;
 817                 goto exit;
 818         }
 819
 820         SYSCTL_IN(req, &requestedLoadTag, sizeof(requestedLoadTag));
 821
 822         LOG(2, "IOStatistics::getUserClientStatistics - requesting kext w/load tag: %d\n", requestedLoadTag);
 823
 824         buffer = (char*)kalloc(calculatedSize);
 825         if (!buffer) {
 826                 error = ENOMEM;
 827                 goto exit;
 828         }
 829
 830         header = (IOStatisticsUserClientHeader*)((void*)buffer);
 831
 832         header->sig = IOSTATISTICS_SIG_USERCLIENT;
 833         header->ver = IOSTATISTICS_VER;
 834
 835         header->seq = sequenceID;
 836
 837         header->processes = 0;
 838
 839         size = copyUserClientStatistics(header, requestedLoadTag);
 840
 841         assert((sizeof(IOStatisticsUserClientHeader) + size) <= calculatedSize);
 842
 843         if (size) {
 844                 error = SYSCTL_OUT(req, buffer, sizeof(IOStatisticsUserClientHeader) + size);
 845         }
 846         else {
 847                 error = EINVAL;
 848         }
 849
 850         kfree(buffer, calculatedSize);
 851
 852 exit:
 853         IORWLockUnlock(IOStatistics::lock);
 854         return error;
 855 }
 856
 857 uint32_t IOStatistics::copyGlobalStatistics(IOStatisticsGlobal *stats)
 858 {
 859         stats->kextCount = loadedKexts;
 860         stats->classCount = registeredClasses;
 861         stats->workloops = registeredWorkloops;
 862
 863         return sizeof(IOStatisticsGlobal);
 864 }
 865
 866 uint32_t IOStatistics::copyKextStatistics(IOStatisticsKext *stats)
 867 {
 868         KextNode *ke;
 869         ClassNode *ce;
 870         uint32_t index = 0;
 871
 872         RB_FOREACH(ke, KextTree, &kextHead) {
 873                 stats->loadTag = ke->loadTag;
 874                 ke->kext->getSizeInfo(&stats->loadSize, &stats->wiredSize);
 875
 876                 stats->classes = ke->classes;
 877
 878                 /* Append indices of owned classes */
 879                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 880                         stats->classIndexes[index++] = ce->classID;
 881                 }
 882
 883                 stats = (IOStatisticsKext *)((void*)((char*)stats + sizeof(IOStatisticsKext) + (ke->classes * sizeof(uint32_t))));
 884         }
 885
 886         return (sizeof(IOStatisticsKext) * loadedKexts + sizeof(uint32_t) * registeredClasses);
 887 }
 888
 889 uint32_t IOStatistics::copyMemoryStatistics(IOStatisticsMemory *stats)
 890 {
 891         KextNode *ke;
 892
 893         RB_FOREACH(ke, KextTree, &kextHead) {
 894                 stats->allocatedSize = ke->memoryCounters[kIOStatisticsMalloc];
 895                 stats->freedSize = ke->memoryCounters[kIOStatisticsFree];
 896                 stats->allocatedAlignedSize = ke->memoryCounters[kIOStatisticsMallocAligned];
 897                 stats->freedAlignedSize = ke->memoryCounters[kIOStatisticsFreeAligned];
 898                 stats->allocatedContiguousSize = ke->memoryCounters[kIOStatisticsMallocContiguous];
 899                 stats->freedContiguousSize = ke->memoryCounters[kIOStatisticsFreeContiguous];
 900                 stats->allocatedPageableSize = ke->memoryCounters[kIOStatisticsMallocPageable];
 901                 stats->freedPageableSize = ke->memoryCounters[kIOStatisticsFreePageable];
 902                 stats++;
 903         }
 904
 905         return (sizeof(IOStatisticsMemory) * loadedKexts);
 906 }
 907
 908 uint32_t IOStatistics::copyClassStatistics(IOStatisticsClass *stats)
 909 {
 910         KextNode *ke;
 911         ClassNode *ce;
 912
 913         RB_FOREACH(ke, KextTree, &kextHead) {
 914                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 915                         stats->classID = ce->classID;
 916                         stats->superClassID = ce->superClassID;
 917                         stats->classSize = ce->metaClass->getClassSize();
 918
 919                         stats++;
 920                 }
 921         }
 922
 923         return sizeof(IOStatisticsClass) * registeredClasses;
 924 }
 925
 926 uint32_t IOStatistics::copyCounterStatistics(IOStatisticsCounter *stats)
 927 {
 928         KextNode *ke;
 929         ClassNode *ce;
 930
 931         RB_FOREACH(ke, KextTree, &kextHead) {
 932                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 933                         IOUserClientCounter *userClientCounter;
 934                         IOEventSourceCounter *counter;
 935
 936                         stats->classID = ce->classID;
 937                         stats->classInstanceCount = ce->metaClass->getInstanceCount();
 938
 939                         IOStatisticsUserClients *uc = &stats->userClientStatistics;
 940
 941                         /* User client counters */
 942                         SLIST_FOREACH(userClientCounter, &ce->userClientList, link) {
 943                                 uc->clientCalls += userClientCounter->clientCalls;
 944                                 uc->created++;
 945                         }
 946
 947                         IOStatisticsInterruptEventSources *iec = &stats->interruptEventSourceStatistics;
 948                         IOStatisticsInterruptEventSources *fiec = &stats->filterInterruptEventSourceStatistics;
 949                         IOStatisticsTimerEventSources *tec = &stats->timerEventSourceStatistics;
 950                         IOStatisticsCommandGates *cgc = &stats->commandGateStatistics;
 951                         IOStatisticsCommandQueues *cqc = &stats->commandQueueStatistics;
 952                         IOStatisticsDerivedEventSources *dec = &stats->derivedEventSourceStatistics;
 953
 954                         /* Event source counters */
 955                         SLIST_FOREACH(counter, &ce->counterList, link) {
 956                                 switch (counter->type) {
 957                                         case kIOStatisticsInterruptEventSourceCounter:
 958                                                 iec->created++;
 959                                                 iec->produced += counter->u.interrupt.produced;
 960                                                 iec->checksForWork += counter->u.interrupt.checksForWork;
 961                                                 break;
 962                                         case kIOStatisticsFilterInterruptEventSourceCounter:
 963                                                 fiec->created++;
 964                                                 fiec->produced += counter->u.filter.produced;
 965                                                 fiec->checksForWork += counter->u.filter.checksForWork;
 966                                                 break;
 967                                         case kIOStatisticsTimerEventSourceCounter:
 968                                                 tec->created++;
 969                                                 tec->timeouts += counter->u.timer.timeouts;
 970                                                 tec->checksForWork += counter->u.timer.checksForWork;
 971                                                 tec->timeOnGate += counter->timeOnGate;
 972                                                 tec->closeGateCalls += counter->closeGateCalls;
 973                                                 tec->openGateCalls += counter->openGateCalls;
 974                                                 break;
 975                                         case kIOStatisticsCommandGateCounter:
 976                                                 cgc->created++;
 977                                                 cgc->timeOnGate += counter->timeOnGate;
 978                                                 cgc->actionCalls += counter->u.commandGate.actionCalls;
 979                                                 break;
 980                                         case kIOStatisticsCommandQueueCounter:
 981                                                 cqc->created++;
 982                                                 cqc->actionCalls += counter->u.commandQueue.actionCalls;
 983                                                 break;
 984                                         case kIOStatisticsDerivedEventSourceCounter:
 985                                                 dec->created++;
 986                                                 dec->timeOnGate += counter->timeOnGate;
 987                                                 dec->closeGateCalls += counter->closeGateCalls;
 988                                                 dec->openGateCalls += counter->openGateCalls;
 989                                                 break;
 990                                         default:
 991                                                 break;
 992                                 }
 993                         }
 994
 995                         stats++;
 996                 }
 997         }
 998
 999         return sizeof(IOStatisticsCounter) * registeredClasses;
1000 }
1001
1002 uint32_t IOStatistics::copyKextIdentifiers(IOStatisticsKextIdentifier *kextIDs)
1003 {
1004         KextNode *ke;
1005
1006         RB_FOREACH(ke, KextTree, &kextHead) {
1007                 strncpy(kextIDs->identifier, ke->kext->getIdentifierCString(), kIOStatisticsDriverNameLength);
1008                 kextIDs++;
1009         }
1010
1011         return (sizeof(IOStatisticsKextIdentifier) * loadedKexts);
1012 }
1013
1014 uint32_t IOStatistics::copyClassNames(IOStatisticsClassName *classNames)
1015 {
1016         KextNode *ke;
1017         ClassNode *ce;
1018
1019         RB_FOREACH(ke, KextTree, &kextHead) {
1020                 SLIST_FOREACH(ce, &ke->classList, lLink) {
1021                         strncpy(classNames->name, ce->metaClass->getClassName(), kIOStatisticsClassNameLength);
1022                         classNames++;
1023                 }
1024         }
1025
1026         return (sizeof(IOStatisticsClassName) * registeredClasses);
1027 }
1028
1029 uint32_t IOStatistics::copyWorkLoopStatistics(IOStatisticsWorkLoop *stats)
1030 {
1031         KextNode *ke;
1032         IOWorkLoopCounter *wlc;
1033         IOWorkLoopDependency *dependentNode;
1034         uint32_t size, accumulatedSize = 0;
1035
1036         RB_FOREACH(ke, KextTree, &kextHead) {
1037                 SLIST_FOREACH(wlc, &ke->workLoopList, link) {
1038                         stats->kextLoadTag = ke->loadTag;
1039                         stats->attachedEventSources = wlc->attachedEventSources;
1040                         stats->timeOnGate = wlc->timeOnGate;
1041                         stats->dependentKexts = 0;
1042                         RB_FOREACH(dependentNode, IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead) {
1043                                 stats->dependentKextLoadTags[stats->dependentKexts] = dependentNode->loadTag;
1044                                 stats->dependentKexts++;
1045                         }
1046
1047                         size = sizeof(IOStatisticsWorkLoop) + (sizeof(uint32_t) * stats->dependentKexts);
1048
1049                         accumulatedSize += size;
1050                         stats = (IOStatisticsWorkLoop*)((void*)((char*)stats + size));
1051                 }
1052         }
1053
1054         return accumulatedSize;
1055 }
1056
1057 uint32_t IOStatistics::copyUserClientStatistics(IOStatisticsUserClientHeader *stats, uint32_t loadTag)
1058 {
1059         KextNode *sought, *found = NULL;
1060         uint32_t procs = 0;
1061         IOUserClientProcessEntry *processEntry;
1062
1063         RB_FOREACH(sought, KextTree, &kextHead) {
1064                 if (sought->loadTag == loadTag) {
1065                         found = sought;
1066                         break;
1067                 }
1068         }
1069
1070         if (!found) {
1071                 return 0;
1072         }
1073
1074         TAILQ_FOREACH(processEntry, &found->userClientCallList, link) {
1075                 strncpy(stats->userClientCalls[procs].processName, processEntry->processName, kIOStatisticsProcessNameLength);
1076                 stats->userClientCalls[procs].pid = processEntry->pid;
1077                 stats->userClientCalls[procs].calls = processEntry->calls;
1078                 stats->processes++;
1079                 procs++;
1080         }
1081
1082         return sizeof(IOStatisticsUserClientCall) * stats->processes;
1083 }
1084
1085 void IOStatistics::storeUserClientCallInfo(IOUserClient *userClient, IOUserClientCounter *counter)
1086 {
1087         OSString *ossUserClientCreator = NULL;
1088         int32_t pid = -1;
1089         KextNode *parentKext;
1090         IOUserClientProcessEntry *entry, *nextEntry, *prevEntry = NULL;
1091         uint32_t count = 0;
1092         const char *ptr = NULL;
1093         OSObject *obj;
1094
1095         /* TODO: see if this can be more efficient */
1096         obj = userClient->copyProperty("IOUserClientCreator",
1097                                         gIOServicePlane,
1098                                         kIORegistryIterateRecursively | kIORegistryIterateParents);
1099
1100         if (!obj)
1101                 goto err_nounlock;
1102
1103         ossUserClientCreator = OSDynamicCast(OSString, obj);
1104
1105         if (ossUserClientCreator) {
1106                 uint32_t len, lenIter = 0;
1107
1108                 ptr = ossUserClientCreator->getCStringNoCopy();
1109                 len = ossUserClientCreator->getLength();
1110
1111                 while ((*ptr != ' ') && (lenIter < len)) {
1112                         ptr++;
1113                         lenIter++;
1114                 }
1115
1116                 if (lenIter < len) {
1117                         ptr++; // Skip the space
1118                         lenIter++;
1119                         pid = 0;
1120                         while ( (*ptr != ',') && (lenIter < len)) {
1121                                 pid = pid*10 + (*ptr - '0');
1122                                 ptr++;
1123                                 lenIter++;
1124                         }
1125
1126                         if(lenIter == len) {
1127                                 pid = -1;
1128                         } else {
1129                                 ptr += 2;
1130                         }
1131                 }
1132         }
1133
1134         if (-1 == pid)
1135                 goto err_nounlock;
1136
1137         IORWLockWrite(lock);
1138
1139         parentKext = counter->parentClass->parentKext;
1140
1141         TAILQ_FOREACH(entry, &parentKext->userClientCallList, link) {
1142                 if (entry->pid == pid) {
1143                         /* Found, so increment count and move to the head */
1144                         entry->calls++;
1145                         if (count) {
1146                                 TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1147                                 break;
1148                         }
1149                         else {
1150                                 /* At the head already, so increment and return */
1151                                 goto err_unlock;
1152                         }
1153                 }
1154
1155                 count++;
1156         }
1157
1158         if (!entry) {
1159                 if (count == IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS) {
1160                         /* Max elements hit, so reuse the last */
1161                         entry = TAILQ_LAST(&parentKext->userClientCallList, ProcessEntryList);
1162                         TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1163                 }
1164                 else {
1165                         /* Otherwise, allocate a new entry */
1166                         entry = (IOUserClientProcessEntry*)kalloc(sizeof(IOUserClientProcessEntry));
1167                         if (!entry) {
1168                             IORWLockUnlock(lock);
1169                                 return;
1170                         }
1171                 }
1172
1173                 strncpy(entry->processName, ptr, kIOStatisticsProcessNameLength);
1174                 entry->pid = pid;
1175                 entry->calls = 1;
1176         }
1177
1178         TAILQ_FOREACH(nextEntry, &parentKext->userClientCallList, link) {
1179                 if (nextEntry->calls <= entry->calls)
1180                         break;
1181
1182                 prevEntry = nextEntry;
1183         }
1184
1185         if (!prevEntry)
1186                 TAILQ_INSERT_HEAD(&parentKext->userClientCallList, entry, link);
1187         else
1188                 TAILQ_INSERT_AFTER(&parentKext->userClientCallList, prevEntry, entry, link);
1189
1190 err_unlock:
1191         IORWLockUnlock(lock);
1192
1193 err_nounlock:
1194         if (obj)
1195                 obj->release();
1196 }
1197
1198 void IOStatistics::countUserClientCall(IOUserClient *client) {
1199         IOUserClient::ExpansionData *data;
1200         IOUserClientCounter *counter;
1201
1202         /* Guard against an uninitialized client object - <rdar://problem/8577946> */
1203         if (!(data = client->reserved)) {
1204                 return;
1205         }
1206
1207         if ((counter = data->counter)) {
1208                 storeUserClientCallInfo(client, counter);
1209                 OSIncrementAtomic(&counter->clientCalls);
1210         }
1211 }
1212
1213 KextNode *IOStatistics::getKextNodeFromBacktrace(boolean_t write) {
1214         const uint32_t btMin = 3;
1215
1216         void *bt[16];
1217         unsigned btCount = sizeof(bt) / sizeof(bt[0]);
1218         vm_offset_t *scanAddr = NULL;
1219         uint32_t i;
1220         KextNode *found = NULL, *ke = NULL;
1221
1222         btCount = OSBacktrace(bt, btCount);
1223
1224         if (write) {
1225                 IORWLockWrite(lock);
1226         } else {
1227                 IORWLockRead(lock);
1228         }
1229
1230         /* Ignore first levels */
1231         scanAddr = (vm_offset_t *)&bt[btMin - 1];
1232
1233         for (i = 0; i < btCount; i++, scanAddr++) {
1234                 ke = RB_ROOT(&kextAddressHead);
1235                 while (ke) {
1236                         if (*scanAddr < ke->address) {
1237                                 ke = RB_LEFT(ke, addressLink);
1238                         }
1239                         else {
1240                                 if ((*scanAddr < ke->address_end) && (*scanAddr >= ke->address)) {
1241                                         if (!ke->kext->isKernelComponent()) {
1242                                                 return ke;
1243                                         } else {
1244                                                 found = ke;
1245                                         }
1246                                 }
1247                                 ke = RB_RIGHT(ke, addressLink);
1248                         }
1249                 }
1250         }
1251
1252         if (!found) {
1253                 IORWLockUnlock(lock);
1254         }
1255
1256         return found;
1257 }
1258
1259 void IOStatistics::releaseKextNode(KextNode *node) {
1260 #pragma unused(node)
1261         IORWLockUnlock(lock);
1262 }
1263
1264 /* IOLib allocations */
1265 void IOStatistics::countAlloc(uint32_t index, vm_size_t size) {
1266         KextNode *ke;
1267
1268         if (!enabled) {
1269                 return;
1270         }
1271
1272         ke = getKextNodeFromBacktrace(FALSE);
1273         if (ke) {
1274                 OSAddAtomic(size, &ke->memoryCounters[index]);
1275                 releaseKextNode(ke);
1276         }
1277 }
1278
1279 #endif /* IOKITSTATS */