iokit/Kernel/IOStatistics.cpp

   1 /*
   2  * Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/sysctl.h>
  30 #include <kern/host.h>
  31
  32 #include <IOKit/system.h>
  33 #include <libkern/c++/OSKext.h>
  34 #include <libkern/OSAtomic.h>
  35
  36 #include <IOKit/IOStatisticsPrivate.h>
  37 #include <IOKit/IOUserClient.h>
  38 #include <IOKit/IOEventSource.h>
  39 #include <IOKit/IOKitDebug.h>
  40
  41 #if IOKITSTATS
  42 bool IOStatistics::enabled = false;
  43
  44 uint32_t IOStatistics::sequenceID = 0;
  45
  46 uint32_t IOStatistics::lastClassIndex = 0;
  47 uint32_t IOStatistics::lastKextIndex = 0;
  48
  49 uint32_t IOStatistics::loadedKexts = 0;
  50 uint32_t IOStatistics::registeredClasses = 0;
  51 uint32_t IOStatistics::registeredCounters = 0;
  52 uint32_t IOStatistics::registeredWorkloops = 0;
  53
  54 uint32_t IOStatistics::attachedEventSources = 0;
  55
  56 IOWorkLoopDependency *IOStatistics::nextWorkLoopDependency = NULL;
  57
  58 /* Logging */
  59
  60 #define LOG_LEVEL 0
  61
  62 #define LOG(level, format, ...) \
  63 do { \
  64         if (level <= LOG_LEVEL) \
  65                 printf(format, ##__VA_ARGS__); \
  66 } while (0)
  67
  68 /* Locks */
  69
  70 IORWLock *IOStatistics::lock = NULL;
  71
  72 /* Kext tree */
  73
  74 KextNode *IOStatistics::kextHint = NULL;
  75
  76 IOStatistics::KextTreeHead IOStatistics::kextHead = RB_INITIALIZER(&IOStatistics::kextHead);
  77
  78 int IOStatistics::kextNodeCompare(KextNode *e1, KextNode *e2)
  79 {
  80     if (e1->kext < e2->kext)
  81         return -1;
  82     else if (e1->kext > e2->kext)
  83         return 1;
  84     else
  85         return 0;
  86 }
  87
  88 RB_GENERATE(IOStatistics::KextTree, KextNode, link, kextNodeCompare);
  89
  90 /* Kext tree ordered by address */
  91
  92 IOStatistics::KextAddressTreeHead IOStatistics::kextAddressHead = RB_INITIALIZER(&IOStatistics::kextAddressHead);
  93
  94 int IOStatistics::kextAddressNodeCompare(KextNode *e1, KextNode *e2)
  95 {
  96     if (e1->address < e2->address)
  97         return -1;
  98     else if (e1->address > e2->address)
  99         return 1;
 100     else
 101         return 0;
 102 }
 103
 104 RB_GENERATE(IOStatistics::KextAddressTree, KextNode, addressLink, kextAddressNodeCompare);
 105
 106 /* Class tree */
 107
 108 IOStatistics::ClassTreeHead IOStatistics::classHead = RB_INITIALIZER(&IOStatistics::classHead);
 109
 110 int IOStatistics::classNodeCompare(ClassNode *e1, ClassNode *e2) {
 111     if (e1->metaClass < e2->metaClass)
 112         return -1;
 113     else if (e1->metaClass > e2->metaClass)
 114         return 1;
 115     else
 116         return 0;
 117 }
 118
 119 RB_GENERATE(IOStatistics::ClassTree, ClassNode, tLink, classNodeCompare);
 120
 121 /* Workloop dependencies */
 122
 123 int IOWorkLoopCounter::loadTagCompare(IOWorkLoopDependency *e1, IOWorkLoopDependency *e2) {
 124     if (e1->loadTag < e2->loadTag)
 125         return -1;
 126     else if (e1->loadTag > e2->loadTag)
 127         return 1;
 128     else
 129         return 0;
 130 }
 131
 132 RB_GENERATE(IOWorkLoopCounter::DependencyTree, IOWorkLoopDependency, link, IOWorkLoopCounter::loadTagCompare);
 133
 134 /* sysctl stuff */
 135
 136 static int
 137 oid_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1, int arg2, struct sysctl_req *req)
 138 {
 139         int error = EINVAL;
 140         uint32_t request = arg2;
 141
 142         switch (request)
 143         {
 144                 case kIOStatisticsGeneral:
 145                         error = IOStatistics::getStatistics(req);
 146                         break;
 147                 case kIOStatisticsWorkLoop:
 148                         error = IOStatistics::getWorkLoopStatistics(req);
 149                         break;
 150                 case kIOStatisticsUserClient:
 151                         error = IOStatistics::getUserClientStatistics(req);
 152                         break;
 153                 default:
 154                         break;
 155         }
 156
 157         return error;
 158 }
 159
 160 SYSCTL_NODE(_debug, OID_AUTO, iokit_statistics, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "IOStatistics");
 161
 162 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, general,
 163             CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 164             0, kIOStatisticsGeneral, oid_sysctl, "S", "");
 165
 166 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, workloop,
 167             CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 168             0, kIOStatisticsWorkLoop, oid_sysctl, "S", "");
 169
 170 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, userclient,
 171             CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 172             0, kIOStatisticsUserClient, oid_sysctl, "S", "");
 173
 174 void IOStatistics::initialize()
 175 {
 176         if (enabled) {
 177                 return;
 178         }
 179
 180         /* Only enabled if the boot argument is set. */
 181         if (!(kIOStatistics & gIOKitDebug)) {
 182                 return;
 183         }
 184
 185         sysctl_register_oid(&sysctl__debug_iokit_statistics_general);
 186         sysctl_register_oid(&sysctl__debug_iokit_statistics_workloop);
 187         sysctl_register_oid(&sysctl__debug_iokit_statistics_userclient);
 188
 189         lock = IORWLockAlloc();
 190         if (!lock) {
 191                 return;
 192         }
 193
 194         nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
 195         if (!nextWorkLoopDependency) {
 196                 return;
 197         }
 198
 199         enabled = true;
 200 }
 201
 202 void IOStatistics::onKextLoad(OSKext *kext, kmod_info_t *kmod_info)
 203 {
 204         KextNode *ke;
 205
 206         assert(kext && kmod_info);
 207
 208         if (!enabled) {
 209                 return;
 210         }
 211
 212         LOG(1, "IOStatistics::onKextLoad: %s, tag %d, address 0x%llx, address end 0x%llx\n",
 213                 kext->getIdentifierCString(), kmod_info->id, (uint64_t)kmod_info->address, (uint64_t)(kmod_info->address + kmod_info->size));
 214
 215         ke = (KextNode *)kalloc(sizeof(KextNode));
 216         if (!ke) {
 217                 return;
 218         }
 219
 220         memset(ke, 0, sizeof(KextNode));
 221
 222         ke->kext = kext;
 223         ke->loadTag = kmod_info->id;
 224         ke->address = kmod_info->address;
 225         ke->address_end = kmod_info->address + kmod_info->size;
 226
 227         SLIST_INIT(&ke->classList);
 228         TAILQ_INIT(&ke->userClientCallList);
 229
 230         IORWLockWrite(lock);
 231
 232         RB_INSERT(KextTree, &kextHead, ke);
 233         RB_INSERT(KextAddressTree, &kextAddressHead, ke);
 234
 235         sequenceID++;
 236         loadedKexts++;
 237         lastKextIndex++;
 238
 239         IORWLockUnlock(lock);
 240 }
 241
 242 void IOStatistics::onKextUnload(OSKext *kext)
 243 {
 244         KextNode sought, *found;
 245
 246         assert(kext);
 247
 248         if (!enabled) {
 249                 return;
 250         }
 251
 252         LOG(1, "IOStatistics::onKextUnload: %s\n", kext->getIdentifierCString());
 253
 254         IORWLockWrite(lock);
 255
 256         sought.kext = kext;
 257         found = RB_FIND(KextTree, &kextHead, &sought);
 258         if (found) {
 259                 IOWorkLoopCounter *wlc;
 260                 IOUserClientProcessEntry *uce;
 261
 262                 /* Free up the list of counters */
 263                 while ((wlc = SLIST_FIRST(&found->workLoopList))) {
 264                         SLIST_REMOVE_HEAD(&found->workLoopList, link);
 265                         kfree(wlc, sizeof(IOWorkLoopCounter));
 266                 }
 267
 268                 /* Free up the user client list */
 269                 while ((uce = TAILQ_FIRST(&found->userClientCallList))) {
 270                         TAILQ_REMOVE(&found->userClientCallList, uce, link);
 271                         kfree(uce, sizeof(IOUserClientProcessEntry));
 272                 }
 273
 274                 /* Remove from kext trees */
 275                 RB_REMOVE(KextTree, &kextHead, found);
 276                 RB_REMOVE(KextAddressTree, &kextAddressHead, found);
 277
 278                 /*
 279                  * Clear a matching kextHint to avoid use after free in
 280                  * onClassAdded() for a class add after a KEXT unload.
 281                  */
 282                 if (found == kextHint) {
 283                         kextHint = NULL;
 284                 }
 285
 286                 /* Finally, free the class node */
 287                 kfree(found, sizeof(KextNode));
 288
 289                 sequenceID++;
 290                 loadedKexts--;
 291         }
 292         else {
 293                 panic("IOStatistics::onKextUnload: cannot find kext: %s", kext->getIdentifierCString());
 294         }
 295
 296         IORWLockUnlock(lock);
 297 }
 298
 299 void IOStatistics::onClassAdded(OSKext *parentKext, OSMetaClass *metaClass)
 300 {
 301         ClassNode *ce;
 302         KextNode soughtKext, *foundKext = NULL;
 303
 304         assert(parentKext && metaClass);
 305
 306         if (!enabled) {
 307                 return;
 308         }
 309
 310         LOG(1, "IOStatistics::onClassAdded: %s\n", metaClass->getClassName());
 311
 312         ce = (ClassNode *)kalloc(sizeof(ClassNode));
 313         if (!ce) {
 314                 return;
 315         }
 316
 317         memset(ce, 0, sizeof(ClassNode));
 318
 319         IORWLockWrite(lock);
 320
 321         /* Hinted? */
 322         if (kextHint && kextHint->kext == parentKext) {
 323                 foundKext = kextHint;
 324         }
 325         else {
 326                 soughtKext.kext = parentKext;
 327                 foundKext = RB_FIND(KextTree, &kextHead, &soughtKext);
 328         }
 329
 330         if (foundKext) {
 331                 ClassNode soughtClass, *foundClass = NULL;
 332                 const OSMetaClass *superClass;
 333
 334                 ce->metaClass = metaClass;
 335                 ce->classID = lastClassIndex++;
 336                 ce->parentKext = foundKext;
 337
 338                 /* Has superclass? */
 339                 superClass = ce->metaClass->getSuperClass();
 340                 if (superClass) {
 341                         soughtClass.metaClass = superClass;
 342                         foundClass = RB_FIND(ClassTree, &classHead, &soughtClass);
 343                 }
 344                 ce->superClassID = foundClass ? foundClass->classID : (uint32_t)(-1);
 345
 346                 SLIST_INIT(&ce->counterList);
 347                 SLIST_INIT(&ce->userClientList);
 348
 349                 RB_INSERT(ClassTree, &classHead, ce);
 350                 SLIST_INSERT_HEAD(&foundKext->classList, ce, lLink);
 351
 352                 foundKext->classes++;
 353
 354                 kextHint = foundKext;
 355
 356                 sequenceID++;
 357                 registeredClasses++;
 358         }
 359         else {
 360                 panic("IOStatistics::onClassAdded: cannot find parent kext: %s", parentKext->getIdentifierCString());
 361         }
 362
 363         IORWLockUnlock(lock);
 364 }
 365
 366 void IOStatistics::onClassRemoved(OSKext *parentKext, OSMetaClass *metaClass)
 367 {
 368         ClassNode sought, *found;
 369
 370         assert(parentKext && metaClass);
 371
 372         if (!enabled) {
 373                 return;
 374         }
 375
 376         LOG(1, "IOStatistics::onClassRemoved: %s\n", metaClass->getClassName());
 377
 378         IORWLockWrite(lock);
 379
 380         sought.metaClass = metaClass;
 381         found = RB_FIND(ClassTree, &classHead, &sought);
 382         if (found) {
 383                 IOEventSourceCounter *esc;
 384                 IOUserClientCounter *ucc;
 385
 386                 /* Free up the list of counters */
 387                 while ((esc = SLIST_FIRST(&found->counterList))) {
 388                         SLIST_REMOVE_HEAD(&found->counterList, link);
 389                         kfree(esc, sizeof(IOEventSourceCounter));
 390                 }
 391
 392                 /* Free up the user client list */
 393                 while ((ucc = SLIST_FIRST(&found->userClientList))) {
 394                         SLIST_REMOVE_HEAD(&found->userClientList, link);
 395                         kfree(ucc, sizeof(IOUserClientCounter));
 396                 }
 397
 398                 /* Remove from class tree */
 399                 RB_REMOVE(ClassTree, &classHead, found);
 400
 401                 /* Remove from parent */
 402                 SLIST_REMOVE(&found->parentKext->classList, found, ClassNode, lLink);
 403
 404                 /* Finally, free the class node */
 405                 kfree(found, sizeof(ClassNode));
 406
 407                 sequenceID++;
 408                 registeredClasses--;
 409         }
 410         else {
 411                 panic("IOStatistics::onClassRemoved: cannot find class: %s", metaClass->getClassName());
 412         }
 413
 414         IORWLockUnlock(lock);
 415 }
 416
 417 IOEventSourceCounter *IOStatistics::registerEventSource(OSObject *inOwner)
 418 {
 419         IOEventSourceCounter *counter = NULL;
 420         ClassNode sought, *found = NULL;
 421         boolean_t createDummyCounter = FALSE;
 422
 423         assert(inOwner);
 424
 425         if (!enabled) {
 426                 return NULL;
 427         }
 428
 429         counter = (IOEventSourceCounter*)kalloc(sizeof(IOEventSourceCounter));
 430         if (!counter) {
 431                 return NULL;
 432         }
 433
 434         memset(counter, 0, sizeof(IOEventSourceCounter));
 435
 436         IORWLockWrite(lock);
 437
 438         /* Workaround for <rdar://problem/7158117> - create a dummy counter when inOwner is bad.
 439          * We use retainCount here as our best indication that the pointer is awry.
 440          */
 441         if (inOwner->retainCount > 0xFFFFFF) {
 442                 kprintf("IOStatistics::registerEventSource - bad metaclass %p\n", inOwner);
 443                 createDummyCounter = TRUE;
 444         }
 445         else {
 446                 sought.metaClass = inOwner->getMetaClass();
 447                 found = RB_FIND(ClassTree, &classHead, &sought);
 448         }
 449
 450         if (found) {
 451                 counter->parentClass = found;
 452                 SLIST_INSERT_HEAD(&found->counterList, counter, link);
 453                 registeredCounters++;
 454         }
 455
 456         if (!(createDummyCounter || found)) {
 457                 panic("IOStatistics::registerEventSource: cannot find parent class: %s", inOwner->getMetaClass()->getClassName());
 458         }
 459
 460         IORWLockUnlock(lock);
 461
 462         return counter;
 463 }
 464
 465 void IOStatistics::unregisterEventSource(IOEventSourceCounter *counter)
 466 {
 467         if (!counter) {
 468                 return;
 469         }
 470
 471         IORWLockWrite(lock);
 472
 473         if (counter->parentClass) {
 474                 SLIST_REMOVE(&counter->parentClass->counterList, counter, IOEventSourceCounter, link);
 475                 registeredCounters--;
 476         }
 477         kfree(counter, sizeof(IOEventSourceCounter));
 478
 479         IORWLockUnlock(lock);
 480 }
 481
 482 IOWorkLoopCounter* IOStatistics::registerWorkLoop(IOWorkLoop *workLoop)
 483 {
 484         IOWorkLoopCounter *counter = NULL;
 485         KextNode *found;
 486
 487         assert(workLoop);
 488
 489         if (!enabled) {
 490                 return NULL;
 491         }
 492
 493         counter = (IOWorkLoopCounter*)kalloc(sizeof(IOWorkLoopCounter));
 494         if (!counter) {
 495                 return NULL;
 496         }
 497
 498         memset(counter, 0, sizeof(IOWorkLoopCounter));
 499
 500         found = getKextNodeFromBacktrace(TRUE);
 501         if (!found) {
 502                 panic("IOStatistics::registerWorkLoop: cannot find parent kext");
 503         }
 504
 505         counter->parentKext = found;
 506         counter->workLoop = workLoop;
 507         RB_INIT(&counter->dependencyHead);
 508         SLIST_INSERT_HEAD(&found->workLoopList, counter, link);
 509         registeredWorkloops++;
 510
 511         releaseKextNode(found);
 512
 513         return counter;
 514 }
 515
 516 void IOStatistics::unregisterWorkLoop(IOWorkLoopCounter *counter)
 517 {
 518         if (!counter) {
 519                 return;
 520         }
 521
 522         IORWLockWrite(lock);
 523
 524         SLIST_REMOVE(&counter->parentKext->workLoopList, counter, IOWorkLoopCounter, link);
 525         kfree(counter, sizeof(IOWorkLoopCounter));
 526         registeredWorkloops--;
 527
 528         IORWLockUnlock(lock);
 529 }
 530
 531 IOUserClientCounter *IOStatistics::registerUserClient(IOUserClient *userClient)
 532 {
 533         ClassNode sought, *found;
 534         IOUserClientCounter *counter = NULL;
 535
 536         assert(userClient);
 537
 538         if (!enabled) {
 539                 return NULL;
 540         }
 541
 542         counter = (IOUserClientCounter*)kalloc(sizeof(IOUserClientCounter));
 543         if (!counter) {
 544                 return NULL;
 545         }
 546
 547         memset(counter, 0, sizeof(IOUserClientCounter));
 548
 549         IORWLockWrite(lock);
 550
 551         sought.metaClass = userClient->getMetaClass();
 552
 553         found = RB_FIND(ClassTree, &classHead, &sought);
 554         if (found) {
 555                 counter->parentClass = found;
 556                 SLIST_INSERT_HEAD(&found->userClientList, counter, link);
 557         }
 558         else {
 559                 panic("IOStatistics::registerUserClient: cannot find parent class: %s", sought.metaClass->getClassName());
 560         }
 561
 562         IORWLockUnlock(lock);
 563
 564         return counter;
 565 }
 566
 567 void IOStatistics::unregisterUserClient(IOUserClientCounter *counter)
 568 {
 569         if (!counter) {
 570                 return;
 571         }
 572
 573         IORWLockWrite(lock);
 574
 575         SLIST_REMOVE(&counter->parentClass->userClientList, counter, IOUserClientCounter, link);
 576         kfree(counter, sizeof(IOUserClientCounter));
 577
 578         IORWLockUnlock(lock);
 579 }
 580
 581 void IOStatistics::attachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
 582 {
 583         if (!wlc) {
 584         return;
 585         }
 586
 587         IORWLockWrite(lock);
 588
 589         if (!nextWorkLoopDependency) {
 590                 return;
 591         }
 592
 593         attachedEventSources++;
 594         wlc->attachedEventSources++;
 595
 596         /* Track the kext dependency */
 597         nextWorkLoopDependency->loadTag = esc->parentClass->parentKext->loadTag;
 598         if (NULL == RB_INSERT(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, nextWorkLoopDependency)) {
 599                 nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
 600         }
 601
 602         IORWLockUnlock(lock);
 603 }
 604
 605 void IOStatistics::detachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
 606 {
 607         IOWorkLoopDependency sought, *found;
 608
 609         if (!wlc) {
 610                 return;
 611         }
 612
 613         IORWLockWrite(lock);
 614
 615         attachedEventSources--;
 616         wlc->attachedEventSources--;
 617
 618         sought.loadTag = esc->parentClass->parentKext->loadTag;
 619
 620         found = RB_FIND(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, &sought);
 621         if (found) {
 622                 RB_REMOVE(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, found);
 623                 kfree(found, sizeof(IOWorkLoopDependency));
 624         }
 625
 626         IORWLockUnlock(lock);
 627 }
 628
 629 int IOStatistics::getStatistics(sysctl_req *req)
 630 {
 631         int error;
 632         uint32_t calculatedSize, size;
 633         char *buffer, *ptr;
 634         IOStatisticsHeader *header;
 635
 636         assert(IOStatistics::enabled && req);
 637
 638         IORWLockRead(IOStatistics::lock);
 639
 640         /* Work out how much we need to allocate. IOStatisticsKext is of variable size. */
 641         calculatedSize = sizeof(IOStatisticsHeader) +
 642                                          sizeof(IOStatisticsGlobal) +
 643                                         (sizeof(IOStatisticsKext) * loadedKexts) + (sizeof(uint32_t) * registeredClasses) +
 644                                         (sizeof(IOStatisticsMemory) * loadedKexts) +
 645                                         (sizeof(IOStatisticsClass) * registeredClasses) +
 646                                         (sizeof(IOStatisticsCounter) * registeredClasses) +
 647                                         (sizeof(IOStatisticsKextIdentifier) * loadedKexts) +
 648                                         (sizeof(IOStatisticsClassName) * registeredClasses);
 649
 650         /* Size request? */
 651         if (req->oldptr == USER_ADDR_NULL) {
 652                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 653                 goto exit;
 654         }
 655
 656         /* Read only */
 657         if (req->newptr != USER_ADDR_NULL) {
 658                 error = EPERM;
 659                 goto exit;
 660         }
 661
 662         buffer = (char*)kalloc(calculatedSize);
 663         if (!buffer) {
 664                 error = ENOMEM;
 665                 goto exit;
 666         }
 667
 668         memset(buffer, 0, calculatedSize);
 669
 670         ptr = buffer;
 671
 672         header = (IOStatisticsHeader*)((void*)ptr);
 673
 674         header->sig = IOSTATISTICS_SIG;
 675         header->ver = IOSTATISTICS_VER;
 676
 677         header->seq = sequenceID;
 678
 679         ptr += sizeof(IOStatisticsHeader);
 680
 681         /* Global data - seq, timers, interrupts, etc) */
 682         header->globalStatsOffset = sizeof(IOStatisticsHeader);
 683         size = copyGlobalStatistics((IOStatisticsGlobal*)((void*)ptr));
 684         ptr += size;
 685
 686         /* Kext statistics */
 687         header->kextStatsOffset = header->globalStatsOffset + size;
 688         size = copyKextStatistics((IOStatisticsKext*)((void*)ptr));
 689         ptr += size;
 690
 691         /* Memory allocation info */
 692         header->memoryStatsOffset = header->kextStatsOffset + size;
 693         size = copyMemoryStatistics((IOStatisticsMemory*)((void*)ptr));
 694         ptr += size;
 695
 696         /* Class statistics */
 697         header->classStatsOffset = header->memoryStatsOffset + size;
 698         size = copyClassStatistics((IOStatisticsClass*)((void*)ptr));
 699         ptr += size;
 700
 701         /* Dynamic class counter data */
 702         header->counterStatsOffset = header->classStatsOffset + size;
 703         size = copyCounterStatistics((IOStatisticsCounter*)((void*)ptr));
 704         ptr += size;
 705
 706         /* Kext identifiers */
 707         header->kextIdentifiersOffset = header->counterStatsOffset + size;
 708         size = copyKextIdentifiers((IOStatisticsKextIdentifier*)((void*)ptr));
 709         ptr += size;
 710
 711         /* Class names */
 712         header->classNamesOffset = header->kextIdentifiersOffset + size;
 713         size = copyClassNames((IOStatisticsClassName*)ptr);
 714         ptr += size;
 715
 716         LOG(2, "IOStatistics::getStatistics - calculatedSize 0x%x, kexts 0x%x, classes 0x%x.\n",
 717                 calculatedSize, loadedKexts, registeredClasses);
 718
 719         assert( (uint32_t)(ptr - buffer) == calculatedSize );
 720
 721         error = SYSCTL_OUT(req, buffer, calculatedSize);
 722
 723         kfree(buffer, calculatedSize);
 724
 725 exit:
 726         IORWLockUnlock(IOStatistics::lock);
 727         return error;
 728 }
 729
 730 int IOStatistics::getWorkLoopStatistics(sysctl_req *req)
 731 {
 732         int error;
 733         uint32_t calculatedSize, size;
 734         char *buffer;
 735         IOStatisticsWorkLoopHeader *header;
 736
 737         assert(IOStatistics::enabled && req);
 738
 739         IORWLockRead(IOStatistics::lock);
 740
 741         /* Approximate how much we need to allocate (worse case estimate) */
 742         calculatedSize = sizeof(IOStatisticsWorkLoop) * registeredWorkloops +
 743                                          sizeof(uint32_t) * attachedEventSources;
 744
 745         /* Size request? */
 746         if (req->oldptr == USER_ADDR_NULL) {
 747                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 748                 goto exit;
 749         }
 750
 751         /* Read only */
 752         if (req->newptr != USER_ADDR_NULL) {
 753                 error = EPERM;
 754                 goto exit;
 755         }
 756
 757         buffer = (char*)kalloc(calculatedSize);
 758         if (!buffer) {
 759                 error = ENOMEM;
 760                 goto exit;
 761         }
 762
 763         header = (IOStatisticsWorkLoopHeader*)((void*)buffer);
 764
 765         header->sig = IOSTATISTICS_SIG_WORKLOOP;
 766         header->ver = IOSTATISTICS_VER;
 767
 768         header->seq = sequenceID;
 769
 770         header->workloopCount = registeredWorkloops;
 771
 772         size = copyWorkLoopStatistics(&header->workLoopStats);
 773
 774         LOG(2, "IOStatistics::getWorkLoopStatistics: calculatedSize %d, size %d\n", calculatedSize, size);
 775
 776         assert( size <= calculatedSize );
 777
 778         error = SYSCTL_OUT(req, buffer, size);
 779
 780         kfree(buffer, calculatedSize);
 781
 782 exit:
 783         IORWLockUnlock(IOStatistics::lock);
 784         return error;
 785 }
 786
 787 int IOStatistics::getUserClientStatistics(sysctl_req *req)
 788 {
 789         int error;
 790         uint32_t calculatedSize, size;
 791         char *buffer;
 792         uint32_t requestedLoadTag = 0;
 793         IOStatisticsUserClientHeader *header;
 794
 795         assert(IOStatistics::enabled && req);
 796
 797         IORWLockRead(IOStatistics::lock);
 798
 799         /* Work out how much we need to allocate */
 800         calculatedSize = sizeof(IOStatisticsUserClientHeader) +
 801                                          sizeof(IOStatisticsUserClientCall) * IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS * loadedKexts;
 802
 803         /* Size request? */
 804         if (req->oldptr == USER_ADDR_NULL) {
 805                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 806                 goto exit;
 807         }
 808
 809         /* Kext request (potentially) valid? */
 810         if (!req->newptr || req->newlen < sizeof(requestedLoadTag)) {
 811                 error = EINVAL;
 812                 goto exit;
 813         }
 814
 815         SYSCTL_IN(req, &requestedLoadTag, sizeof(requestedLoadTag));
 816
 817         LOG(2, "IOStatistics::getUserClientStatistics - requesting kext w/load tag: %d\n", requestedLoadTag);
 818
 819         buffer = (char*)kalloc(calculatedSize);
 820         if (!buffer) {
 821                 error = ENOMEM;
 822                 goto exit;
 823         }
 824
 825         header = (IOStatisticsUserClientHeader*)((void*)buffer);
 826
 827         header->sig = IOSTATISTICS_SIG_USERCLIENT;
 828         header->ver = IOSTATISTICS_VER;
 829
 830         header->seq = sequenceID;
 831
 832         header->processes = 0;
 833
 834         size = copyUserClientStatistics(header, requestedLoadTag);
 835
 836         assert((sizeof(IOStatisticsUserClientHeader) + size) <= calculatedSize);
 837
 838         if (size) {
 839                 error = SYSCTL_OUT(req, buffer, sizeof(IOStatisticsUserClientHeader) + size);
 840         }
 841         else {
 842                 error = EINVAL;
 843         }
 844
 845         kfree(buffer, calculatedSize);
 846
 847 exit:
 848         IORWLockUnlock(IOStatistics::lock);
 849         return error;
 850 }
 851
 852 uint32_t IOStatistics::copyGlobalStatistics(IOStatisticsGlobal *stats)
 853 {
 854         stats->kextCount = loadedKexts;
 855         stats->classCount = registeredClasses;
 856         stats->workloops = registeredWorkloops;
 857
 858         return sizeof(IOStatisticsGlobal);
 859 }
 860
 861 uint32_t IOStatistics::copyKextStatistics(IOStatisticsKext *stats)
 862 {
 863         KextNode *ke;
 864         ClassNode *ce;
 865         uint32_t index = 0;
 866
 867         RB_FOREACH(ke, KextTree, &kextHead) {
 868                 stats->loadTag = ke->loadTag;
 869                 ke->kext->getSizeInfo(&stats->loadSize, &stats->wiredSize);
 870
 871                 stats->classes = ke->classes;
 872
 873                 /* Append indices of owned classes */
 874                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 875                         stats->classIndexes[index++] = ce->classID;
 876                 }
 877
 878                 stats = (IOStatisticsKext *)((void*)((char*)stats + sizeof(IOStatisticsKext) + (ke->classes * sizeof(uint32_t))));
 879         }
 880
 881         return (sizeof(IOStatisticsKext) * loadedKexts + sizeof(uint32_t) * registeredClasses);
 882 }
 883
 884 uint32_t IOStatistics::copyMemoryStatistics(IOStatisticsMemory *stats)
 885 {
 886         KextNode *ke;
 887
 888         RB_FOREACH(ke, KextTree, &kextHead) {
 889                 stats->allocatedSize = ke->memoryCounters[kIOStatisticsMalloc];
 890                 stats->freedSize = ke->memoryCounters[kIOStatisticsFree];
 891                 stats->allocatedAlignedSize = ke->memoryCounters[kIOStatisticsMallocAligned];
 892                 stats->freedAlignedSize = ke->memoryCounters[kIOStatisticsFreeAligned];
 893                 stats->allocatedContiguousSize = ke->memoryCounters[kIOStatisticsMallocContiguous];
 894                 stats->freedContiguousSize = ke->memoryCounters[kIOStatisticsFreeContiguous];
 895                 stats->allocatedPageableSize = ke->memoryCounters[kIOStatisticsMallocPageable];
 896                 stats->freedPageableSize = ke->memoryCounters[kIOStatisticsFreePageable];
 897                 stats++;
 898         }
 899
 900         return (sizeof(IOStatisticsMemory) * loadedKexts);
 901 }
 902
 903 uint32_t IOStatistics::copyClassStatistics(IOStatisticsClass *stats)
 904 {
 905         KextNode *ke;
 906         ClassNode *ce;
 907
 908         RB_FOREACH(ke, KextTree, &kextHead) {
 909                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 910                         stats->classID = ce->classID;
 911                         stats->superClassID = ce->superClassID;
 912                         stats->classSize = ce->metaClass->getClassSize();
 913
 914                         stats++;
 915                 }
 916         }
 917
 918         return sizeof(IOStatisticsClass) * registeredClasses;
 919 }
 920
 921 uint32_t IOStatistics::copyCounterStatistics(IOStatisticsCounter *stats)
 922 {
 923         KextNode *ke;
 924         ClassNode *ce;
 925
 926         RB_FOREACH(ke, KextTree, &kextHead) {
 927                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 928                         IOUserClientCounter *userClientCounter;
 929                         IOEventSourceCounter *counter;
 930
 931                         stats->classID = ce->classID;
 932                         stats->classInstanceCount = ce->metaClass->getInstanceCount();
 933
 934                         IOStatisticsUserClients *uc = &stats->userClientStatistics;
 935
 936                         /* User client counters */
 937                         SLIST_FOREACH(userClientCounter, &ce->userClientList, link) {
 938                                 uc->clientCalls += userClientCounter->clientCalls;
 939                                 uc->created++;
 940                         }
 941
 942                         IOStatisticsInterruptEventSources *iec = &stats->interruptEventSourceStatistics;
 943                         IOStatisticsInterruptEventSources *fiec = &stats->filterInterruptEventSourceStatistics;
 944                         IOStatisticsTimerEventSources *tec = &stats->timerEventSourceStatistics;
 945                         IOStatisticsCommandGates *cgc = &stats->commandGateStatistics;
 946                         IOStatisticsCommandQueues *cqc = &stats->commandQueueStatistics;
 947                         IOStatisticsDerivedEventSources *dec = &stats->derivedEventSourceStatistics;
 948
 949                         /* Event source counters */
 950                         SLIST_FOREACH(counter, &ce->counterList, link) {
 951                                 switch (counter->type) {
 952                                         case kIOStatisticsInterruptEventSourceCounter:
 953                                                 iec->created++;
 954                                                 iec->produced += counter->u.interrupt.produced;
 955                                                 iec->checksForWork += counter->u.interrupt.checksForWork;
 956                                                 break;
 957                                         case kIOStatisticsFilterInterruptEventSourceCounter:
 958                                                 fiec->created++;
 959                                                 fiec->produced += counter->u.filter.produced;
 960                                                 fiec->checksForWork += counter->u.filter.checksForWork;
 961                                                 break;
 962                                         case kIOStatisticsTimerEventSourceCounter:
 963                                                 tec->created++;
 964                                                 tec->timeouts += counter->u.timer.timeouts;
 965                                                 tec->checksForWork += counter->u.timer.checksForWork;
 966                                                 tec->timeOnGate += counter->timeOnGate;
 967                                                 tec->closeGateCalls += counter->closeGateCalls;
 968                                                 tec->openGateCalls += counter->openGateCalls;
 969                                                 break;
 970                                         case kIOStatisticsCommandGateCounter:
 971                                                 cgc->created++;
 972                                                 cgc->timeOnGate += counter->timeOnGate;
 973                                                 cgc->actionCalls += counter->u.commandGate.actionCalls;
 974                                                 break;
 975                                         case kIOStatisticsCommandQueueCounter:
 976                                                 cqc->created++;
 977                                                 cqc->actionCalls += counter->u.commandQueue.actionCalls;
 978                                                 break;
 979                                         case kIOStatisticsDerivedEventSourceCounter:
 980                                                 dec->created++;
 981                                                 dec->timeOnGate += counter->timeOnGate;
 982                                                 dec->closeGateCalls += counter->closeGateCalls;
 983                                                 dec->openGateCalls += counter->openGateCalls;
 984                                                 break;
 985                                         default:
 986                                                 break;
 987                                 }
 988                         }
 989
 990                         stats++;
 991                 }
 992         }
 993
 994         return sizeof(IOStatisticsCounter) * registeredClasses;
 995 }
 996
 997 uint32_t IOStatistics::copyKextIdentifiers(IOStatisticsKextIdentifier *kextIDs)
 998 {
 999         KextNode *ke;
1000
1001         RB_FOREACH(ke, KextTree, &kextHead) {
1002                 strncpy(kextIDs->identifier, ke->kext->getIdentifierCString(), kIOStatisticsDriverNameLength);
1003                 kextIDs++;
1004         }
1005
1006         return (sizeof(IOStatisticsKextIdentifier) * loadedKexts);
1007 }
1008
1009 uint32_t IOStatistics::copyClassNames(IOStatisticsClassName *classNames)
1010 {
1011         KextNode *ke;
1012         ClassNode *ce;
1013
1014         RB_FOREACH(ke, KextTree, &kextHead) {
1015                 SLIST_FOREACH(ce, &ke->classList, lLink) {
1016                         strncpy(classNames->name, ce->metaClass->getClassName(), kIOStatisticsClassNameLength);
1017                         classNames++;
1018                 }
1019         }
1020
1021         return (sizeof(IOStatisticsClassName) * registeredClasses);
1022 }
1023
1024 uint32_t IOStatistics::copyWorkLoopStatistics(IOStatisticsWorkLoop *stats)
1025 {
1026         KextNode *ke;
1027         IOWorkLoopCounter *wlc;
1028         IOWorkLoopDependency *dependentNode;
1029         uint32_t size, accumulatedSize = 0;
1030
1031         RB_FOREACH(ke, KextTree, &kextHead) {
1032                 SLIST_FOREACH(wlc, &ke->workLoopList, link) {
1033                         stats->kextLoadTag = ke->loadTag;
1034                         stats->attachedEventSources = wlc->attachedEventSources;
1035                         stats->timeOnGate = wlc->timeOnGate;
1036                         stats->dependentKexts = 0;
1037                         RB_FOREACH(dependentNode, IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead) {
1038                                 stats->dependentKextLoadTags[stats->dependentKexts] = dependentNode->loadTag;
1039                                 stats->dependentKexts++;
1040                         }
1041
1042                         size = sizeof(IOStatisticsWorkLoop) + (sizeof(uint32_t) * stats->dependentKexts);
1043
1044                         accumulatedSize += size;
1045                         stats = (IOStatisticsWorkLoop*)((void*)((char*)stats + size));
1046                 }
1047         }
1048
1049         return accumulatedSize;
1050 }
1051
1052 uint32_t IOStatistics::copyUserClientStatistics(IOStatisticsUserClientHeader *stats, uint32_t loadTag)
1053 {
1054         KextNode *sought, *found = NULL;
1055         uint32_t procs = 0;
1056         IOUserClientProcessEntry *processEntry;
1057
1058         RB_FOREACH(sought, KextTree, &kextHead) {
1059                 if (sought->loadTag == loadTag) {
1060                         found = sought;
1061                         break;
1062                 }
1063         }
1064
1065         if (!found) {
1066                 return 0;
1067         }
1068
1069         TAILQ_FOREACH(processEntry, &found->userClientCallList, link) {
1070                 strncpy(stats->userClientCalls[procs].processName, processEntry->processName, kIOStatisticsProcessNameLength);
1071                 stats->userClientCalls[procs].pid = processEntry->pid;
1072                 stats->userClientCalls[procs].calls = processEntry->calls;
1073                 stats->processes++;
1074                 procs++;
1075         }
1076
1077         return sizeof(IOStatisticsUserClientCall) * stats->processes;
1078 }
1079
1080 void IOStatistics::storeUserClientCallInfo(IOUserClient *userClient, IOUserClientCounter *counter)
1081 {
1082         OSString *ossUserClientCreator = NULL;
1083         int32_t pid = -1;
1084         KextNode *parentKext;
1085         IOUserClientProcessEntry *entry, *nextEntry, *prevEntry = NULL;
1086         uint32_t count = 0;
1087         const char *ptr = NULL;
1088         OSObject *obj;
1089
1090         /* TODO: see if this can be more efficient */
1091         obj = userClient->copyProperty("IOUserClientCreator",
1092                                         gIOServicePlane,
1093                                         kIORegistryIterateRecursively | kIORegistryIterateParents);
1094
1095         if (!obj)
1096                 goto err_nounlock;
1097
1098         ossUserClientCreator = OSDynamicCast(OSString, obj);
1099
1100         if (ossUserClientCreator) {
1101                 uint32_t len, lenIter = 0;
1102
1103                 ptr = ossUserClientCreator->getCStringNoCopy();
1104                 len = ossUserClientCreator->getLength();
1105
1106                 while ((*ptr != ' ') && (lenIter < len)) {
1107                         ptr++;
1108                         lenIter++;
1109                 }
1110
1111                 if (lenIter < len) {
1112                         ptr++; // Skip the space
1113                         lenIter++;
1114                         pid = 0;
1115                         while ( (*ptr != ',') && (lenIter < len)) {
1116                                 pid = pid*10 + (*ptr - '0');
1117                                 ptr++;
1118                                 lenIter++;
1119                         }
1120
1121                         if(lenIter == len) {
1122                                 pid = -1;
1123                         } else {
1124                                 ptr += 2;
1125                         }
1126                 }
1127         }
1128
1129         if (-1 == pid)
1130                 goto err_nounlock;
1131
1132         IORWLockWrite(lock);
1133
1134         parentKext = counter->parentClass->parentKext;
1135
1136         TAILQ_FOREACH(entry, &parentKext->userClientCallList, link) {
1137                 if (entry->pid == pid) {
1138                         /* Found, so increment count and move to the head */
1139                         entry->calls++;
1140                         if (count) {
1141                                 TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1142                                 break;
1143                         }
1144                         else {
1145                                 /* At the head already, so increment and return */
1146                                 goto err_unlock;
1147                         }
1148                 }
1149
1150                 count++;
1151         }
1152
1153         if (!entry) {
1154                 if (count == IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS) {
1155                         /* Max elements hit, so reuse the last */
1156                         entry = TAILQ_LAST(&parentKext->userClientCallList, ProcessEntryList);
1157                         TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1158                 }
1159                 else {
1160                         /* Otherwise, allocate a new entry */
1161                         entry = (IOUserClientProcessEntry*)kalloc(sizeof(IOUserClientProcessEntry));
1162                         if (!entry) {
1163                             IORWLockUnlock(lock);
1164                                 return;
1165                         }
1166                 }
1167
1168                 strncpy(entry->processName, ptr, kIOStatisticsProcessNameLength);
1169                 entry->pid = pid;
1170                 entry->calls = 1;
1171         }
1172
1173         TAILQ_FOREACH(nextEntry, &parentKext->userClientCallList, link) {
1174                 if (nextEntry->calls <= entry->calls)
1175                         break;
1176
1177                 prevEntry = nextEntry;
1178         }
1179
1180         if (!prevEntry)
1181                 TAILQ_INSERT_HEAD(&parentKext->userClientCallList, entry, link);
1182         else
1183                 TAILQ_INSERT_AFTER(&parentKext->userClientCallList, prevEntry, entry, link);
1184
1185 err_unlock:
1186         IORWLockUnlock(lock);
1187
1188 err_nounlock:
1189         if (obj)
1190                 obj->release();
1191 }
1192
1193 void IOStatistics::countUserClientCall(IOUserClient *client) {
1194         IOUserClient::ExpansionData *data;
1195         IOUserClientCounter *counter;
1196
1197         /* Guard against an uninitialized client object - <rdar://problem/8577946> */
1198         if (!(data = client->reserved)) {
1199                 return;
1200         }
1201
1202         if ((counter = data->counter)) {
1203                 storeUserClientCallInfo(client, counter);
1204                 OSIncrementAtomic(&counter->clientCalls);
1205         }
1206 }
1207
1208 KextNode *IOStatistics::getKextNodeFromBacktrace(boolean_t write) {
1209         const uint32_t btMin = 3;
1210
1211         void *bt[16];
1212         unsigned btCount = sizeof(bt) / sizeof(bt[0]);
1213         vm_offset_t *scanAddr = NULL;
1214         uint32_t i;
1215         KextNode *found = NULL, *ke = NULL;
1216
1217         btCount = OSBacktrace(bt, btCount);
1218
1219         if (write) {
1220                 IORWLockWrite(lock);
1221         } else {
1222                 IORWLockRead(lock);
1223         }
1224
1225         /* Ignore first levels */
1226         scanAddr = (vm_offset_t *)&bt[btMin - 1];
1227
1228         for (i = btMin - 1; i < btCount; i++, scanAddr++) {
1229                 ke = RB_ROOT(&kextAddressHead);
1230                 while (ke) {
1231                         if (*scanAddr < ke->address) {
1232                                 ke = RB_LEFT(ke, addressLink);
1233                         }
1234                         else {
1235                                 if ((*scanAddr < ke->address_end) && (*scanAddr >= ke->address)) {
1236                                         if (!ke->kext->isKernelComponent()) {
1237                                                 return ke;
1238                                         } else {
1239                                                 found = ke;
1240                                         }
1241                                 }
1242                                 ke = RB_RIGHT(ke, addressLink);
1243                         }
1244                 }
1245         }
1246
1247         if (!found) {
1248                 IORWLockUnlock(lock);
1249         }
1250
1251         return found;
1252 }
1253
1254 void IOStatistics::releaseKextNode(KextNode *node) {
1255 #pragma unused(node)
1256         IORWLockUnlock(lock);
1257 }
1258
1259 /* IOLib allocations */
1260 void IOStatistics::countAlloc(uint32_t index, vm_size_t size) {
1261         KextNode *ke;
1262
1263         if (!enabled) {
1264                 return;
1265         }
1266
1267         ke = getKextNodeFromBacktrace(FALSE);
1268         if (ke) {
1269                 OSAddAtomic(size, &ke->memoryCounters[index]);
1270                 releaseKextNode(ke);
1271         }
1272 }
1273
1274 #endif /* IOKITSTATS */