iokit/Kernel/IOStatistics.cpp

   1 /*
   2  * Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/sysctl.h>
  30 #include <kern/host.h>
  31 #include <kern/zalloc.h>
  32
  33 #include <IOKit/system.h>
  34 #include <libkern/c++/OSKext.h>
  35 #include <libkern/OSAtomic.h>
  36
  37 #include <IOKit/IOStatisticsPrivate.h>
  38 #include <IOKit/IOUserClient.h>
  39 #include <IOKit/IOEventSource.h>
  40 #include <IOKit/IOKitDebug.h>
  41
  42 #if IOKITSTATS
  43 bool IOStatistics::enabled = false;
  44
  45 uint32_t IOStatistics::sequenceID = 0;
  46
  47 uint32_t IOStatistics::lastClassIndex = 0;
  48 uint32_t IOStatistics::lastKextIndex = 0;
  49
  50 uint32_t IOStatistics::loadedKexts = 0;
  51 uint32_t IOStatistics::registeredClasses = 0;
  52 uint32_t IOStatistics::registeredCounters = 0;
  53 uint32_t IOStatistics::registeredWorkloops = 0;
  54
  55 uint32_t IOStatistics::attachedEventSources = 0;
  56
  57 IOWorkLoopDependency *IOStatistics::nextWorkLoopDependency = NULL;
  58
  59 /* Logging */
  60
  61 #define LOG_LEVEL 0
  62
  63 #define LOG(level, format, ...) \
  64 do { \
  65         if (level <= LOG_LEVEL) \
  66                 printf(format, ##__VA_ARGS__); \
  67 } while (0)
  68
  69 /* Locks */
  70
  71 IORWLock *IOStatistics::lock = NULL;
  72
  73 /* Kext tree */
  74
  75 KextNode *IOStatistics::kextHint = NULL;
  76
  77 IOStatistics::KextTreeHead IOStatistics::kextHead = RB_INITIALIZER(&IOStatistics::kextHead);
  78
  79 int IOStatistics::kextNodeCompare(KextNode *e1, KextNode *e2)
  80 {
  81     if (e1->kext < e2->kext)
  82         return -1;
  83     else if (e1->kext > e2->kext)
  84         return 1;
  85     else
  86         return 0;
  87 }
  88
  89 RB_GENERATE(IOStatistics::KextTree, KextNode, link, kextNodeCompare);
  90
  91 /* Kext tree ordered by address */
  92
  93 IOStatistics::KextAddressTreeHead IOStatistics::kextAddressHead = RB_INITIALIZER(&IOStatistics::kextAddressHead);
  94
  95 int IOStatistics::kextAddressNodeCompare(KextNode *e1, KextNode *e2)
  96 {
  97     if (e1->address < e2->address)
  98         return -1;
  99     else if (e1->address > e2->address)
 100         return 1;
 101     else
 102         return 0;
 103 }
 104
 105 RB_GENERATE(IOStatistics::KextAddressTree, KextNode, addressLink, kextAddressNodeCompare);
 106
 107 /* Class tree */
 108
 109 IOStatistics::ClassTreeHead IOStatistics::classHead = RB_INITIALIZER(&IOStatistics::classHead);
 110
 111 int IOStatistics::classNodeCompare(ClassNode *e1, ClassNode *e2) {
 112     if (e1->metaClass < e2->metaClass)
 113         return -1;
 114     else if (e1->metaClass > e2->metaClass)
 115         return 1;
 116     else
 117         return 0;
 118 }
 119
 120 RB_GENERATE(IOStatistics::ClassTree, ClassNode, tLink, classNodeCompare);
 121
 122 /* Workloop dependencies */
 123
 124 int IOWorkLoopCounter::loadTagCompare(IOWorkLoopDependency *e1, IOWorkLoopDependency *e2) {
 125     if (e1->loadTag < e2->loadTag)
 126         return -1;
 127     else if (e1->loadTag > e2->loadTag)
 128         return 1;
 129     else
 130         return 0;
 131 }
 132
 133 RB_GENERATE(IOWorkLoopCounter::DependencyTree, IOWorkLoopDependency, link, IOWorkLoopCounter::loadTagCompare);
 134
 135 /* sysctl stuff */
 136
 137 static int
 138 oid_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1, int arg2, struct sysctl_req *req)
 139 {
 140         int error = EINVAL;
 141         uint32_t request = arg2;
 142
 143         switch (request)
 144         {
 145                 case kIOStatisticsGeneral:
 146                         error = IOStatistics::getStatistics(req);
 147                         break;
 148                 case kIOStatisticsWorkLoop:
 149                         error = IOStatistics::getWorkLoopStatistics(req);
 150                         break;
 151                 case kIOStatisticsUserClient:
 152                         error = IOStatistics::getUserClientStatistics(req);
 153                         break;
 154                 default:
 155                         break;
 156         }
 157
 158         return error;
 159 }
 160
 161 SYSCTL_NODE(_debug, OID_AUTO, iokit_statistics, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "IOStatistics");
 162
 163 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, general,
 164             CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 165             0, kIOStatisticsGeneral, oid_sysctl, "S", "");
 166
 167 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, workloop,
 168             CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 169             0, kIOStatisticsWorkLoop, oid_sysctl, "S", "");
 170
 171 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, userclient,
 172             CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 173             0, kIOStatisticsUserClient, oid_sysctl, "S", "");
 174
 175 void IOStatistics::initialize()
 176 {
 177         if (enabled) {
 178                 return;
 179         }
 180
 181         /* Only enabled if the boot argument is set. */
 182         if (!(kIOStatistics & gIOKitDebug)) {
 183                 return;
 184         }
 185
 186         sysctl_register_oid(&sysctl__debug_iokit_statistics_general);
 187         sysctl_register_oid(&sysctl__debug_iokit_statistics_workloop);
 188         sysctl_register_oid(&sysctl__debug_iokit_statistics_userclient);
 189
 190         lock = IORWLockAlloc();
 191         if (!lock) {
 192                 return;
 193         }
 194
 195         nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
 196         if (!nextWorkLoopDependency) {
 197                 return;
 198         }
 199
 200         enabled = true;
 201 }
 202
 203 void IOStatistics::onKextLoad(OSKext *kext, kmod_info_t *kmod_info)
 204 {
 205         KextNode *ke;
 206
 207         assert(kext && kmod_info);
 208
 209         if (!enabled) {
 210                 return;
 211         }
 212
 213         LOG(1, "IOStatistics::onKextLoad: %s, tag %d, address 0x%llx, address end 0x%llx\n",
 214                 kext->getIdentifierCString(), kmod_info->id, (uint64_t)kmod_info->address, (uint64_t)(kmod_info->address + kmod_info->size));
 215
 216         ke = (KextNode *)kalloc(sizeof(KextNode));
 217         if (!ke) {
 218                 return;
 219         }
 220
 221         memset(ke, 0, sizeof(KextNode));
 222
 223         ke->kext = kext;
 224         ke->loadTag = kmod_info->id;
 225         ke->address = kmod_info->address;
 226         ke->address_end = kmod_info->address + kmod_info->size;
 227
 228         SLIST_INIT(&ke->classList);
 229         TAILQ_INIT(&ke->userClientCallList);
 230
 231         IORWLockWrite(lock);
 232
 233         RB_INSERT(KextTree, &kextHead, ke);
 234         RB_INSERT(KextAddressTree, &kextAddressHead, ke);
 235
 236         sequenceID++;
 237         loadedKexts++;
 238         lastKextIndex++;
 239
 240         IORWLockUnlock(lock);
 241 }
 242
 243 void IOStatistics::onKextUnload(OSKext *kext)
 244 {
 245         KextNode sought, *found;
 246
 247         assert(kext);
 248
 249         if (!enabled) {
 250                 return;
 251         }
 252
 253         LOG(1, "IOStatistics::onKextUnload: %s\n", kext->getIdentifierCString());
 254
 255         IORWLockWrite(lock);
 256
 257         sought.kext = kext;
 258         found = RB_FIND(KextTree, &kextHead, &sought);
 259         if (found) {
 260                 IOWorkLoopCounter *wlc;
 261                 IOUserClientProcessEntry *uce;
 262
 263                 /* Disconnect workloop counters; cleanup takes place in unregisterWorkLoop() */
 264                 while ((wlc = SLIST_FIRST(&found->workLoopList))) {
 265                         SLIST_REMOVE_HEAD(&found->workLoopList, link);
 266                         wlc->parentKext = NULL;
 267                 }
 268
 269                 /* Free up the user client list */
 270                 while ((uce = TAILQ_FIRST(&found->userClientCallList))) {
 271                         TAILQ_REMOVE(&found->userClientCallList, uce, link);
 272                         kfree(uce, sizeof(IOUserClientProcessEntry));
 273                 }
 274
 275                 /* Remove from kext trees */
 276                 RB_REMOVE(KextTree, &kextHead, found);
 277                 RB_REMOVE(KextAddressTree, &kextAddressHead, found);
 278
 279                 /*
 280                  * Clear a matching kextHint to avoid use after free in
 281                  * onClassAdded() for a class add after a KEXT unload.
 282                  */
 283                 if (found == kextHint) {
 284                         kextHint = NULL;
 285                 }
 286
 287                 /* Finally, free the class node */
 288                 kfree(found, sizeof(KextNode));
 289
 290                 sequenceID++;
 291                 loadedKexts--;
 292         }
 293         else {
 294                 panic("IOStatistics::onKextUnload: cannot find kext: %s", kext->getIdentifierCString());
 295         }
 296
 297         IORWLockUnlock(lock);
 298 }
 299
 300 void IOStatistics::onClassAdded(OSKext *parentKext, OSMetaClass *metaClass)
 301 {
 302         ClassNode *ce;
 303         KextNode soughtKext, *foundKext = NULL;
 304
 305         assert(parentKext && metaClass);
 306
 307         if (!enabled) {
 308                 return;
 309         }
 310
 311         LOG(1, "IOStatistics::onClassAdded: %s\n", metaClass->getClassName());
 312
 313         ce = (ClassNode *)kalloc(sizeof(ClassNode));
 314         if (!ce) {
 315                 return;
 316         }
 317
 318         memset(ce, 0, sizeof(ClassNode));
 319
 320         IORWLockWrite(lock);
 321
 322         /* Hinted? */
 323         if (kextHint && kextHint->kext == parentKext) {
 324                 foundKext = kextHint;
 325         }
 326         else {
 327                 soughtKext.kext = parentKext;
 328                 foundKext = RB_FIND(KextTree, &kextHead, &soughtKext);
 329         }
 330
 331         if (foundKext) {
 332                 ClassNode soughtClass, *foundClass = NULL;
 333                 const OSMetaClass *superClass;
 334
 335                 ce->metaClass = metaClass;
 336                 ce->classID = lastClassIndex++;
 337                 ce->parentKext = foundKext;
 338
 339                 /* Has superclass? */
 340                 superClass = ce->metaClass->getSuperClass();
 341                 if (superClass) {
 342                         soughtClass.metaClass = superClass;
 343                         foundClass = RB_FIND(ClassTree, &classHead, &soughtClass);
 344                 }
 345                 ce->superClassID = foundClass ? foundClass->classID : (uint32_t)(-1);
 346
 347                 SLIST_INIT(&ce->counterList);
 348                 SLIST_INIT(&ce->userClientList);
 349
 350                 RB_INSERT(ClassTree, &classHead, ce);
 351                 SLIST_INSERT_HEAD(&foundKext->classList, ce, lLink);
 352
 353                 foundKext->classes++;
 354
 355                 kextHint = foundKext;
 356
 357                 sequenceID++;
 358                 registeredClasses++;
 359         }
 360         else {
 361                 panic("IOStatistics::onClassAdded: cannot find parent kext: %s", parentKext->getIdentifierCString());
 362         }
 363
 364         IORWLockUnlock(lock);
 365 }
 366
 367 void IOStatistics::onClassRemoved(OSKext *parentKext, OSMetaClass *metaClass)
 368 {
 369         ClassNode sought, *found;
 370
 371         assert(parentKext && metaClass);
 372
 373         if (!enabled) {
 374                 return;
 375         }
 376
 377         LOG(1, "IOStatistics::onClassRemoved: %s\n", metaClass->getClassName());
 378
 379         IORWLockWrite(lock);
 380
 381         sought.metaClass = metaClass;
 382         found = RB_FIND(ClassTree, &classHead, &sought);
 383         if (found) {
 384                 IOEventSourceCounter *esc;
 385                 IOUserClientCounter *ucc;
 386
 387                 /* Free up the list of counters */
 388                 while ((esc = SLIST_FIRST(&found->counterList))) {
 389                         SLIST_REMOVE_HEAD(&found->counterList, link);
 390                         kfree(esc, sizeof(IOEventSourceCounter));
 391                 }
 392
 393                 /* Free up the user client list */
 394                 while ((ucc = SLIST_FIRST(&found->userClientList))) {
 395                         SLIST_REMOVE_HEAD(&found->userClientList, link);
 396                         kfree(ucc, sizeof(IOUserClientCounter));
 397                 }
 398
 399                 /* Remove from class tree */
 400                 RB_REMOVE(ClassTree, &classHead, found);
 401
 402                 /* Remove from parent */
 403                 SLIST_REMOVE(&found->parentKext->classList, found, ClassNode, lLink);
 404
 405                 /* Finally, free the class node */
 406                 kfree(found, sizeof(ClassNode));
 407
 408                 sequenceID++;
 409                 registeredClasses--;
 410         }
 411         else {
 412                 panic("IOStatistics::onClassRemoved: cannot find class: %s", metaClass->getClassName());
 413         }
 414
 415         IORWLockUnlock(lock);
 416 }
 417
 418 IOEventSourceCounter *IOStatistics::registerEventSource(OSObject *inOwner)
 419 {
 420         IOEventSourceCounter *counter = NULL;
 421         ClassNode sought, *found = NULL;
 422         boolean_t createDummyCounter = FALSE;
 423
 424         assert(inOwner);
 425
 426         if (!enabled) {
 427                 return NULL;
 428         }
 429
 430         counter = (IOEventSourceCounter*)kalloc(sizeof(IOEventSourceCounter));
 431         if (!counter) {
 432                 return NULL;
 433         }
 434
 435         memset(counter, 0, sizeof(IOEventSourceCounter));
 436
 437         IORWLockWrite(lock);
 438
 439         /* Workaround for <rdar://problem/7158117> - create a dummy counter when inOwner is bad.
 440          * We use retainCount here as our best indication that the pointer is awry.
 441          */
 442         if (inOwner->retainCount > 0xFFFFFF) {
 443                 kprintf("IOStatistics::registerEventSource - bad metaclass %p\n", inOwner);
 444                 createDummyCounter = TRUE;
 445         }
 446         else {
 447                 sought.metaClass = inOwner->getMetaClass();
 448                 found = RB_FIND(ClassTree, &classHead, &sought);
 449         }
 450
 451         if (found) {
 452                 counter->parentClass = found;
 453                 SLIST_INSERT_HEAD(&found->counterList, counter, link);
 454                 registeredCounters++;
 455         }
 456
 457         if (!(createDummyCounter || found)) {
 458                 panic("IOStatistics::registerEventSource: cannot find parent class: %s", inOwner->getMetaClass()->getClassName());
 459         }
 460
 461         IORWLockUnlock(lock);
 462
 463         return counter;
 464 }
 465
 466 void IOStatistics::unregisterEventSource(IOEventSourceCounter *counter)
 467 {
 468         if (!counter) {
 469                 return;
 470         }
 471
 472         IORWLockWrite(lock);
 473
 474         if (counter->parentClass) {
 475                 SLIST_REMOVE(&counter->parentClass->counterList, counter, IOEventSourceCounter, link);
 476                 registeredCounters--;
 477         }
 478         kfree(counter, sizeof(IOEventSourceCounter));
 479
 480         IORWLockUnlock(lock);
 481 }
 482
 483 IOWorkLoopCounter* IOStatistics::registerWorkLoop(IOWorkLoop *workLoop)
 484 {
 485         IOWorkLoopCounter *counter = NULL;
 486         KextNode *found;
 487
 488         assert(workLoop);
 489
 490         if (!enabled) {
 491                 return NULL;
 492         }
 493
 494         counter = (IOWorkLoopCounter*)kalloc(sizeof(IOWorkLoopCounter));
 495         if (!counter) {
 496                 return NULL;
 497         }
 498
 499         memset(counter, 0, sizeof(IOWorkLoopCounter));
 500
 501         found = getKextNodeFromBacktrace(TRUE);
 502         if (!found) {
 503                 panic("IOStatistics::registerWorkLoop: cannot find parent kext");
 504         }
 505
 506         counter->parentKext = found;
 507         counter->workLoop = workLoop;
 508         RB_INIT(&counter->dependencyHead);
 509         SLIST_INSERT_HEAD(&found->workLoopList, counter, link);
 510         registeredWorkloops++;
 511
 512         releaseKextNode(found);
 513
 514         return counter;
 515 }
 516
 517 void IOStatistics::unregisterWorkLoop(IOWorkLoopCounter *counter)
 518 {
 519         if (!counter) {
 520                 return;
 521         }
 522
 523         IORWLockWrite(lock);
 524         if (counter->parentKext) {
 525                 SLIST_REMOVE(&counter->parentKext->workLoopList, counter, IOWorkLoopCounter, link);
 526         }
 527         kfree(counter, sizeof(IOWorkLoopCounter));
 528         registeredWorkloops--;
 529
 530         IORWLockUnlock(lock);
 531 }
 532
 533 IOUserClientCounter *IOStatistics::registerUserClient(IOUserClient *userClient)
 534 {
 535         ClassNode sought, *found;
 536         IOUserClientCounter *counter = NULL;
 537
 538         assert(userClient);
 539
 540         if (!enabled) {
 541                 return NULL;
 542         }
 543
 544         counter = (IOUserClientCounter*)kalloc(sizeof(IOUserClientCounter));
 545         if (!counter) {
 546                 return NULL;
 547         }
 548
 549         memset(counter, 0, sizeof(IOUserClientCounter));
 550
 551         IORWLockWrite(lock);
 552
 553         sought.metaClass = userClient->getMetaClass();
 554
 555         found = RB_FIND(ClassTree, &classHead, &sought);
 556         if (found) {
 557                 counter->parentClass = found;
 558                 SLIST_INSERT_HEAD(&found->userClientList, counter, link);
 559         }
 560         else {
 561                 panic("IOStatistics::registerUserClient: cannot find parent class: %s", sought.metaClass->getClassName());
 562         }
 563
 564         IORWLockUnlock(lock);
 565
 566         return counter;
 567 }
 568
 569 void IOStatistics::unregisterUserClient(IOUserClientCounter *counter)
 570 {
 571         if (!counter) {
 572                 return;
 573         }
 574
 575         IORWLockWrite(lock);
 576
 577         SLIST_REMOVE(&counter->parentClass->userClientList, counter, IOUserClientCounter, link);
 578         kfree(counter, sizeof(IOUserClientCounter));
 579
 580         IORWLockUnlock(lock);
 581 }
 582
 583 void IOStatistics::attachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
 584 {
 585         if (!wlc) {
 586         return;
 587         }
 588
 589         IORWLockWrite(lock);
 590
 591         if (!nextWorkLoopDependency) {
 592                 return;
 593         }
 594
 595         attachedEventSources++;
 596         wlc->attachedEventSources++;
 597
 598         /* Track the kext dependency */
 599         nextWorkLoopDependency->loadTag = esc->parentClass->parentKext->loadTag;
 600         if (NULL == RB_INSERT(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, nextWorkLoopDependency)) {
 601                 nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
 602         }
 603
 604         IORWLockUnlock(lock);
 605 }
 606
 607 void IOStatistics::detachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
 608 {
 609         IOWorkLoopDependency sought, *found;
 610
 611         if (!wlc) {
 612                 return;
 613         }
 614
 615         IORWLockWrite(lock);
 616
 617         attachedEventSources--;
 618         wlc->attachedEventSources--;
 619
 620         sought.loadTag = esc->parentClass->parentKext->loadTag;
 621
 622         found = RB_FIND(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, &sought);
 623         if (found) {
 624                 RB_REMOVE(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, found);
 625                 kfree(found, sizeof(IOWorkLoopDependency));
 626         }
 627
 628         IORWLockUnlock(lock);
 629 }
 630
 631 int IOStatistics::getStatistics(sysctl_req *req)
 632 {
 633         int error;
 634         uint32_t calculatedSize, size;
 635         char *buffer, *ptr;
 636         IOStatisticsHeader *header;
 637
 638         assert(IOStatistics::enabled && req);
 639
 640         IORWLockRead(IOStatistics::lock);
 641
 642         /* Work out how much we need to allocate. IOStatisticsKext is of variable size. */
 643         calculatedSize = sizeof(IOStatisticsHeader) +
 644                                          sizeof(IOStatisticsGlobal) +
 645                                         (sizeof(IOStatisticsKext) * loadedKexts) + (sizeof(uint32_t) * registeredClasses) +
 646                                         (sizeof(IOStatisticsMemory) * loadedKexts) +
 647                                         (sizeof(IOStatisticsClass) * registeredClasses) +
 648                                         (sizeof(IOStatisticsCounter) * registeredClasses) +
 649                                         (sizeof(IOStatisticsKextIdentifier) * loadedKexts) +
 650                                         (sizeof(IOStatisticsClassName) * registeredClasses);
 651
 652         /* Size request? */
 653         if (req->oldptr == USER_ADDR_NULL) {
 654                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 655                 goto exit;
 656         }
 657
 658         /* Read only */
 659         if (req->newptr != USER_ADDR_NULL) {
 660                 error = EPERM;
 661                 goto exit;
 662         }
 663
 664         buffer = (char*)kalloc(calculatedSize);
 665         if (!buffer) {
 666                 error = ENOMEM;
 667                 goto exit;
 668         }
 669
 670         memset(buffer, 0, calculatedSize);
 671
 672         ptr = buffer;
 673
 674         header = (IOStatisticsHeader*)((void*)ptr);
 675
 676         header->sig = IOSTATISTICS_SIG;
 677         header->ver = IOSTATISTICS_VER;
 678
 679         header->seq = sequenceID;
 680
 681         ptr += sizeof(IOStatisticsHeader);
 682
 683         /* Global data - seq, timers, interrupts, etc) */
 684         header->globalStatsOffset = sizeof(IOStatisticsHeader);
 685         size = copyGlobalStatistics((IOStatisticsGlobal*)((void*)ptr));
 686         ptr += size;
 687
 688         /* Kext statistics */
 689         header->kextStatsOffset = header->globalStatsOffset + size;
 690         size = copyKextStatistics((IOStatisticsKext*)((void*)ptr));
 691         ptr += size;
 692
 693         /* Memory allocation info */
 694         header->memoryStatsOffset = header->kextStatsOffset + size;
 695         size = copyMemoryStatistics((IOStatisticsMemory*)((void*)ptr));
 696         ptr += size;
 697
 698         /* Class statistics */
 699         header->classStatsOffset = header->memoryStatsOffset + size;
 700         size = copyClassStatistics((IOStatisticsClass*)((void*)ptr));
 701         ptr += size;
 702
 703         /* Dynamic class counter data */
 704         header->counterStatsOffset = header->classStatsOffset + size;
 705         size = copyCounterStatistics((IOStatisticsCounter*)((void*)ptr));
 706         ptr += size;
 707
 708         /* Kext identifiers */
 709         header->kextIdentifiersOffset = header->counterStatsOffset + size;
 710         size = copyKextIdentifiers((IOStatisticsKextIdentifier*)((void*)ptr));
 711         ptr += size;
 712
 713         /* Class names */
 714         header->classNamesOffset = header->kextIdentifiersOffset + size;
 715         size = copyClassNames((IOStatisticsClassName*)ptr);
 716         ptr += size;
 717
 718         LOG(2, "IOStatistics::getStatistics - calculatedSize 0x%x, kexts 0x%x, classes 0x%x.\n",
 719                 calculatedSize, loadedKexts, registeredClasses);
 720
 721         assert( (uint32_t)(ptr - buffer) == calculatedSize );
 722
 723         error = SYSCTL_OUT(req, buffer, calculatedSize);
 724
 725         kfree(buffer, calculatedSize);
 726
 727 exit:
 728         IORWLockUnlock(IOStatistics::lock);
 729         return error;
 730 }
 731
 732 int IOStatistics::getWorkLoopStatistics(sysctl_req *req)
 733 {
 734         int error;
 735         uint32_t calculatedSize, size;
 736         char *buffer;
 737         IOStatisticsWorkLoopHeader *header;
 738
 739         assert(IOStatistics::enabled && req);
 740
 741         IORWLockRead(IOStatistics::lock);
 742
 743         /* Approximate how much we need to allocate (worse case estimate) */
 744         calculatedSize = sizeof(IOStatisticsWorkLoop) * registeredWorkloops +
 745                                          sizeof(uint32_t) * attachedEventSources;
 746
 747         /* Size request? */
 748         if (req->oldptr == USER_ADDR_NULL) {
 749                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 750                 goto exit;
 751         }
 752
 753         /* Read only */
 754         if (req->newptr != USER_ADDR_NULL) {
 755                 error = EPERM;
 756                 goto exit;
 757         }
 758
 759         buffer = (char*)kalloc(calculatedSize);
 760         if (!buffer) {
 761                 error = ENOMEM;
 762                 goto exit;
 763         }
 764
 765         header = (IOStatisticsWorkLoopHeader*)((void*)buffer);
 766
 767         header->sig = IOSTATISTICS_SIG_WORKLOOP;
 768         header->ver = IOSTATISTICS_VER;
 769
 770         header->seq = sequenceID;
 771
 772         header->workloopCount = registeredWorkloops;
 773
 774         size = copyWorkLoopStatistics(&header->workLoopStats);
 775
 776         LOG(2, "IOStatistics::getWorkLoopStatistics: calculatedSize %d, size %d\n", calculatedSize, size);
 777
 778         assert( size <= calculatedSize );
 779
 780         error = SYSCTL_OUT(req, buffer, size);
 781
 782         kfree(buffer, calculatedSize);
 783
 784 exit:
 785         IORWLockUnlock(IOStatistics::lock);
 786         return error;
 787 }
 788
 789 int IOStatistics::getUserClientStatistics(sysctl_req *req)
 790 {
 791         int error;
 792         uint32_t calculatedSize, size;
 793         char *buffer;
 794         uint32_t requestedLoadTag = 0;
 795         IOStatisticsUserClientHeader *header;
 796
 797         assert(IOStatistics::enabled && req);
 798
 799         IORWLockRead(IOStatistics::lock);
 800
 801         /* Work out how much we need to allocate */
 802         calculatedSize = sizeof(IOStatisticsUserClientHeader) +
 803                                          sizeof(IOStatisticsUserClientCall) * IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS * loadedKexts;
 804
 805         /* Size request? */
 806         if (req->oldptr == USER_ADDR_NULL) {
 807                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 808                 goto exit;
 809         }
 810
 811         /* Kext request (potentially) valid? */
 812         if (!req->newptr || req->newlen < sizeof(requestedLoadTag)) {
 813                 error = EINVAL;
 814                 goto exit;
 815         }
 816
 817         SYSCTL_IN(req, &requestedLoadTag, sizeof(requestedLoadTag));
 818
 819         LOG(2, "IOStatistics::getUserClientStatistics - requesting kext w/load tag: %d\n", requestedLoadTag);
 820
 821         buffer = (char*)kalloc(calculatedSize);
 822         if (!buffer) {
 823                 error = ENOMEM;
 824                 goto exit;
 825         }
 826
 827         header = (IOStatisticsUserClientHeader*)((void*)buffer);
 828
 829         header->sig = IOSTATISTICS_SIG_USERCLIENT;
 830         header->ver = IOSTATISTICS_VER;
 831
 832         header->seq = sequenceID;
 833
 834         header->processes = 0;
 835
 836         size = copyUserClientStatistics(header, requestedLoadTag);
 837
 838         assert((sizeof(IOStatisticsUserClientHeader) + size) <= calculatedSize);
 839
 840         if (size) {
 841                 error = SYSCTL_OUT(req, buffer, sizeof(IOStatisticsUserClientHeader) + size);
 842         }
 843         else {
 844                 error = EINVAL;
 845         }
 846
 847         kfree(buffer, calculatedSize);
 848
 849 exit:
 850         IORWLockUnlock(IOStatistics::lock);
 851         return error;
 852 }
 853
 854 uint32_t IOStatistics::copyGlobalStatistics(IOStatisticsGlobal *stats)
 855 {
 856         stats->kextCount = loadedKexts;
 857         stats->classCount = registeredClasses;
 858         stats->workloops = registeredWorkloops;
 859
 860         return sizeof(IOStatisticsGlobal);
 861 }
 862
 863 uint32_t IOStatistics::copyKextStatistics(IOStatisticsKext *stats)
 864 {
 865         KextNode *ke;
 866         ClassNode *ce;
 867         uint32_t index = 0;
 868
 869         RB_FOREACH(ke, KextTree, &kextHead) {
 870                 stats->loadTag = ke->loadTag;
 871                 ke->kext->getSizeInfo(&stats->loadSize, &stats->wiredSize);
 872
 873                 stats->classes = ke->classes;
 874
 875                 /* Append indices of owned classes */
 876                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 877                         stats->classIndexes[index++] = ce->classID;
 878                 }
 879
 880                 stats = (IOStatisticsKext *)((void*)((char*)stats + sizeof(IOStatisticsKext) + (ke->classes * sizeof(uint32_t))));
 881         }
 882
 883         return (sizeof(IOStatisticsKext) * loadedKexts + sizeof(uint32_t) * registeredClasses);
 884 }
 885
 886 uint32_t IOStatistics::copyMemoryStatistics(IOStatisticsMemory *stats)
 887 {
 888         KextNode *ke;
 889
 890         RB_FOREACH(ke, KextTree, &kextHead) {
 891                 stats->allocatedSize = ke->memoryCounters[kIOStatisticsMalloc];
 892                 stats->freedSize = ke->memoryCounters[kIOStatisticsFree];
 893                 stats->allocatedAlignedSize = ke->memoryCounters[kIOStatisticsMallocAligned];
 894                 stats->freedAlignedSize = ke->memoryCounters[kIOStatisticsFreeAligned];
 895                 stats->allocatedContiguousSize = ke->memoryCounters[kIOStatisticsMallocContiguous];
 896                 stats->freedContiguousSize = ke->memoryCounters[kIOStatisticsFreeContiguous];
 897                 stats->allocatedPageableSize = ke->memoryCounters[kIOStatisticsMallocPageable];
 898                 stats->freedPageableSize = ke->memoryCounters[kIOStatisticsFreePageable];
 899                 stats++;
 900         }
 901
 902         return (sizeof(IOStatisticsMemory) * loadedKexts);
 903 }
 904
 905 uint32_t IOStatistics::copyClassStatistics(IOStatisticsClass *stats)
 906 {
 907         KextNode *ke;
 908         ClassNode *ce;
 909
 910         RB_FOREACH(ke, KextTree, &kextHead) {
 911                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 912                         stats->classID = ce->classID;
 913                         stats->superClassID = ce->superClassID;
 914                         stats->classSize = ce->metaClass->getClassSize();
 915
 916                         stats++;
 917                 }
 918         }
 919
 920         return sizeof(IOStatisticsClass) * registeredClasses;
 921 }
 922
 923 uint32_t IOStatistics::copyCounterStatistics(IOStatisticsCounter *stats)
 924 {
 925         KextNode *ke;
 926         ClassNode *ce;
 927
 928         RB_FOREACH(ke, KextTree, &kextHead) {
 929                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 930                         IOUserClientCounter *userClientCounter;
 931                         IOEventSourceCounter *counter;
 932
 933                         stats->classID = ce->classID;
 934                         stats->classInstanceCount = ce->metaClass->getInstanceCount();
 935
 936                         IOStatisticsUserClients *uc = &stats->userClientStatistics;
 937
 938                         /* User client counters */
 939                         SLIST_FOREACH(userClientCounter, &ce->userClientList, link) {
 940                                 uc->clientCalls += userClientCounter->clientCalls;
 941                                 uc->created++;
 942                         }
 943
 944                         IOStatisticsInterruptEventSources *iec = &stats->interruptEventSourceStatistics;
 945                         IOStatisticsInterruptEventSources *fiec = &stats->filterInterruptEventSourceStatistics;
 946                         IOStatisticsTimerEventSources *tec = &stats->timerEventSourceStatistics;
 947                         IOStatisticsCommandGates *cgc = &stats->commandGateStatistics;
 948                         IOStatisticsCommandQueues *cqc = &stats->commandQueueStatistics;
 949                         IOStatisticsDerivedEventSources *dec = &stats->derivedEventSourceStatistics;
 950
 951                         /* Event source counters */
 952                         SLIST_FOREACH(counter, &ce->counterList, link) {
 953                                 switch (counter->type) {
 954                                         case kIOStatisticsInterruptEventSourceCounter:
 955                                                 iec->created++;
 956                                                 iec->produced += counter->u.interrupt.produced;
 957                                                 iec->checksForWork += counter->u.interrupt.checksForWork;
 958                                                 break;
 959                                         case kIOStatisticsFilterInterruptEventSourceCounter:
 960                                                 fiec->created++;
 961                                                 fiec->produced += counter->u.filter.produced;
 962                                                 fiec->checksForWork += counter->u.filter.checksForWork;
 963                                                 break;
 964                                         case kIOStatisticsTimerEventSourceCounter:
 965                                                 tec->created++;
 966                                                 tec->timeouts += counter->u.timer.timeouts;
 967                                                 tec->checksForWork += counter->u.timer.checksForWork;
 968                                                 tec->timeOnGate += counter->timeOnGate;
 969                                                 tec->closeGateCalls += counter->closeGateCalls;
 970                                                 tec->openGateCalls += counter->openGateCalls;
 971                                                 break;
 972                                         case kIOStatisticsCommandGateCounter:
 973                                                 cgc->created++;
 974                                                 cgc->timeOnGate += counter->timeOnGate;
 975                                                 cgc->actionCalls += counter->u.commandGate.actionCalls;
 976                                                 break;
 977                                         case kIOStatisticsCommandQueueCounter:
 978                                                 cqc->created++;
 979                                                 cqc->actionCalls += counter->u.commandQueue.actionCalls;
 980                                                 break;
 981                                         case kIOStatisticsDerivedEventSourceCounter:
 982                                                 dec->created++;
 983                                                 dec->timeOnGate += counter->timeOnGate;
 984                                                 dec->closeGateCalls += counter->closeGateCalls;
 985                                                 dec->openGateCalls += counter->openGateCalls;
 986                                                 break;
 987                                         default:
 988                                                 break;
 989                                 }
 990                         }
 991
 992                         stats++;
 993                 }
 994         }
 995
 996         return sizeof(IOStatisticsCounter) * registeredClasses;
 997 }
 998
 999 uint32_t IOStatistics::copyKextIdentifiers(IOStatisticsKextIdentifier *kextIDs)
1000 {
1001         KextNode *ke;
1002
1003         RB_FOREACH(ke, KextTree, &kextHead) {
1004                 strncpy(kextIDs->identifier, ke->kext->getIdentifierCString(), kIOStatisticsDriverNameLength);
1005                 kextIDs++;
1006         }
1007
1008         return (sizeof(IOStatisticsKextIdentifier) * loadedKexts);
1009 }
1010
1011 uint32_t IOStatistics::copyClassNames(IOStatisticsClassName *classNames)
1012 {
1013         KextNode *ke;
1014         ClassNode *ce;
1015
1016         RB_FOREACH(ke, KextTree, &kextHead) {
1017                 SLIST_FOREACH(ce, &ke->classList, lLink) {
1018                         strncpy(classNames->name, ce->metaClass->getClassName(), kIOStatisticsClassNameLength);
1019                         classNames++;
1020                 }
1021         }
1022
1023         return (sizeof(IOStatisticsClassName) * registeredClasses);
1024 }
1025
1026 uint32_t IOStatistics::copyWorkLoopStatistics(IOStatisticsWorkLoop *stats)
1027 {
1028         KextNode *ke;
1029         IOWorkLoopCounter *wlc;
1030         IOWorkLoopDependency *dependentNode;
1031         uint32_t size, accumulatedSize = 0;
1032
1033         RB_FOREACH(ke, KextTree, &kextHead) {
1034                 SLIST_FOREACH(wlc, &ke->workLoopList, link) {
1035                         stats->kextLoadTag = ke->loadTag;
1036                         stats->attachedEventSources = wlc->attachedEventSources;
1037                         stats->timeOnGate = wlc->timeOnGate;
1038                         stats->dependentKexts = 0;
1039                         RB_FOREACH(dependentNode, IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead) {
1040                                 stats->dependentKextLoadTags[stats->dependentKexts] = dependentNode->loadTag;
1041                                 stats->dependentKexts++;
1042                         }
1043
1044                         size = sizeof(IOStatisticsWorkLoop) + (sizeof(uint32_t) * stats->dependentKexts);
1045
1046                         accumulatedSize += size;
1047                         stats = (IOStatisticsWorkLoop*)((void*)((char*)stats + size));
1048                 }
1049         }
1050
1051         return accumulatedSize;
1052 }
1053
1054 uint32_t IOStatistics::copyUserClientStatistics(IOStatisticsUserClientHeader *stats, uint32_t loadTag)
1055 {
1056         KextNode *sought, *found = NULL;
1057         uint32_t procs = 0;
1058         IOUserClientProcessEntry *processEntry;
1059
1060         RB_FOREACH(sought, KextTree, &kextHead) {
1061                 if (sought->loadTag == loadTag) {
1062                         found = sought;
1063                         break;
1064                 }
1065         }
1066
1067         if (!found) {
1068                 return 0;
1069         }
1070
1071         TAILQ_FOREACH(processEntry, &found->userClientCallList, link) {
1072                 strncpy(stats->userClientCalls[procs].processName, processEntry->processName, kIOStatisticsProcessNameLength);
1073                 stats->userClientCalls[procs].pid = processEntry->pid;
1074                 stats->userClientCalls[procs].calls = processEntry->calls;
1075                 stats->processes++;
1076                 procs++;
1077         }
1078
1079         return sizeof(IOStatisticsUserClientCall) * stats->processes;
1080 }
1081
1082 void IOStatistics::storeUserClientCallInfo(IOUserClient *userClient, IOUserClientCounter *counter)
1083 {
1084         OSString *ossUserClientCreator = NULL;
1085         int32_t pid = -1;
1086         KextNode *parentKext;
1087         IOUserClientProcessEntry *entry, *nextEntry, *prevEntry = NULL;
1088         uint32_t count = 0;
1089         const char *ptr = NULL;
1090         OSObject *obj;
1091
1092         /* TODO: see if this can be more efficient */
1093         obj = userClient->copyProperty("IOUserClientCreator",
1094                                         gIOServicePlane,
1095                                         kIORegistryIterateRecursively | kIORegistryIterateParents);
1096
1097         if (!obj)
1098                 goto err_nounlock;
1099
1100         ossUserClientCreator = OSDynamicCast(OSString, obj);
1101
1102         if (ossUserClientCreator) {
1103                 uint32_t len, lenIter = 0;
1104
1105                 ptr = ossUserClientCreator->getCStringNoCopy();
1106                 len = ossUserClientCreator->getLength();
1107
1108                 while ((*ptr != ' ') && (lenIter < len)) {
1109                         ptr++;
1110                         lenIter++;
1111                 }
1112
1113                 if (lenIter < len) {
1114                         ptr++; // Skip the space
1115                         lenIter++;
1116                         pid = 0;
1117                         while ( (*ptr != ',') && (lenIter < len)) {
1118                                 pid = pid*10 + (*ptr - '0');
1119                                 ptr++;
1120                                 lenIter++;
1121                         }
1122
1123                         if(lenIter == len) {
1124                                 pid = -1;
1125                         } else {
1126                                 ptr += 2;
1127                         }
1128                 }
1129         }
1130
1131         if (-1 == pid)
1132                 goto err_nounlock;
1133
1134         IORWLockWrite(lock);
1135
1136         parentKext = counter->parentClass->parentKext;
1137
1138         TAILQ_FOREACH(entry, &parentKext->userClientCallList, link) {
1139                 if (entry->pid == pid) {
1140                         /* Found, so increment count and move to the head */
1141                         entry->calls++;
1142                         if (count) {
1143                                 TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1144                                 break;
1145                         }
1146                         else {
1147                                 /* At the head already, so increment and return */
1148                                 goto err_unlock;
1149                         }
1150                 }
1151
1152                 count++;
1153         }
1154
1155         if (!entry) {
1156                 if (count == IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS) {
1157                         /* Max elements hit, so reuse the last */
1158                         entry = TAILQ_LAST(&parentKext->userClientCallList, ProcessEntryList);
1159                         TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1160                 }
1161                 else {
1162                         /* Otherwise, allocate a new entry */
1163                         entry = (IOUserClientProcessEntry*)kalloc(sizeof(IOUserClientProcessEntry));
1164                         if (!entry) {
1165                             IORWLockUnlock(lock);
1166                                 return;
1167                         }
1168                 }
1169
1170                 strncpy(entry->processName, ptr, kIOStatisticsProcessNameLength);
1171                 entry->pid = pid;
1172                 entry->calls = 1;
1173         }
1174
1175         TAILQ_FOREACH(nextEntry, &parentKext->userClientCallList, link) {
1176                 if (nextEntry->calls <= entry->calls)
1177                         break;
1178
1179                 prevEntry = nextEntry;
1180         }
1181
1182         if (!prevEntry)
1183                 TAILQ_INSERT_HEAD(&parentKext->userClientCallList, entry, link);
1184         else
1185                 TAILQ_INSERT_AFTER(&parentKext->userClientCallList, prevEntry, entry, link);
1186
1187 err_unlock:
1188         IORWLockUnlock(lock);
1189
1190 err_nounlock:
1191         if (obj)
1192                 obj->release();
1193 }
1194
1195 void IOStatistics::countUserClientCall(IOUserClient *client) {
1196         IOUserClient::ExpansionData *data;
1197         IOUserClientCounter *counter;
1198
1199         /* Guard against an uninitialized client object - <rdar://problem/8577946> */
1200         if (!(data = client->reserved)) {
1201                 return;
1202         }
1203
1204         if ((counter = data->counter)) {
1205                 storeUserClientCallInfo(client, counter);
1206                 OSIncrementAtomic(&counter->clientCalls);
1207         }
1208 }
1209
1210 KextNode *IOStatistics::getKextNodeFromBacktrace(boolean_t write) {
1211         const uint32_t btMin = 3;
1212
1213         void *bt[16];
1214         unsigned btCount = sizeof(bt) / sizeof(bt[0]);
1215         vm_offset_t *scanAddr = NULL;
1216         uint32_t i;
1217         KextNode *found = NULL, *ke = NULL;
1218
1219         /*
1220          * Gathering the backtrace is a significant source of
1221          * overhead. OSBacktrace does many safety checks that
1222          * are not needed in this situation.
1223          */
1224         btCount = fastbacktrace((uintptr_t*)bt, btCount);
1225
1226         if (write) {
1227                 IORWLockWrite(lock);
1228         } else {
1229                 IORWLockRead(lock);
1230         }
1231
1232         /* Ignore first levels */
1233         scanAddr = (vm_offset_t *)&bt[btMin - 1];
1234
1235         for (i = btMin - 1; i < btCount; i++, scanAddr++) {
1236                 ke = RB_ROOT(&kextAddressHead);
1237                 while (ke) {
1238                         if (*scanAddr < ke->address) {
1239                                 ke = RB_LEFT(ke, addressLink);
1240                         }
1241                         else {
1242                                 if ((*scanAddr < ke->address_end) && (*scanAddr >= ke->address)) {
1243                                         if (!ke->kext->isKernelComponent()) {
1244                                                 return ke;
1245                                         } else {
1246                                                 found = ke;
1247                                         }
1248                                 }
1249                                 ke = RB_RIGHT(ke, addressLink);
1250                         }
1251                 }
1252         }
1253
1254         if (!found) {
1255                 IORWLockUnlock(lock);
1256         }
1257
1258         return found;
1259 }
1260
1261 void IOStatistics::releaseKextNode(KextNode *node) {
1262 #pragma unused(node)
1263         IORWLockUnlock(lock);
1264 }
1265
1266 /* IOLib allocations */
1267 void IOStatistics::countAlloc(uint32_t index, vm_size_t size) {
1268         KextNode *ke;
1269
1270         if (!enabled) {
1271                 return;
1272         }
1273
1274         ke = getKextNodeFromBacktrace(FALSE);
1275         if (ke) {
1276                 OSAddAtomic(size, &ke->memoryCounters[index]);
1277                 releaseKextNode(ke);
1278         }
1279 }
1280
1281 #endif /* IOKITSTATS */