iokit/Kernel/IOStatistics.cpp

   1 /*
   2  * Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/sysctl.h>
  30 #include <kern/backtrace.h>
  31 #include <kern/host.h>
  32 #include <kern/zalloc.h>
  33
  34 #include <IOKit/system.h>
  35 #include <libkern/c++/OSKext.h>
  36 #include <libkern/OSAtomic.h>
  37
  38 #include <IOKit/IOStatisticsPrivate.h>
  39 #include <IOKit/IOUserClient.h>
  40 #include <IOKit/IOEventSource.h>
  41 #include <IOKit/IOKitDebug.h>
  42
  43 #if IOKITSTATS
  44 bool IOStatistics::enabled = false;
  45
  46 uint32_t IOStatistics::sequenceID = 0;
  47
  48 uint32_t IOStatistics::lastClassIndex = 0;
  49 uint32_t IOStatistics::lastKextIndex = 0;
  50
  51 uint32_t IOStatistics::loadedKexts = 0;
  52 uint32_t IOStatistics::registeredClasses = 0;
  53 uint32_t IOStatistics::registeredCounters = 0;
  54 uint32_t IOStatistics::registeredWorkloops = 0;
  55
  56 uint32_t IOStatistics::attachedEventSources = 0;
  57
  58 IOWorkLoopDependency *IOStatistics::nextWorkLoopDependency = NULL;
  59
  60 /* Logging */
  61
  62 #define LOG_LEVEL 0
  63
  64 #define LOG(level, format, ...) \
  65 do { \
  66         if (level <= LOG_LEVEL) \
  67                 printf(format, ##__VA_ARGS__); \
  68 } while (0)
  69
  70 /* Locks */
  71
  72 IORWLock *IOStatistics::lock = NULL;
  73
  74 /* Kext tree */
  75
  76 KextNode *IOStatistics::kextHint = NULL;
  77
  78 IOStatistics::KextTreeHead IOStatistics::kextHead = RB_INITIALIZER(&IOStatistics::kextHead);
  79
  80 int
  81 IOStatistics::kextNodeCompare(KextNode *e1, KextNode *e2)
  82 {
  83         if (e1->kext < e2->kext) {
  84                 return -1;
  85         } else if (e1->kext > e2->kext) {
  86                 return 1;
  87         } else {
  88                 return 0;
  89         }
  90 }
  91
  92 RB_GENERATE(IOStatistics::KextTree, KextNode, link, kextNodeCompare);
  93
  94 /* Kext tree ordered by address */
  95
  96 IOStatistics::KextAddressTreeHead IOStatistics::kextAddressHead = RB_INITIALIZER(&IOStatistics::kextAddressHead);
  97
  98 int
  99 IOStatistics::kextAddressNodeCompare(KextNode *e1, KextNode *e2)
 100 {
 101         if (e1->address < e2->address) {
 102                 return -1;
 103         } else if (e1->address > e2->address) {
 104                 return 1;
 105         } else {
 106                 return 0;
 107         }
 108 }
 109
 110 RB_GENERATE(IOStatistics::KextAddressTree, KextNode, addressLink, kextAddressNodeCompare);
 111
 112 /* Class tree */
 113
 114 IOStatistics::ClassTreeHead IOStatistics::classHead = RB_INITIALIZER(&IOStatistics::classHead);
 115
 116 int
 117 IOStatistics::classNodeCompare(ClassNode *e1, ClassNode *e2)
 118 {
 119         if (e1->metaClass < e2->metaClass) {
 120                 return -1;
 121         } else if (e1->metaClass > e2->metaClass) {
 122                 return 1;
 123         } else {
 124                 return 0;
 125         }
 126 }
 127
 128 RB_GENERATE(IOStatistics::ClassTree, ClassNode, tLink, classNodeCompare);
 129
 130 /* Workloop dependencies */
 131
 132 int
 133 IOWorkLoopCounter::loadTagCompare(IOWorkLoopDependency *e1, IOWorkLoopDependency *e2)
 134 {
 135         if (e1->loadTag < e2->loadTag) {
 136                 return -1;
 137         } else if (e1->loadTag > e2->loadTag) {
 138                 return 1;
 139         } else {
 140                 return 0;
 141         }
 142 }
 143
 144 RB_GENERATE(IOWorkLoopCounter::DependencyTree, IOWorkLoopDependency, link, IOWorkLoopCounter::loadTagCompare);
 145
 146 /* sysctl stuff */
 147
 148 static int
 149 oid_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1, int arg2, struct sysctl_req *req)
 150 {
 151         int error = EINVAL;
 152         uint32_t request = arg2;
 153
 154         switch (request) {
 155         case kIOStatisticsGeneral:
 156                 error = IOStatistics::getStatistics(req);
 157                 break;
 158         case kIOStatisticsWorkLoop:
 159                 error = IOStatistics::getWorkLoopStatistics(req);
 160                 break;
 161         case kIOStatisticsUserClient:
 162                 error = IOStatistics::getUserClientStatistics(req);
 163                 break;
 164         default:
 165                 break;
 166         }
 167
 168         return error;
 169 }
 170
 171 SYSCTL_NODE(_debug, OID_AUTO, iokit_statistics, CTLFLAG_RW | CTLFLAG_LOCKED, NULL, "IOStatistics");
 172
 173 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, general,
 174     CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 175     NULL, kIOStatisticsGeneral, oid_sysctl, "S", "");
 176
 177 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, workloop,
 178     CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 179     NULL, kIOStatisticsWorkLoop, oid_sysctl, "S", "");
 180
 181 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, userclient,
 182     CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 183     NULL, kIOStatisticsUserClient, oid_sysctl, "S", "");
 184
 185 void
 186 IOStatistics::initialize()
 187 {
 188         if (enabled) {
 189                 return;
 190         }
 191
 192         /* Only enabled if the boot argument is set. */
 193         if (!(kIOStatistics & gIOKitDebug)) {
 194                 return;
 195         }
 196
 197         sysctl_register_oid(&sysctl__debug_iokit_statistics_general);
 198         sysctl_register_oid(&sysctl__debug_iokit_statistics_workloop);
 199         sysctl_register_oid(&sysctl__debug_iokit_statistics_userclient);
 200
 201         lock = IORWLockAlloc();
 202         if (!lock) {
 203                 return;
 204         }
 205
 206         nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
 207         if (!nextWorkLoopDependency) {
 208                 return;
 209         }
 210
 211         enabled = true;
 212 }
 213
 214 void
 215 IOStatistics::onKextLoad(OSKext *kext, kmod_info_t *kmod_info)
 216 {
 217         KextNode *ke;
 218
 219         assert(kext && kmod_info);
 220
 221         if (!enabled) {
 222                 return;
 223         }
 224
 225         LOG(1, "IOStatistics::onKextLoad: %s, tag %d, address 0x%llx, address end 0x%llx\n",
 226             kext->getIdentifierCString(), kmod_info->id, (uint64_t)kmod_info->address, (uint64_t)(kmod_info->address + kmod_info->size));
 227
 228         ke = (KextNode *)kalloc(sizeof(KextNode));
 229         if (!ke) {
 230                 return;
 231         }
 232
 233         memset(ke, 0, sizeof(KextNode));
 234
 235         ke->kext = kext;
 236         ke->loadTag = kmod_info->id;
 237         ke->address = kmod_info->address;
 238         ke->address_end = kmod_info->address + kmod_info->size;
 239
 240         SLIST_INIT(&ke->classList);
 241         TAILQ_INIT(&ke->userClientCallList);
 242
 243         IORWLockWrite(lock);
 244
 245         RB_INSERT(KextTree, &kextHead, ke);
 246         RB_INSERT(KextAddressTree, &kextAddressHead, ke);
 247
 248         sequenceID++;
 249         loadedKexts++;
 250         lastKextIndex++;
 251
 252         IORWLockUnlock(lock);
 253 }
 254
 255 void
 256 IOStatistics::onKextUnload(OSKext *kext)
 257 {
 258         KextNode sought, *found;
 259
 260         assert(kext);
 261
 262         if (!enabled) {
 263                 return;
 264         }
 265
 266         LOG(1, "IOStatistics::onKextUnload: %s\n", kext->getIdentifierCString());
 267
 268         IORWLockWrite(lock);
 269
 270         sought.kext = kext;
 271         found = RB_FIND(KextTree, &kextHead, &sought);
 272         if (found) {
 273                 IOWorkLoopCounter *wlc;
 274                 IOUserClientProcessEntry *uce;
 275
 276                 /* Disconnect workloop counters; cleanup takes place in unregisterWorkLoop() */
 277                 while ((wlc = SLIST_FIRST(&found->workLoopList))) {
 278                         SLIST_REMOVE_HEAD(&found->workLoopList, link);
 279                         wlc->parentKext = NULL;
 280                 }
 281
 282                 /* Free up the user client list */
 283                 while ((uce = TAILQ_FIRST(&found->userClientCallList))) {
 284                         TAILQ_REMOVE(&found->userClientCallList, uce, link);
 285                         kfree(uce, sizeof(IOUserClientProcessEntry));
 286                 }
 287
 288                 /* Remove from kext trees */
 289                 RB_REMOVE(KextTree, &kextHead, found);
 290                 RB_REMOVE(KextAddressTree, &kextAddressHead, found);
 291
 292                 /*
 293                  * Clear a matching kextHint to avoid use after free in
 294                  * onClassAdded() for a class add after a KEXT unload.
 295                  */
 296                 if (found == kextHint) {
 297                         kextHint = NULL;
 298                 }
 299
 300                 /* Finally, free the class node */
 301                 kfree(found, sizeof(KextNode));
 302
 303                 sequenceID++;
 304                 loadedKexts--;
 305         } else {
 306                 panic("IOStatistics::onKextUnload: cannot find kext: %s", kext->getIdentifierCString());
 307         }
 308
 309         IORWLockUnlock(lock);
 310 }
 311
 312 void
 313 IOStatistics::onClassAdded(OSKext *parentKext, OSMetaClass *metaClass)
 314 {
 315         ClassNode *ce;
 316         KextNode soughtKext, *foundKext = NULL;
 317
 318         assert(parentKext && metaClass);
 319
 320         if (!enabled) {
 321                 return;
 322         }
 323
 324         LOG(1, "IOStatistics::onClassAdded: %s\n", metaClass->getClassName());
 325
 326         ce = (ClassNode *)kalloc(sizeof(ClassNode));
 327         if (!ce) {
 328                 return;
 329         }
 330
 331         memset(ce, 0, sizeof(ClassNode));
 332
 333         IORWLockWrite(lock);
 334
 335         /* Hinted? */
 336         if (kextHint && kextHint->kext == parentKext) {
 337                 foundKext = kextHint;
 338         } else {
 339                 soughtKext.kext = parentKext;
 340                 foundKext = RB_FIND(KextTree, &kextHead, &soughtKext);
 341         }
 342
 343         if (foundKext) {
 344                 ClassNode soughtClass, *foundClass = NULL;
 345                 const OSMetaClass *superClass;
 346
 347                 ce->metaClass = metaClass;
 348                 ce->classID = lastClassIndex++;
 349                 ce->parentKext = foundKext;
 350
 351                 /* Has superclass? */
 352                 superClass = ce->metaClass->getSuperClass();
 353                 if (superClass) {
 354                         soughtClass.metaClass = superClass;
 355                         foundClass = RB_FIND(ClassTree, &classHead, &soughtClass);
 356                 }
 357                 ce->superClassID = foundClass ? foundClass->classID : (uint32_t)(-1);
 358
 359                 SLIST_INIT(&ce->counterList);
 360                 SLIST_INIT(&ce->userClientList);
 361
 362                 RB_INSERT(ClassTree, &classHead, ce);
 363                 SLIST_INSERT_HEAD(&foundKext->classList, ce, lLink);
 364
 365                 foundKext->classes++;
 366
 367                 kextHint = foundKext;
 368
 369                 sequenceID++;
 370                 registeredClasses++;
 371         } else {
 372                 panic("IOStatistics::onClassAdded: cannot find parent kext: %s", parentKext->getIdentifierCString());
 373         }
 374
 375         IORWLockUnlock(lock);
 376 }
 377
 378 void
 379 IOStatistics::onClassRemoved(OSKext *parentKext, OSMetaClass *metaClass)
 380 {
 381         ClassNode sought, *found;
 382
 383         assert(parentKext && metaClass);
 384
 385         if (!enabled) {
 386                 return;
 387         }
 388
 389         LOG(1, "IOStatistics::onClassRemoved: %s\n", metaClass->getClassName());
 390
 391         IORWLockWrite(lock);
 392
 393         sought.metaClass = metaClass;
 394         found = RB_FIND(ClassTree, &classHead, &sought);
 395         if (found) {
 396                 IOEventSourceCounter *esc;
 397                 IOUserClientCounter *ucc;
 398
 399                 /* Free up the list of counters */
 400                 while ((esc = SLIST_FIRST(&found->counterList))) {
 401                         SLIST_REMOVE_HEAD(&found->counterList, link);
 402                         kfree(esc, sizeof(IOEventSourceCounter));
 403                 }
 404
 405                 /* Free up the user client list */
 406                 while ((ucc = SLIST_FIRST(&found->userClientList))) {
 407                         SLIST_REMOVE_HEAD(&found->userClientList, link);
 408                         kfree(ucc, sizeof(IOUserClientCounter));
 409                 }
 410
 411                 /* Remove from class tree */
 412                 RB_REMOVE(ClassTree, &classHead, found);
 413
 414                 /* Remove from parent */
 415                 SLIST_REMOVE(&found->parentKext->classList, found, ClassNode, lLink);
 416
 417                 /* Finally, free the class node */
 418                 kfree(found, sizeof(ClassNode));
 419
 420                 sequenceID++;
 421                 registeredClasses--;
 422         } else {
 423                 panic("IOStatistics::onClassRemoved: cannot find class: %s", metaClass->getClassName());
 424         }
 425
 426         IORWLockUnlock(lock);
 427 }
 428
 429 IOEventSourceCounter *
 430 IOStatistics::registerEventSource(OSObject *inOwner)
 431 {
 432         IOEventSourceCounter *counter = NULL;
 433         ClassNode sought, *found = NULL;
 434         boolean_t createDummyCounter = FALSE;
 435
 436         assert(inOwner);
 437
 438         if (!enabled) {
 439                 return NULL;
 440         }
 441
 442         counter = (IOEventSourceCounter*)kalloc(sizeof(IOEventSourceCounter));
 443         if (!counter) {
 444                 return NULL;
 445         }
 446
 447         memset(counter, 0, sizeof(IOEventSourceCounter));
 448
 449         IORWLockWrite(lock);
 450
 451         /* Workaround for <rdar://problem/7158117> - create a dummy counter when inOwner is bad.
 452          * We use retainCount here as our best indication that the pointer is awry.
 453          */
 454         if (inOwner->retainCount > 0xFFFFFF) {
 455                 kprintf("IOStatistics::registerEventSource - bad metaclass %p\n", inOwner);
 456                 createDummyCounter = TRUE;
 457         } else {
 458                 sought.metaClass = inOwner->getMetaClass();
 459                 found = RB_FIND(ClassTree, &classHead, &sought);
 460         }
 461
 462         if (found) {
 463                 counter->parentClass = found;
 464                 SLIST_INSERT_HEAD(&found->counterList, counter, link);
 465                 registeredCounters++;
 466         }
 467
 468         if (!(createDummyCounter || found)) {
 469                 panic("IOStatistics::registerEventSource: cannot find parent class: %s", inOwner->getMetaClass()->getClassName());
 470         }
 471
 472         IORWLockUnlock(lock);
 473
 474         return counter;
 475 }
 476
 477 void
 478 IOStatistics::unregisterEventSource(IOEventSourceCounter *counter)
 479 {
 480         if (!counter) {
 481                 return;
 482         }
 483
 484         IORWLockWrite(lock);
 485
 486         if (counter->parentClass) {
 487                 SLIST_REMOVE(&counter->parentClass->counterList, counter, IOEventSourceCounter, link);
 488                 registeredCounters--;
 489         }
 490         kfree(counter, sizeof(IOEventSourceCounter));
 491
 492         IORWLockUnlock(lock);
 493 }
 494
 495 IOWorkLoopCounter*
 496 IOStatistics::registerWorkLoop(IOWorkLoop *workLoop)
 497 {
 498         IOWorkLoopCounter *counter = NULL;
 499         KextNode *found;
 500
 501         assert(workLoop);
 502
 503         if (!enabled) {
 504                 return NULL;
 505         }
 506
 507         counter = (IOWorkLoopCounter*)kalloc(sizeof(IOWorkLoopCounter));
 508         if (!counter) {
 509                 return NULL;
 510         }
 511
 512         memset(counter, 0, sizeof(IOWorkLoopCounter));
 513
 514         found = getKextNodeFromBacktrace(TRUE);
 515         if (!found) {
 516                 panic("IOStatistics::registerWorkLoop: cannot find parent kext");
 517         }
 518
 519         counter->parentKext = found;
 520         counter->workLoop = workLoop;
 521         RB_INIT(&counter->dependencyHead);
 522         SLIST_INSERT_HEAD(&found->workLoopList, counter, link);
 523         registeredWorkloops++;
 524
 525         releaseKextNode(found);
 526
 527         return counter;
 528 }
 529
 530 void
 531 IOStatistics::unregisterWorkLoop(IOWorkLoopCounter *counter)
 532 {
 533         if (!counter) {
 534                 return;
 535         }
 536
 537         IORWLockWrite(lock);
 538         if (counter->parentKext) {
 539                 SLIST_REMOVE(&counter->parentKext->workLoopList, counter, IOWorkLoopCounter, link);
 540         }
 541         kfree(counter, sizeof(IOWorkLoopCounter));
 542         registeredWorkloops--;
 543
 544         IORWLockUnlock(lock);
 545 }
 546
 547 IOUserClientCounter *
 548 IOStatistics::registerUserClient(IOUserClient *userClient)
 549 {
 550         ClassNode sought, *found;
 551         IOUserClientCounter *counter = NULL;
 552
 553         assert(userClient);
 554
 555         if (!enabled) {
 556                 return NULL;
 557         }
 558
 559         counter = (IOUserClientCounter*)kalloc(sizeof(IOUserClientCounter));
 560         if (!counter) {
 561                 return NULL;
 562         }
 563
 564         memset(counter, 0, sizeof(IOUserClientCounter));
 565
 566         IORWLockWrite(lock);
 567
 568         sought.metaClass = userClient->getMetaClass();
 569
 570         found = RB_FIND(ClassTree, &classHead, &sought);
 571         if (found) {
 572                 counter->parentClass = found;
 573                 SLIST_INSERT_HEAD(&found->userClientList, counter, link);
 574         } else {
 575                 panic("IOStatistics::registerUserClient: cannot find parent class: %s", sought.metaClass->getClassName());
 576         }
 577
 578         IORWLockUnlock(lock);
 579
 580         return counter;
 581 }
 582
 583 void
 584 IOStatistics::unregisterUserClient(IOUserClientCounter *counter)
 585 {
 586         if (!counter) {
 587                 return;
 588         }
 589
 590         IORWLockWrite(lock);
 591
 592         SLIST_REMOVE(&counter->parentClass->userClientList, counter, IOUserClientCounter, link);
 593         kfree(counter, sizeof(IOUserClientCounter));
 594
 595         IORWLockUnlock(lock);
 596 }
 597
 598 void
 599 IOStatistics::attachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
 600 {
 601         if (!wlc) {
 602                 return;
 603         }
 604
 605         IORWLockWrite(lock);
 606
 607         if (!nextWorkLoopDependency) {
 608                 return;
 609         }
 610
 611         attachedEventSources++;
 612         wlc->attachedEventSources++;
 613
 614         /* Track the kext dependency */
 615         nextWorkLoopDependency->loadTag = esc->parentClass->parentKext->loadTag;
 616         if (NULL == RB_INSERT(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, nextWorkLoopDependency)) {
 617                 nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
 618         }
 619
 620         IORWLockUnlock(lock);
 621 }
 622
 623 void
 624 IOStatistics::detachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
 625 {
 626         IOWorkLoopDependency sought, *found;
 627
 628         if (!wlc) {
 629                 return;
 630         }
 631
 632         IORWLockWrite(lock);
 633
 634         attachedEventSources--;
 635         wlc->attachedEventSources--;
 636
 637         sought.loadTag = esc->parentClass->parentKext->loadTag;
 638
 639         found = RB_FIND(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, &sought);
 640         if (found) {
 641                 RB_REMOVE(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, found);
 642                 kfree(found, sizeof(IOWorkLoopDependency));
 643         }
 644
 645         IORWLockUnlock(lock);
 646 }
 647
 648 int
 649 IOStatistics::getStatistics(sysctl_req *req)
 650 {
 651         int error;
 652         uint32_t calculatedSize, size;
 653         char *buffer, *ptr;
 654         IOStatisticsHeader *header;
 655
 656         assert(IOStatistics::enabled && req);
 657
 658         IORWLockRead(IOStatistics::lock);
 659
 660         /* Work out how much we need to allocate. IOStatisticsKext is of variable size. */
 661         calculatedSize = sizeof(IOStatisticsHeader) +
 662             sizeof(IOStatisticsGlobal) +
 663             (sizeof(IOStatisticsKext) * loadedKexts) + (sizeof(uint32_t) * registeredClasses) +
 664             (sizeof(IOStatisticsMemory) * loadedKexts) +
 665             (sizeof(IOStatisticsClass) * registeredClasses) +
 666             (sizeof(IOStatisticsCounter) * registeredClasses) +
 667             (sizeof(IOStatisticsKextIdentifier) * loadedKexts) +
 668             (sizeof(IOStatisticsClassName) * registeredClasses);
 669
 670         /* Size request? */
 671         if (req->oldptr == USER_ADDR_NULL) {
 672                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 673                 goto exit;
 674         }
 675
 676         /* Read only */
 677         if (req->newptr != USER_ADDR_NULL) {
 678                 error = EPERM;
 679                 goto exit;
 680         }
 681
 682         buffer = (char*)kheap_alloc(KHEAP_TEMP, calculatedSize,
 683             (zalloc_flags_t)(Z_WAITOK | Z_ZERO));
 684         if (!buffer) {
 685                 error = ENOMEM;
 686                 goto exit;
 687         }
 688
 689         ptr = buffer;
 690
 691         header = (IOStatisticsHeader*)((void*)ptr);
 692
 693         header->sig = IOSTATISTICS_SIG;
 694         header->ver = IOSTATISTICS_VER;
 695
 696         header->seq = sequenceID;
 697
 698         ptr += sizeof(IOStatisticsHeader);
 699
 700         /* Global data - seq, timers, interrupts, etc) */
 701         header->globalStatsOffset = sizeof(IOStatisticsHeader);
 702         size = copyGlobalStatistics((IOStatisticsGlobal*)((void*)ptr));
 703         ptr += size;
 704
 705         /* Kext statistics */
 706         header->kextStatsOffset = header->globalStatsOffset + size;
 707         size = copyKextStatistics((IOStatisticsKext*)((void*)ptr));
 708         ptr += size;
 709
 710         /* Memory allocation info */
 711         header->memoryStatsOffset = header->kextStatsOffset + size;
 712         size = copyMemoryStatistics((IOStatisticsMemory*)((void*)ptr));
 713         ptr += size;
 714
 715         /* Class statistics */
 716         header->classStatsOffset = header->memoryStatsOffset + size;
 717         size = copyClassStatistics((IOStatisticsClass*)((void*)ptr));
 718         ptr += size;
 719
 720         /* Dynamic class counter data */
 721         header->counterStatsOffset = header->classStatsOffset + size;
 722         size = copyCounterStatistics((IOStatisticsCounter*)((void*)ptr));
 723         ptr += size;
 724
 725         /* Kext identifiers */
 726         header->kextIdentifiersOffset = header->counterStatsOffset + size;
 727         size = copyKextIdentifiers((IOStatisticsKextIdentifier*)((void*)ptr));
 728         ptr += size;
 729
 730         /* Class names */
 731         header->classNamesOffset = header->kextIdentifiersOffset + size;
 732         size = copyClassNames((IOStatisticsClassName*)ptr);
 733         ptr += size;
 734
 735         LOG(2, "IOStatistics::getStatistics - calculatedSize 0x%x, kexts 0x%x, classes 0x%x.\n",
 736             calculatedSize, loadedKexts, registeredClasses);
 737
 738         assert((uint32_t)(ptr - buffer) == calculatedSize );
 739
 740         error = SYSCTL_OUT(req, buffer, calculatedSize);
 741
 742         kheap_free(KHEAP_TEMP, buffer, calculatedSize);
 743
 744 exit:
 745         IORWLockUnlock(IOStatistics::lock);
 746         return error;
 747 }
 748
 749 int
 750 IOStatistics::getWorkLoopStatistics(sysctl_req *req)
 751 {
 752         int error;
 753         uint32_t calculatedSize, size;
 754         char *buffer;
 755         IOStatisticsWorkLoopHeader *header;
 756
 757         assert(IOStatistics::enabled && req);
 758
 759         IORWLockRead(IOStatistics::lock);
 760
 761         /* Approximate how much we need to allocate (worse case estimate) */
 762         calculatedSize = sizeof(IOStatisticsWorkLoop) * registeredWorkloops +
 763             sizeof(uint32_t) * attachedEventSources;
 764
 765         /* Size request? */
 766         if (req->oldptr == USER_ADDR_NULL) {
 767                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 768                 goto exit;
 769         }
 770
 771         /* Read only */
 772         if (req->newptr != USER_ADDR_NULL) {
 773                 error = EPERM;
 774                 goto exit;
 775         }
 776
 777         buffer = (char*)kheap_alloc(KHEAP_TEMP, calculatedSize,
 778             (zalloc_flags_t)(Z_WAITOK | Z_ZERO));
 779         if (!buffer) {
 780                 error = ENOMEM;
 781                 goto exit;
 782         }
 783         header = (IOStatisticsWorkLoopHeader*)((void*)buffer);
 784
 785         header->sig = IOSTATISTICS_SIG_WORKLOOP;
 786         header->ver = IOSTATISTICS_VER;
 787
 788         header->seq = sequenceID;
 789
 790         header->workloopCount = registeredWorkloops;
 791
 792         size = copyWorkLoopStatistics(&header->workLoopStats);
 793
 794         LOG(2, "IOStatistics::getWorkLoopStatistics: calculatedSize %d, size %d\n", calculatedSize, size);
 795
 796         assert( size <= calculatedSize );
 797
 798         error = SYSCTL_OUT(req, buffer, size);
 799
 800         kheap_free(KHEAP_TEMP, buffer, calculatedSize);
 801
 802 exit:
 803         IORWLockUnlock(IOStatistics::lock);
 804         return error;
 805 }
 806
 807 int
 808 IOStatistics::getUserClientStatistics(sysctl_req *req)
 809 {
 810         int error;
 811         uint32_t calculatedSize, size;
 812         char *buffer;
 813         uint32_t requestedLoadTag = 0;
 814         IOStatisticsUserClientHeader *header;
 815
 816         assert(IOStatistics::enabled && req);
 817
 818         IORWLockRead(IOStatistics::lock);
 819
 820         /* Work out how much we need to allocate */
 821         calculatedSize = sizeof(IOStatisticsUserClientHeader) +
 822             sizeof(IOStatisticsUserClientCall) * IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS * loadedKexts;
 823
 824         /* Size request? */
 825         if (req->oldptr == USER_ADDR_NULL) {
 826                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 827                 goto exit;
 828         }
 829
 830         /* Kext request (potentially) valid? */
 831         if (!req->newptr || req->newlen < sizeof(requestedLoadTag)) {
 832                 error = EINVAL;
 833                 goto exit;
 834         }
 835
 836         error = SYSCTL_IN(req, &requestedLoadTag, sizeof(requestedLoadTag));
 837         if (error) {
 838                 goto exit;
 839         }
 840
 841         LOG(2, "IOStatistics::getUserClientStatistics - requesting kext w/load tag: %d\n", requestedLoadTag);
 842
 843         buffer = (char*)kheap_alloc(KHEAP_TEMP, calculatedSize,
 844             (zalloc_flags_t)(Z_WAITOK | Z_ZERO));
 845         if (!buffer) {
 846                 error = ENOMEM;
 847                 goto exit;
 848         }
 849         header = (IOStatisticsUserClientHeader*)((void*)buffer);
 850
 851         header->sig = IOSTATISTICS_SIG_USERCLIENT;
 852         header->ver = IOSTATISTICS_VER;
 853
 854         header->seq = sequenceID;
 855
 856         header->processes = 0;
 857
 858         size = copyUserClientStatistics(header, requestedLoadTag);
 859
 860         assert((sizeof(IOStatisticsUserClientHeader) + size) <= calculatedSize);
 861
 862         if (size) {
 863                 error = SYSCTL_OUT(req, buffer, sizeof(IOStatisticsUserClientHeader) + size);
 864         } else {
 865                 error = EINVAL;
 866         }
 867
 868         kheap_free(KHEAP_TEMP, buffer, calculatedSize);
 869
 870 exit:
 871         IORWLockUnlock(IOStatistics::lock);
 872         return error;
 873 }
 874
 875 uint32_t
 876 IOStatistics::copyGlobalStatistics(IOStatisticsGlobal *stats)
 877 {
 878         stats->kextCount = loadedKexts;
 879         stats->classCount = registeredClasses;
 880         stats->workloops = registeredWorkloops;
 881
 882         return sizeof(IOStatisticsGlobal);
 883 }
 884
 885 uint32_t
 886 IOStatistics::copyKextStatistics(IOStatisticsKext *stats)
 887 {
 888         KextNode *ke;
 889         ClassNode *ce;
 890         uint32_t index = 0;
 891
 892         RB_FOREACH(ke, KextTree, &kextHead) {
 893                 stats->loadTag = ke->loadTag;
 894                 ke->kext->getSizeInfo(&stats->loadSize, &stats->wiredSize);
 895
 896                 stats->classes = ke->classes;
 897
 898                 /* Append indices of owned classes */
 899                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 900                         stats->classIndexes[index++] = ce->classID;
 901                 }
 902
 903                 stats = (IOStatisticsKext *)((void*)((char*)stats + sizeof(IOStatisticsKext) + (ke->classes * sizeof(uint32_t))));
 904         }
 905
 906         return sizeof(IOStatisticsKext) * loadedKexts + sizeof(uint32_t) * registeredClasses;
 907 }
 908
 909 uint32_t
 910 IOStatistics::copyMemoryStatistics(IOStatisticsMemory *stats)
 911 {
 912         KextNode *ke;
 913
 914         RB_FOREACH(ke, KextTree, &kextHead) {
 915                 stats->allocatedSize = ke->memoryCounters[kIOStatisticsMalloc];
 916                 stats->freedSize = ke->memoryCounters[kIOStatisticsFree];
 917                 stats->allocatedAlignedSize = ke->memoryCounters[kIOStatisticsMallocAligned];
 918                 stats->freedAlignedSize = ke->memoryCounters[kIOStatisticsFreeAligned];
 919                 stats->allocatedContiguousSize = ke->memoryCounters[kIOStatisticsMallocContiguous];
 920                 stats->freedContiguousSize = ke->memoryCounters[kIOStatisticsFreeContiguous];
 921                 stats->allocatedPageableSize = ke->memoryCounters[kIOStatisticsMallocPageable];
 922                 stats->freedPageableSize = ke->memoryCounters[kIOStatisticsFreePageable];
 923                 stats++;
 924         }
 925
 926         return sizeof(IOStatisticsMemory) * loadedKexts;
 927 }
 928
 929 uint32_t
 930 IOStatistics::copyClassStatistics(IOStatisticsClass *stats)
 931 {
 932         KextNode *ke;
 933         ClassNode *ce;
 934
 935         RB_FOREACH(ke, KextTree, &kextHead) {
 936                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 937                         stats->classID = ce->classID;
 938                         stats->superClassID = ce->superClassID;
 939                         stats->classSize = ce->metaClass->getClassSize();
 940
 941                         stats++;
 942                 }
 943         }
 944
 945         return sizeof(IOStatisticsClass) * registeredClasses;
 946 }
 947
 948 uint32_t
 949 IOStatistics::copyCounterStatistics(IOStatisticsCounter *stats)
 950 {
 951         KextNode *ke;
 952         ClassNode *ce;
 953
 954         RB_FOREACH(ke, KextTree, &kextHead) {
 955                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 956                         IOUserClientCounter *userClientCounter;
 957                         IOEventSourceCounter *counter;
 958
 959                         stats->classID = ce->classID;
 960                         stats->classInstanceCount = ce->metaClass->getInstanceCount();
 961
 962                         IOStatisticsUserClients *uc = &stats->userClientStatistics;
 963
 964                         /* User client counters */
 965                         SLIST_FOREACH(userClientCounter, &ce->userClientList, link) {
 966                                 uc->clientCalls += userClientCounter->clientCalls;
 967                                 uc->created++;
 968                         }
 969
 970                         IOStatisticsInterruptEventSources *iec = &stats->interruptEventSourceStatistics;
 971                         IOStatisticsInterruptEventSources *fiec = &stats->filterInterruptEventSourceStatistics;
 972                         IOStatisticsTimerEventSources *tec = &stats->timerEventSourceStatistics;
 973                         IOStatisticsCommandGates *cgc = &stats->commandGateStatistics;
 974                         IOStatisticsCommandQueues *cqc = &stats->commandQueueStatistics;
 975                         IOStatisticsDerivedEventSources *dec = &stats->derivedEventSourceStatistics;
 976
 977                         /* Event source counters */
 978                         SLIST_FOREACH(counter, &ce->counterList, link) {
 979                                 switch (counter->type) {
 980                                 case kIOStatisticsInterruptEventSourceCounter:
 981                                         iec->created++;
 982                                         iec->produced += counter->u.interrupt.produced;
 983                                         iec->checksForWork += counter->u.interrupt.checksForWork;
 984                                         break;
 985                                 case kIOStatisticsFilterInterruptEventSourceCounter:
 986                                         fiec->created++;
 987                                         fiec->produced += counter->u.filter.produced;
 988                                         fiec->checksForWork += counter->u.filter.checksForWork;
 989                                         break;
 990                                 case kIOStatisticsTimerEventSourceCounter:
 991                                         tec->created++;
 992                                         tec->timeouts += counter->u.timer.timeouts;
 993                                         tec->checksForWork += counter->u.timer.checksForWork;
 994                                         tec->timeOnGate += counter->timeOnGate;
 995                                         tec->closeGateCalls += counter->closeGateCalls;
 996                                         tec->openGateCalls += counter->openGateCalls;
 997                                         break;
 998                                 case kIOStatisticsCommandGateCounter:
 999                                         cgc->created++;
1000                                         cgc->timeOnGate += counter->timeOnGate;
1001                                         cgc->actionCalls += counter->u.commandGate.actionCalls;
1002                                         break;
1003                                 case kIOStatisticsCommandQueueCounter:
1004                                         cqc->created++;
1005                                         cqc->actionCalls += counter->u.commandQueue.actionCalls;
1006                                         break;
1007                                 case kIOStatisticsDerivedEventSourceCounter:
1008                                         dec->created++;
1009                                         dec->timeOnGate += counter->timeOnGate;
1010                                         dec->closeGateCalls += counter->closeGateCalls;
1011                                         dec->openGateCalls += counter->openGateCalls;
1012                                         break;
1013                                 default:
1014                                         break;
1015                                 }
1016                         }
1017
1018                         stats++;
1019                 }
1020         }
1021
1022         return sizeof(IOStatisticsCounter) * registeredClasses;
1023 }
1024
1025 uint32_t
1026 IOStatistics::copyKextIdentifiers(IOStatisticsKextIdentifier *kextIDs)
1027 {
1028         KextNode *ke;
1029
1030         RB_FOREACH(ke, KextTree, &kextHead) {
1031                 strncpy(kextIDs->identifier, ke->kext->getIdentifierCString(), kIOStatisticsDriverNameLength);
1032                 kextIDs++;
1033         }
1034
1035         return sizeof(IOStatisticsKextIdentifier) * loadedKexts;
1036 }
1037
1038 uint32_t
1039 IOStatistics::copyClassNames(IOStatisticsClassName *classNames)
1040 {
1041         KextNode *ke;
1042         ClassNode *ce;
1043
1044         RB_FOREACH(ke, KextTree, &kextHead) {
1045                 SLIST_FOREACH(ce, &ke->classList, lLink) {
1046                         strncpy(classNames->name, ce->metaClass->getClassName(), kIOStatisticsClassNameLength);
1047                         classNames++;
1048                 }
1049         }
1050
1051         return sizeof(IOStatisticsClassName) * registeredClasses;
1052 }
1053
1054 uint32_t
1055 IOStatistics::copyWorkLoopStatistics(IOStatisticsWorkLoop *stats)
1056 {
1057         KextNode *ke;
1058         IOWorkLoopCounter *wlc;
1059         IOWorkLoopDependency *dependentNode;
1060         uint32_t size, accumulatedSize = 0;
1061
1062         RB_FOREACH(ke, KextTree, &kextHead) {
1063                 SLIST_FOREACH(wlc, &ke->workLoopList, link) {
1064                         stats->kextLoadTag = ke->loadTag;
1065                         stats->attachedEventSources = wlc->attachedEventSources;
1066                         stats->timeOnGate = wlc->timeOnGate;
1067                         stats->dependentKexts = 0;
1068                         RB_FOREACH(dependentNode, IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead) {
1069                                 stats->dependentKextLoadTags[stats->dependentKexts] = dependentNode->loadTag;
1070                                 stats->dependentKexts++;
1071                         }
1072
1073                         size = sizeof(IOStatisticsWorkLoop) + (sizeof(uint32_t) * stats->dependentKexts);
1074
1075                         accumulatedSize += size;
1076                         stats = (IOStatisticsWorkLoop*)((void*)((char*)stats + size));
1077                 }
1078         }
1079
1080         return accumulatedSize;
1081 }
1082
1083 uint32_t
1084 IOStatistics::copyUserClientStatistics(IOStatisticsUserClientHeader *stats, uint32_t loadTag)
1085 {
1086         KextNode *sought, *found = NULL;
1087         uint32_t procs = 0;
1088         IOUserClientProcessEntry *processEntry;
1089
1090         RB_FOREACH(sought, KextTree, &kextHead) {
1091                 if (sought->loadTag == loadTag) {
1092                         found = sought;
1093                         break;
1094                 }
1095         }
1096
1097         if (!found) {
1098                 return 0;
1099         }
1100
1101         TAILQ_FOREACH(processEntry, &found->userClientCallList, link) {
1102                 strncpy(stats->userClientCalls[procs].processName, processEntry->processName, kIOStatisticsProcessNameLength);
1103                 stats->userClientCalls[procs].pid = processEntry->pid;
1104                 stats->userClientCalls[procs].calls = processEntry->calls;
1105                 stats->processes++;
1106                 procs++;
1107         }
1108
1109         return sizeof(IOStatisticsUserClientCall) * stats->processes;
1110 }
1111
1112 void
1113 IOStatistics::storeUserClientCallInfo(IOUserClient *userClient, IOUserClientCounter *counter)
1114 {
1115         OSString *ossUserClientCreator = NULL;
1116         int32_t pid = -1;
1117         KextNode *parentKext;
1118         IOUserClientProcessEntry *entry, *nextEntry, *prevEntry = NULL;
1119         uint32_t count = 0;
1120         const char *ptr = NULL;
1121         OSObject *obj;
1122
1123         /* TODO: see if this can be more efficient */
1124         obj = userClient->copyProperty("IOUserClientCreator",
1125             gIOServicePlane,
1126             kIORegistryIterateRecursively | kIORegistryIterateParents);
1127
1128         if (!obj) {
1129                 goto err_nounlock;
1130         }
1131
1132         ossUserClientCreator = OSDynamicCast(OSString, obj);
1133
1134         if (ossUserClientCreator) {
1135                 uint32_t len, lenIter = 0;
1136
1137                 ptr = ossUserClientCreator->getCStringNoCopy();
1138                 len = ossUserClientCreator->getLength();
1139
1140                 while ((*ptr != ' ') && (lenIter < len)) {
1141                         ptr++;
1142                         lenIter++;
1143                 }
1144
1145                 if (lenIter < len) {
1146                         ptr++; // Skip the space
1147                         lenIter++;
1148                         pid = 0;
1149                         while ((*ptr != ',') && (lenIter < len)) {
1150                                 pid = pid * 10 + (*ptr - '0');
1151                                 ptr++;
1152                                 lenIter++;
1153                         }
1154
1155                         if (lenIter == len) {
1156                                 pid = -1;
1157                         } else {
1158                                 ptr += 2;
1159                         }
1160                 }
1161         }
1162
1163         if (-1 == pid) {
1164                 goto err_nounlock;
1165         }
1166
1167         IORWLockWrite(lock);
1168
1169         parentKext = counter->parentClass->parentKext;
1170
1171         TAILQ_FOREACH(entry, &parentKext->userClientCallList, link) {
1172                 if (entry->pid == pid) {
1173                         /* Found, so increment count and move to the head */
1174                         entry->calls++;
1175                         if (count) {
1176                                 TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1177                                 break;
1178                         } else {
1179                                 /* At the head already, so increment and return */
1180                                 goto err_unlock;
1181                         }
1182                 }
1183
1184                 count++;
1185         }
1186
1187         if (!entry) {
1188                 if (count == IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS) {
1189                         /* Max elements hit, so reuse the last */
1190                         entry = TAILQ_LAST(&parentKext->userClientCallList, ProcessEntryList);
1191                         TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1192                 } else {
1193                         /* Otherwise, allocate a new entry */
1194                         entry = (IOUserClientProcessEntry*)kalloc(sizeof(IOUserClientProcessEntry));
1195                         if (!entry) {
1196                                 IORWLockUnlock(lock);
1197                                 return;
1198                         }
1199                 }
1200
1201                 strncpy(entry->processName, ptr, kIOStatisticsProcessNameLength);
1202                 entry->pid = pid;
1203                 entry->calls = 1;
1204         }
1205
1206         TAILQ_FOREACH(nextEntry, &parentKext->userClientCallList, link) {
1207                 if (nextEntry->calls <= entry->calls) {
1208                         break;
1209                 }
1210
1211                 prevEntry = nextEntry;
1212         }
1213
1214         if (!prevEntry) {
1215                 TAILQ_INSERT_HEAD(&parentKext->userClientCallList, entry, link);
1216         } else {
1217                 TAILQ_INSERT_AFTER(&parentKext->userClientCallList, prevEntry, entry, link);
1218         }
1219
1220 err_unlock:
1221         IORWLockUnlock(lock);
1222
1223 err_nounlock:
1224         if (obj) {
1225                 obj->release();
1226         }
1227 }
1228
1229 void
1230 IOStatistics::countUserClientCall(IOUserClient *client)
1231 {
1232         IOUserClient::ExpansionData *data;
1233         IOUserClientCounter *counter;
1234
1235         /* Guard against an uninitialized client object - <rdar://problem/8577946> */
1236         if (!(data = client->reserved)) {
1237                 return;
1238         }
1239
1240         if ((counter = data->counter)) {
1241                 storeUserClientCallInfo(client, counter);
1242                 OSIncrementAtomic(&counter->clientCalls);
1243         }
1244 }
1245
1246 KextNode *
1247 IOStatistics::getKextNodeFromBacktrace(boolean_t write)
1248 {
1249         const uint32_t btMin = 3;
1250
1251         void *bt[16];
1252         unsigned btCount = sizeof(bt) / sizeof(bt[0]);
1253         vm_offset_t *scanAddr = NULL;
1254         uint32_t i;
1255         KextNode *found = NULL, *ke = NULL;
1256
1257         /*
1258          * Gathering the backtrace is a significant source of
1259          * overhead. OSBacktrace does many safety checks that
1260          * are not needed in this situation.
1261          */
1262         btCount = backtrace((uintptr_t*)bt, btCount, NULL);
1263
1264         if (write) {
1265                 IORWLockWrite(lock);
1266         } else {
1267                 IORWLockRead(lock);
1268         }
1269
1270         /* Ignore first levels */
1271         scanAddr = (vm_offset_t *)&bt[btMin - 1];
1272
1273         for (i = btMin - 1; i < btCount; i++, scanAddr++) {
1274                 ke = RB_ROOT(&kextAddressHead);
1275                 while (ke) {
1276                         if (*scanAddr < ke->address) {
1277                                 ke = RB_LEFT(ke, addressLink);
1278                         } else {
1279                                 if ((*scanAddr < ke->address_end) && (*scanAddr >= ke->address)) {
1280                                         if (!ke->kext->isKernelComponent()) {
1281                                                 return ke;
1282                                         } else {
1283                                                 found = ke;
1284                                         }
1285                                 }
1286                                 ke = RB_RIGHT(ke, addressLink);
1287                         }
1288                 }
1289         }
1290
1291         if (!found) {
1292                 IORWLockUnlock(lock);
1293         }
1294
1295         return found;
1296 }
1297
1298 void
1299 IOStatistics::releaseKextNode(KextNode *node)
1300 {
1301 #pragma unused(node)
1302         IORWLockUnlock(lock);
1303 }
1304
1305 /* IOLib allocations */
1306 void
1307 IOStatistics::countAlloc(uint32_t index, vm_size_t size)
1308 {
1309         KextNode *ke;
1310
1311         if (!enabled) {
1312                 return;
1313         }
1314         if (size > INT_MAX) {
1315                 return;
1316         }
1317
1318         ke = getKextNodeFromBacktrace(FALSE);
1319         if (ke) {
1320                 OSAddAtomic((SInt32) size, &ke->memoryCounters[index]);
1321                 releaseKextNode(ke);
1322         }
1323 }
1324
1325 #endif /* IOKITSTATS */