iokit/Kernel/IOStatistics.cpp

   1 /*
   2  * Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/sysctl.h>
  30 #include <kern/backtrace.h>
  31 #include <kern/host.h>
  32 #include <kern/zalloc.h>
  33
  34 #include <IOKit/system.h>
  35 #include <libkern/c++/OSKext.h>
  36 #include <libkern/OSAtomic.h>
  37
  38 #include <IOKit/IOStatisticsPrivate.h>
  39 #include <IOKit/IOUserClient.h>
  40 #include <IOKit/IOEventSource.h>
  41 #include <IOKit/IOKitDebug.h>
  42
  43 #if IOKITSTATS
  44 bool IOStatistics::enabled = false;
  45
  46 uint32_t IOStatistics::sequenceID = 0;
  47
  48 uint32_t IOStatistics::lastClassIndex = 0;
  49 uint32_t IOStatistics::lastKextIndex = 0;
  50
  51 uint32_t IOStatistics::loadedKexts = 0;
  52 uint32_t IOStatistics::registeredClasses = 0;
  53 uint32_t IOStatistics::registeredCounters = 0;
  54 uint32_t IOStatistics::registeredWorkloops = 0;
  55
  56 uint32_t IOStatistics::attachedEventSources = 0;
  57
  58 IOWorkLoopDependency *IOStatistics::nextWorkLoopDependency = NULL;
  59
  60 /* Logging */
  61
  62 #define LOG_LEVEL 0
  63
  64 #define LOG(level, format, ...) \
  65 do { \
  66         if (level <= LOG_LEVEL) \
  67                 printf(format, ##__VA_ARGS__); \
  68 } while (0)
  69
  70 /* Locks */
  71
  72 IORWLock *IOStatistics::lock = NULL;
  73
  74 /* Kext tree */
  75
  76 KextNode *IOStatistics::kextHint = NULL;
  77
  78 IOStatistics::KextTreeHead IOStatistics::kextHead = RB_INITIALIZER(&IOStatistics::kextHead);
  79
  80 int
  81 IOStatistics::kextNodeCompare(KextNode *e1, KextNode *e2)
  82 {
  83         if (e1->kext < e2->kext) {
  84                 return -1;
  85         } else if (e1->kext > e2->kext) {
  86                 return 1;
  87         } else {
  88                 return 0;
  89         }
  90 }
  91
  92 RB_GENERATE(IOStatistics::KextTree, KextNode, link, kextNodeCompare);
  93
  94 /* Kext tree ordered by address */
  95
  96 IOStatistics::KextAddressTreeHead IOStatistics::kextAddressHead = RB_INITIALIZER(&IOStatistics::kextAddressHead);
  97
  98 int
  99 IOStatistics::kextAddressNodeCompare(KextNode *e1, KextNode *e2)
 100 {
 101         if (e1->address < e2->address) {
 102                 return -1;
 103         } else if (e1->address > e2->address) {
 104                 return 1;
 105         } else {
 106                 return 0;
 107         }
 108 }
 109
 110 RB_GENERATE(IOStatistics::KextAddressTree, KextNode, addressLink, kextAddressNodeCompare);
 111
 112 /* Class tree */
 113
 114 IOStatistics::ClassTreeHead IOStatistics::classHead = RB_INITIALIZER(&IOStatistics::classHead);
 115
 116 int
 117 IOStatistics::classNodeCompare(ClassNode *e1, ClassNode *e2)
 118 {
 119         if (e1->metaClass < e2->metaClass) {
 120                 return -1;
 121         } else if (e1->metaClass > e2->metaClass) {
 122                 return 1;
 123         } else {
 124                 return 0;
 125         }
 126 }
 127
 128 RB_GENERATE(IOStatistics::ClassTree, ClassNode, tLink, classNodeCompare);
 129
 130 /* Workloop dependencies */
 131
 132 int
 133 IOWorkLoopCounter::loadTagCompare(IOWorkLoopDependency *e1, IOWorkLoopDependency *e2)
 134 {
 135         if (e1->loadTag < e2->loadTag) {
 136                 return -1;
 137         } else if (e1->loadTag > e2->loadTag) {
 138                 return 1;
 139         } else {
 140                 return 0;
 141         }
 142 }
 143
 144 RB_GENERATE(IOWorkLoopCounter::DependencyTree, IOWorkLoopDependency, link, IOWorkLoopCounter::loadTagCompare);
 145
 146 /* sysctl stuff */
 147
 148 static int
 149 oid_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1, int arg2, struct sysctl_req *req)
 150 {
 151         int error = EINVAL;
 152         uint32_t request = arg2;
 153
 154         switch (request) {
 155         case kIOStatisticsGeneral:
 156                 error = IOStatistics::getStatistics(req);
 157                 break;
 158         case kIOStatisticsWorkLoop:
 159                 error = IOStatistics::getWorkLoopStatistics(req);
 160                 break;
 161         case kIOStatisticsUserClient:
 162                 error = IOStatistics::getUserClientStatistics(req);
 163                 break;
 164         default:
 165                 break;
 166         }
 167
 168         return error;
 169 }
 170
 171 SYSCTL_NODE(_debug, OID_AUTO, iokit_statistics, CTLFLAG_RW | CTLFLAG_LOCKED, NULL, "IOStatistics");
 172
 173 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, general,
 174     CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 175     NULL, kIOStatisticsGeneral, oid_sysctl, "S", "");
 176
 177 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, workloop,
 178     CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 179     NULL, kIOStatisticsWorkLoop, oid_sysctl, "S", "");
 180
 181 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, userclient,
 182     CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 183     NULL, kIOStatisticsUserClient, oid_sysctl, "S", "");
 184
 185 void
 186 IOStatistics::initialize()
 187 {
 188         if (enabled) {
 189                 return;
 190         }
 191
 192         /* Only enabled if the boot argument is set. */
 193         if (!(kIOStatistics & gIOKitDebug)) {
 194                 return;
 195         }
 196
 197         sysctl_register_oid(&sysctl__debug_iokit_statistics_general);
 198         sysctl_register_oid(&sysctl__debug_iokit_statistics_workloop);
 199         sysctl_register_oid(&sysctl__debug_iokit_statistics_userclient);
 200
 201         lock = IORWLockAlloc();
 202         if (!lock) {
 203                 return;
 204         }
 205
 206         nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
 207         if (!nextWorkLoopDependency) {
 208                 return;
 209         }
 210
 211         enabled = true;
 212 }
 213
 214 void
 215 IOStatistics::onKextLoad(OSKext *kext, kmod_info_t *kmod_info)
 216 {
 217         KextNode *ke;
 218
 219         assert(kext && kmod_info);
 220
 221         if (!enabled) {
 222                 return;
 223         }
 224
 225         LOG(1, "IOStatistics::onKextLoad: %s, tag %d, address 0x%llx, address end 0x%llx\n",
 226             kext->getIdentifierCString(), kmod_info->id, (uint64_t)kmod_info->address, (uint64_t)(kmod_info->address + kmod_info->size));
 227
 228         ke = (KextNode *)kalloc(sizeof(KextNode));
 229         if (!ke) {
 230                 return;
 231         }
 232
 233         memset(ke, 0, sizeof(KextNode));
 234
 235         ke->kext = kext;
 236         ke->loadTag = kmod_info->id;
 237         ke->address = kmod_info->address;
 238         ke->address_end = kmod_info->address + kmod_info->size;
 239
 240         SLIST_INIT(&ke->classList);
 241         TAILQ_INIT(&ke->userClientCallList);
 242
 243         IORWLockWrite(lock);
 244
 245         RB_INSERT(KextTree, &kextHead, ke);
 246         RB_INSERT(KextAddressTree, &kextAddressHead, ke);
 247
 248         sequenceID++;
 249         loadedKexts++;
 250         lastKextIndex++;
 251
 252         IORWLockUnlock(lock);
 253 }
 254
 255 void
 256 IOStatistics::onKextUnload(OSKext *kext)
 257 {
 258         KextNode sought, *found;
 259
 260         assert(kext);
 261
 262         if (!enabled) {
 263                 return;
 264         }
 265
 266         LOG(1, "IOStatistics::onKextUnload: %s\n", kext->getIdentifierCString());
 267
 268         IORWLockWrite(lock);
 269
 270         sought.kext = kext;
 271         found = RB_FIND(KextTree, &kextHead, &sought);
 272         if (found) {
 273                 IOWorkLoopCounter *wlc;
 274                 IOUserClientProcessEntry *uce;
 275
 276                 /* Disconnect workloop counters; cleanup takes place in unregisterWorkLoop() */
 277                 while ((wlc = SLIST_FIRST(&found->workLoopList))) {
 278                         SLIST_REMOVE_HEAD(&found->workLoopList, link);
 279                         wlc->parentKext = NULL;
 280                 }
 281
 282                 /* Free up the user client list */
 283                 while ((uce = TAILQ_FIRST(&found->userClientCallList))) {
 284                         TAILQ_REMOVE(&found->userClientCallList, uce, link);
 285                         kfree(uce, sizeof(IOUserClientProcessEntry));
 286                 }
 287
 288                 /* Remove from kext trees */
 289                 RB_REMOVE(KextTree, &kextHead, found);
 290                 RB_REMOVE(KextAddressTree, &kextAddressHead, found);
 291
 292                 /*
 293                  * Clear a matching kextHint to avoid use after free in
 294                  * onClassAdded() for a class add after a KEXT unload.
 295                  */
 296                 if (found == kextHint) {
 297                         kextHint = NULL;
 298                 }
 299
 300                 /* Finally, free the class node */
 301                 kfree(found, sizeof(KextNode));
 302
 303                 sequenceID++;
 304                 loadedKexts--;
 305         } else {
 306                 panic("IOStatistics::onKextUnload: cannot find kext: %s", kext->getIdentifierCString());
 307         }
 308
 309         IORWLockUnlock(lock);
 310 }
 311
 312 void
 313 IOStatistics::onClassAdded(OSKext *parentKext, OSMetaClass *metaClass)
 314 {
 315         ClassNode *ce;
 316         KextNode soughtKext, *foundKext = NULL;
 317
 318         assert(parentKext && metaClass);
 319
 320         if (!enabled) {
 321                 return;
 322         }
 323
 324         LOG(1, "IOStatistics::onClassAdded: %s\n", metaClass->getClassName());
 325
 326         ce = (ClassNode *)kalloc(sizeof(ClassNode));
 327         if (!ce) {
 328                 return;
 329         }
 330
 331         memset(ce, 0, sizeof(ClassNode));
 332
 333         IORWLockWrite(lock);
 334
 335         /* Hinted? */
 336         if (kextHint && kextHint->kext == parentKext) {
 337                 foundKext = kextHint;
 338         } else {
 339                 soughtKext.kext = parentKext;
 340                 foundKext = RB_FIND(KextTree, &kextHead, &soughtKext);
 341         }
 342
 343         if (foundKext) {
 344                 ClassNode soughtClass, *foundClass = NULL;
 345                 const OSMetaClass *superClass;
 346
 347                 ce->metaClass = metaClass;
 348                 ce->classID = lastClassIndex++;
 349                 ce->parentKext = foundKext;
 350
 351                 /* Has superclass? */
 352                 superClass = ce->metaClass->getSuperClass();
 353                 if (superClass) {
 354                         soughtClass.metaClass = superClass;
 355                         foundClass = RB_FIND(ClassTree, &classHead, &soughtClass);
 356                 }
 357                 ce->superClassID = foundClass ? foundClass->classID : (uint32_t)(-1);
 358
 359                 SLIST_INIT(&ce->counterList);
 360                 SLIST_INIT(&ce->userClientList);
 361
 362                 RB_INSERT(ClassTree, &classHead, ce);
 363                 SLIST_INSERT_HEAD(&foundKext->classList, ce, lLink);
 364
 365                 foundKext->classes++;
 366
 367                 kextHint = foundKext;
 368
 369                 sequenceID++;
 370                 registeredClasses++;
 371         } else {
 372                 panic("IOStatistics::onClassAdded: cannot find parent kext: %s", parentKext->getIdentifierCString());
 373         }
 374
 375         IORWLockUnlock(lock);
 376 }
 377
 378 void
 379 IOStatistics::onClassRemoved(OSKext *parentKext, OSMetaClass *metaClass)
 380 {
 381         ClassNode sought, *found;
 382
 383         assert(parentKext && metaClass);
 384
 385         if (!enabled) {
 386                 return;
 387         }
 388
 389         LOG(1, "IOStatistics::onClassRemoved: %s\n", metaClass->getClassName());
 390
 391         IORWLockWrite(lock);
 392
 393         sought.metaClass = metaClass;
 394         found = RB_FIND(ClassTree, &classHead, &sought);
 395         if (found) {
 396                 IOEventSourceCounter *esc;
 397                 IOUserClientCounter *ucc;
 398
 399                 /* Free up the list of counters */
 400                 while ((esc = SLIST_FIRST(&found->counterList))) {
 401                         SLIST_REMOVE_HEAD(&found->counterList, link);
 402                         kfree(esc, sizeof(IOEventSourceCounter));
 403                 }
 404
 405                 /* Free up the user client list */
 406                 while ((ucc = SLIST_FIRST(&found->userClientList))) {
 407                         SLIST_REMOVE_HEAD(&found->userClientList, link);
 408                         kfree(ucc, sizeof(IOUserClientCounter));
 409                 }
 410
 411                 /* Remove from class tree */
 412                 RB_REMOVE(ClassTree, &classHead, found);
 413
 414                 /* Remove from parent */
 415                 SLIST_REMOVE(&found->parentKext->classList, found, ClassNode, lLink);
 416
 417                 /* Finally, free the class node */
 418                 kfree(found, sizeof(ClassNode));
 419
 420                 sequenceID++;
 421                 registeredClasses--;
 422         } else {
 423                 panic("IOStatistics::onClassRemoved: cannot find class: %s", metaClass->getClassName());
 424         }
 425
 426         IORWLockUnlock(lock);
 427 }
 428
 429 IOEventSourceCounter *
 430 IOStatistics::registerEventSource(OSObject *inOwner)
 431 {
 432         IOEventSourceCounter *counter = NULL;
 433         ClassNode sought, *found = NULL;
 434         boolean_t createDummyCounter = FALSE;
 435
 436         assert(inOwner);
 437
 438         if (!enabled) {
 439                 return NULL;
 440         }
 441
 442         counter = (IOEventSourceCounter*)kalloc(sizeof(IOEventSourceCounter));
 443         if (!counter) {
 444                 return NULL;
 445         }
 446
 447         memset(counter, 0, sizeof(IOEventSourceCounter));
 448
 449         IORWLockWrite(lock);
 450
 451         /* Workaround for <rdar://problem/7158117> - create a dummy counter when inOwner is bad.
 452          * We use retainCount here as our best indication that the pointer is awry.
 453          */
 454         if (inOwner->retainCount > 0xFFFFFF) {
 455                 kprintf("IOStatistics::registerEventSource - bad metaclass %p\n", inOwner);
 456                 createDummyCounter = TRUE;
 457         } else {
 458                 sought.metaClass = inOwner->getMetaClass();
 459                 found = RB_FIND(ClassTree, &classHead, &sought);
 460         }
 461
 462         if (found) {
 463                 counter->parentClass = found;
 464                 SLIST_INSERT_HEAD(&found->counterList, counter, link);
 465                 registeredCounters++;
 466         }
 467
 468         if (!(createDummyCounter || found)) {
 469                 panic("IOStatistics::registerEventSource: cannot find parent class: %s", inOwner->getMetaClass()->getClassName());
 470         }
 471
 472         IORWLockUnlock(lock);
 473
 474         return counter;
 475 }
 476
 477 void
 478 IOStatistics::unregisterEventSource(IOEventSourceCounter *counter)
 479 {
 480         if (!counter) {
 481                 return;
 482         }
 483
 484         IORWLockWrite(lock);
 485
 486         if (counter->parentClass) {
 487                 SLIST_REMOVE(&counter->parentClass->counterList, counter, IOEventSourceCounter, link);
 488                 registeredCounters--;
 489         }
 490         kfree(counter, sizeof(IOEventSourceCounter));
 491
 492         IORWLockUnlock(lock);
 493 }
 494
 495 IOWorkLoopCounter*
 496 IOStatistics::registerWorkLoop(IOWorkLoop *workLoop)
 497 {
 498         IOWorkLoopCounter *counter = NULL;
 499         KextNode *found;
 500
 501         assert(workLoop);
 502
 503         if (!enabled) {
 504                 return NULL;
 505         }
 506
 507         counter = (IOWorkLoopCounter*)kalloc(sizeof(IOWorkLoopCounter));
 508         if (!counter) {
 509                 return NULL;
 510         }
 511
 512         memset(counter, 0, sizeof(IOWorkLoopCounter));
 513
 514         found = getKextNodeFromBacktrace(TRUE);
 515         if (!found) {
 516                 panic("IOStatistics::registerWorkLoop: cannot find parent kext");
 517         }
 518
 519         counter->parentKext = found;
 520         counter->workLoop = workLoop;
 521         RB_INIT(&counter->dependencyHead);
 522         SLIST_INSERT_HEAD(&found->workLoopList, counter, link);
 523         registeredWorkloops++;
 524
 525         releaseKextNode(found);
 526
 527         return counter;
 528 }
 529
 530 void
 531 IOStatistics::unregisterWorkLoop(IOWorkLoopCounter *counter)
 532 {
 533         if (!counter) {
 534                 return;
 535         }
 536
 537         IORWLockWrite(lock);
 538         if (counter->parentKext) {
 539                 SLIST_REMOVE(&counter->parentKext->workLoopList, counter, IOWorkLoopCounter, link);
 540         }
 541         kfree(counter, sizeof(IOWorkLoopCounter));
 542         registeredWorkloops--;
 543
 544         IORWLockUnlock(lock);
 545 }
 546
 547 IOUserClientCounter *
 548 IOStatistics::registerUserClient(IOUserClient *userClient)
 549 {
 550         ClassNode sought, *found;
 551         IOUserClientCounter *counter = NULL;
 552
 553         assert(userClient);
 554
 555         if (!enabled) {
 556                 return NULL;
 557         }
 558
 559         counter = (IOUserClientCounter*)kalloc(sizeof(IOUserClientCounter));
 560         if (!counter) {
 561                 return NULL;
 562         }
 563
 564         memset(counter, 0, sizeof(IOUserClientCounter));
 565
 566         IORWLockWrite(lock);
 567
 568         sought.metaClass = userClient->getMetaClass();
 569
 570         found = RB_FIND(ClassTree, &classHead, &sought);
 571         if (found) {
 572                 counter->parentClass = found;
 573                 SLIST_INSERT_HEAD(&found->userClientList, counter, link);
 574         } else {
 575                 panic("IOStatistics::registerUserClient: cannot find parent class: %s", sought.metaClass->getClassName());
 576         }
 577
 578         IORWLockUnlock(lock);
 579
 580         return counter;
 581 }
 582
 583 void
 584 IOStatistics::unregisterUserClient(IOUserClientCounter *counter)
 585 {
 586         if (!counter) {
 587                 return;
 588         }
 589
 590         IORWLockWrite(lock);
 591
 592         SLIST_REMOVE(&counter->parentClass->userClientList, counter, IOUserClientCounter, link);
 593         kfree(counter, sizeof(IOUserClientCounter));
 594
 595         IORWLockUnlock(lock);
 596 }
 597
 598 void
 599 IOStatistics::attachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
 600 {
 601         if (!wlc) {
 602                 return;
 603         }
 604
 605         IORWLockWrite(lock);
 606
 607         if (!nextWorkLoopDependency) {
 608                 return;
 609         }
 610
 611         attachedEventSources++;
 612         wlc->attachedEventSources++;
 613
 614         /* Track the kext dependency */
 615         nextWorkLoopDependency->loadTag = esc->parentClass->parentKext->loadTag;
 616         if (NULL == RB_INSERT(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, nextWorkLoopDependency)) {
 617                 nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
 618         }
 619
 620         IORWLockUnlock(lock);
 621 }
 622
 623 void
 624 IOStatistics::detachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
 625 {
 626         IOWorkLoopDependency sought, *found;
 627
 628         if (!wlc) {
 629                 return;
 630         }
 631
 632         IORWLockWrite(lock);
 633
 634         attachedEventSources--;
 635         wlc->attachedEventSources--;
 636
 637         sought.loadTag = esc->parentClass->parentKext->loadTag;
 638
 639         found = RB_FIND(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, &sought);
 640         if (found) {
 641                 RB_REMOVE(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, found);
 642                 kfree(found, sizeof(IOWorkLoopDependency));
 643         }
 644
 645         IORWLockUnlock(lock);
 646 }
 647
 648 int
 649 IOStatistics::getStatistics(sysctl_req *req)
 650 {
 651         int error;
 652         uint32_t calculatedSize, size;
 653         char *buffer, *ptr;
 654         IOStatisticsHeader *header;
 655
 656         assert(IOStatistics::enabled && req);
 657
 658         IORWLockRead(IOStatistics::lock);
 659
 660         /* Work out how much we need to allocate. IOStatisticsKext is of variable size. */
 661         calculatedSize = sizeof(IOStatisticsHeader) +
 662             sizeof(IOStatisticsGlobal) +
 663             (sizeof(IOStatisticsKext) * loadedKexts) + (sizeof(uint32_t) * registeredClasses) +
 664             (sizeof(IOStatisticsMemory) * loadedKexts) +
 665             (sizeof(IOStatisticsClass) * registeredClasses) +
 666             (sizeof(IOStatisticsCounter) * registeredClasses) +
 667             (sizeof(IOStatisticsKextIdentifier) * loadedKexts) +
 668             (sizeof(IOStatisticsClassName) * registeredClasses);
 669
 670         /* Size request? */
 671         if (req->oldptr == USER_ADDR_NULL) {
 672                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 673                 goto exit;
 674         }
 675
 676         /* Read only */
 677         if (req->newptr != USER_ADDR_NULL) {
 678                 error = EPERM;
 679                 goto exit;
 680         }
 681
 682         buffer = (char*)kalloc(calculatedSize);
 683         if (!buffer) {
 684                 error = ENOMEM;
 685                 goto exit;
 686         }
 687
 688         memset(buffer, 0, calculatedSize);
 689
 690         ptr = buffer;
 691
 692         header = (IOStatisticsHeader*)((void*)ptr);
 693
 694         header->sig = IOSTATISTICS_SIG;
 695         header->ver = IOSTATISTICS_VER;
 696
 697         header->seq = sequenceID;
 698
 699         ptr += sizeof(IOStatisticsHeader);
 700
 701         /* Global data - seq, timers, interrupts, etc) */
 702         header->globalStatsOffset = sizeof(IOStatisticsHeader);
 703         size = copyGlobalStatistics((IOStatisticsGlobal*)((void*)ptr));
 704         ptr += size;
 705
 706         /* Kext statistics */
 707         header->kextStatsOffset = header->globalStatsOffset + size;
 708         size = copyKextStatistics((IOStatisticsKext*)((void*)ptr));
 709         ptr += size;
 710
 711         /* Memory allocation info */
 712         header->memoryStatsOffset = header->kextStatsOffset + size;
 713         size = copyMemoryStatistics((IOStatisticsMemory*)((void*)ptr));
 714         ptr += size;
 715
 716         /* Class statistics */
 717         header->classStatsOffset = header->memoryStatsOffset + size;
 718         size = copyClassStatistics((IOStatisticsClass*)((void*)ptr));
 719         ptr += size;
 720
 721         /* Dynamic class counter data */
 722         header->counterStatsOffset = header->classStatsOffset + size;
 723         size = copyCounterStatistics((IOStatisticsCounter*)((void*)ptr));
 724         ptr += size;
 725
 726         /* Kext identifiers */
 727         header->kextIdentifiersOffset = header->counterStatsOffset + size;
 728         size = copyKextIdentifiers((IOStatisticsKextIdentifier*)((void*)ptr));
 729         ptr += size;
 730
 731         /* Class names */
 732         header->classNamesOffset = header->kextIdentifiersOffset + size;
 733         size = copyClassNames((IOStatisticsClassName*)ptr);
 734         ptr += size;
 735
 736         LOG(2, "IOStatistics::getStatistics - calculatedSize 0x%x, kexts 0x%x, classes 0x%x.\n",
 737             calculatedSize, loadedKexts, registeredClasses);
 738
 739         assert((uint32_t)(ptr - buffer) == calculatedSize );
 740
 741         error = SYSCTL_OUT(req, buffer, calculatedSize);
 742
 743         kfree(buffer, calculatedSize);
 744
 745 exit:
 746         IORWLockUnlock(IOStatistics::lock);
 747         return error;
 748 }
 749
 750 int
 751 IOStatistics::getWorkLoopStatistics(sysctl_req *req)
 752 {
 753         int error;
 754         uint32_t calculatedSize, size;
 755         char *buffer;
 756         IOStatisticsWorkLoopHeader *header;
 757
 758         assert(IOStatistics::enabled && req);
 759
 760         IORWLockRead(IOStatistics::lock);
 761
 762         /* Approximate how much we need to allocate (worse case estimate) */
 763         calculatedSize = sizeof(IOStatisticsWorkLoop) * registeredWorkloops +
 764             sizeof(uint32_t) * attachedEventSources;
 765
 766         /* Size request? */
 767         if (req->oldptr == USER_ADDR_NULL) {
 768                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 769                 goto exit;
 770         }
 771
 772         /* Read only */
 773         if (req->newptr != USER_ADDR_NULL) {
 774                 error = EPERM;
 775                 goto exit;
 776         }
 777
 778         buffer = (char*)kalloc(calculatedSize);
 779         if (!buffer) {
 780                 error = ENOMEM;
 781                 goto exit;
 782         }
 783         memset(buffer, 0, calculatedSize);
 784         header = (IOStatisticsWorkLoopHeader*)((void*)buffer);
 785
 786         header->sig = IOSTATISTICS_SIG_WORKLOOP;
 787         header->ver = IOSTATISTICS_VER;
 788
 789         header->seq = sequenceID;
 790
 791         header->workloopCount = registeredWorkloops;
 792
 793         size = copyWorkLoopStatistics(&header->workLoopStats);
 794
 795         LOG(2, "IOStatistics::getWorkLoopStatistics: calculatedSize %d, size %d\n", calculatedSize, size);
 796
 797         assert( size <= calculatedSize );
 798
 799         error = SYSCTL_OUT(req, buffer, size);
 800
 801         kfree(buffer, calculatedSize);
 802
 803 exit:
 804         IORWLockUnlock(IOStatistics::lock);
 805         return error;
 806 }
 807
 808 int
 809 IOStatistics::getUserClientStatistics(sysctl_req *req)
 810 {
 811         int error;
 812         uint32_t calculatedSize, size;
 813         char *buffer;
 814         uint32_t requestedLoadTag = 0;
 815         IOStatisticsUserClientHeader *header;
 816
 817         assert(IOStatistics::enabled && req);
 818
 819         IORWLockRead(IOStatistics::lock);
 820
 821         /* Work out how much we need to allocate */
 822         calculatedSize = sizeof(IOStatisticsUserClientHeader) +
 823             sizeof(IOStatisticsUserClientCall) * IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS * loadedKexts;
 824
 825         /* Size request? */
 826         if (req->oldptr == USER_ADDR_NULL) {
 827                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 828                 goto exit;
 829         }
 830
 831         /* Kext request (potentially) valid? */
 832         if (!req->newptr || req->newlen < sizeof(requestedLoadTag)) {
 833                 error = EINVAL;
 834                 goto exit;
 835         }
 836
 837         error = SYSCTL_IN(req, &requestedLoadTag, sizeof(requestedLoadTag));
 838         if (error) {
 839                 goto exit;
 840         }
 841
 842         LOG(2, "IOStatistics::getUserClientStatistics - requesting kext w/load tag: %d\n", requestedLoadTag);
 843
 844         buffer = (char*)kalloc(calculatedSize);
 845         if (!buffer) {
 846                 error = ENOMEM;
 847                 goto exit;
 848         }
 849         memset(buffer, 0, calculatedSize);
 850         header = (IOStatisticsUserClientHeader*)((void*)buffer);
 851
 852         header->sig = IOSTATISTICS_SIG_USERCLIENT;
 853         header->ver = IOSTATISTICS_VER;
 854
 855         header->seq = sequenceID;
 856
 857         header->processes = 0;
 858
 859         size = copyUserClientStatistics(header, requestedLoadTag);
 860
 861         assert((sizeof(IOStatisticsUserClientHeader) + size) <= calculatedSize);
 862
 863         if (size) {
 864                 error = SYSCTL_OUT(req, buffer, sizeof(IOStatisticsUserClientHeader) + size);
 865         } else {
 866                 error = EINVAL;
 867         }
 868
 869         kfree(buffer, calculatedSize);
 870
 871 exit:
 872         IORWLockUnlock(IOStatistics::lock);
 873         return error;
 874 }
 875
 876 uint32_t
 877 IOStatistics::copyGlobalStatistics(IOStatisticsGlobal *stats)
 878 {
 879         stats->kextCount = loadedKexts;
 880         stats->classCount = registeredClasses;
 881         stats->workloops = registeredWorkloops;
 882
 883         return sizeof(IOStatisticsGlobal);
 884 }
 885
 886 uint32_t
 887 IOStatistics::copyKextStatistics(IOStatisticsKext *stats)
 888 {
 889         KextNode *ke;
 890         ClassNode *ce;
 891         uint32_t index = 0;
 892
 893         RB_FOREACH(ke, KextTree, &kextHead) {
 894                 stats->loadTag = ke->loadTag;
 895                 ke->kext->getSizeInfo(&stats->loadSize, &stats->wiredSize);
 896
 897                 stats->classes = ke->classes;
 898
 899                 /* Append indices of owned classes */
 900                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 901                         stats->classIndexes[index++] = ce->classID;
 902                 }
 903
 904                 stats = (IOStatisticsKext *)((void*)((char*)stats + sizeof(IOStatisticsKext) + (ke->classes * sizeof(uint32_t))));
 905         }
 906
 907         return sizeof(IOStatisticsKext) * loadedKexts + sizeof(uint32_t) * registeredClasses;
 908 }
 909
 910 uint32_t
 911 IOStatistics::copyMemoryStatistics(IOStatisticsMemory *stats)
 912 {
 913         KextNode *ke;
 914
 915         RB_FOREACH(ke, KextTree, &kextHead) {
 916                 stats->allocatedSize = ke->memoryCounters[kIOStatisticsMalloc];
 917                 stats->freedSize = ke->memoryCounters[kIOStatisticsFree];
 918                 stats->allocatedAlignedSize = ke->memoryCounters[kIOStatisticsMallocAligned];
 919                 stats->freedAlignedSize = ke->memoryCounters[kIOStatisticsFreeAligned];
 920                 stats->allocatedContiguousSize = ke->memoryCounters[kIOStatisticsMallocContiguous];
 921                 stats->freedContiguousSize = ke->memoryCounters[kIOStatisticsFreeContiguous];
 922                 stats->allocatedPageableSize = ke->memoryCounters[kIOStatisticsMallocPageable];
 923                 stats->freedPageableSize = ke->memoryCounters[kIOStatisticsFreePageable];
 924                 stats++;
 925         }
 926
 927         return sizeof(IOStatisticsMemory) * loadedKexts;
 928 }
 929
 930 uint32_t
 931 IOStatistics::copyClassStatistics(IOStatisticsClass *stats)
 932 {
 933         KextNode *ke;
 934         ClassNode *ce;
 935
 936         RB_FOREACH(ke, KextTree, &kextHead) {
 937                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 938                         stats->classID = ce->classID;
 939                         stats->superClassID = ce->superClassID;
 940                         stats->classSize = ce->metaClass->getClassSize();
 941
 942                         stats++;
 943                 }
 944         }
 945
 946         return sizeof(IOStatisticsClass) * registeredClasses;
 947 }
 948
 949 uint32_t
 950 IOStatistics::copyCounterStatistics(IOStatisticsCounter *stats)
 951 {
 952         KextNode *ke;
 953         ClassNode *ce;
 954
 955         RB_FOREACH(ke, KextTree, &kextHead) {
 956                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 957                         IOUserClientCounter *userClientCounter;
 958                         IOEventSourceCounter *counter;
 959
 960                         stats->classID = ce->classID;
 961                         stats->classInstanceCount = ce->metaClass->getInstanceCount();
 962
 963                         IOStatisticsUserClients *uc = &stats->userClientStatistics;
 964
 965                         /* User client counters */
 966                         SLIST_FOREACH(userClientCounter, &ce->userClientList, link) {
 967                                 uc->clientCalls += userClientCounter->clientCalls;
 968                                 uc->created++;
 969                         }
 970
 971                         IOStatisticsInterruptEventSources *iec = &stats->interruptEventSourceStatistics;
 972                         IOStatisticsInterruptEventSources *fiec = &stats->filterInterruptEventSourceStatistics;
 973                         IOStatisticsTimerEventSources *tec = &stats->timerEventSourceStatistics;
 974                         IOStatisticsCommandGates *cgc = &stats->commandGateStatistics;
 975                         IOStatisticsCommandQueues *cqc = &stats->commandQueueStatistics;
 976                         IOStatisticsDerivedEventSources *dec = &stats->derivedEventSourceStatistics;
 977
 978                         /* Event source counters */
 979                         SLIST_FOREACH(counter, &ce->counterList, link) {
 980                                 switch (counter->type) {
 981                                 case kIOStatisticsInterruptEventSourceCounter:
 982                                         iec->created++;
 983                                         iec->produced += counter->u.interrupt.produced;
 984                                         iec->checksForWork += counter->u.interrupt.checksForWork;
 985                                         break;
 986                                 case kIOStatisticsFilterInterruptEventSourceCounter:
 987                                         fiec->created++;
 988                                         fiec->produced += counter->u.filter.produced;
 989                                         fiec->checksForWork += counter->u.filter.checksForWork;
 990                                         break;
 991                                 case kIOStatisticsTimerEventSourceCounter:
 992                                         tec->created++;
 993                                         tec->timeouts += counter->u.timer.timeouts;
 994                                         tec->checksForWork += counter->u.timer.checksForWork;
 995                                         tec->timeOnGate += counter->timeOnGate;
 996                                         tec->closeGateCalls += counter->closeGateCalls;
 997                                         tec->openGateCalls += counter->openGateCalls;
 998                                         break;
 999                                 case kIOStatisticsCommandGateCounter:
1000                                         cgc->created++;
1001                                         cgc->timeOnGate += counter->timeOnGate;
1002                                         cgc->actionCalls += counter->u.commandGate.actionCalls;
1003                                         break;
1004                                 case kIOStatisticsCommandQueueCounter:
1005                                         cqc->created++;
1006                                         cqc->actionCalls += counter->u.commandQueue.actionCalls;
1007                                         break;
1008                                 case kIOStatisticsDerivedEventSourceCounter:
1009                                         dec->created++;
1010                                         dec->timeOnGate += counter->timeOnGate;
1011                                         dec->closeGateCalls += counter->closeGateCalls;
1012                                         dec->openGateCalls += counter->openGateCalls;
1013                                         break;
1014                                 default:
1015                                         break;
1016                                 }
1017                         }
1018
1019                         stats++;
1020                 }
1021         }
1022
1023         return sizeof(IOStatisticsCounter) * registeredClasses;
1024 }
1025
1026 uint32_t
1027 IOStatistics::copyKextIdentifiers(IOStatisticsKextIdentifier *kextIDs)
1028 {
1029         KextNode *ke;
1030
1031         RB_FOREACH(ke, KextTree, &kextHead) {
1032                 strncpy(kextIDs->identifier, ke->kext->getIdentifierCString(), kIOStatisticsDriverNameLength);
1033                 kextIDs++;
1034         }
1035
1036         return sizeof(IOStatisticsKextIdentifier) * loadedKexts;
1037 }
1038
1039 uint32_t
1040 IOStatistics::copyClassNames(IOStatisticsClassName *classNames)
1041 {
1042         KextNode *ke;
1043         ClassNode *ce;
1044
1045         RB_FOREACH(ke, KextTree, &kextHead) {
1046                 SLIST_FOREACH(ce, &ke->classList, lLink) {
1047                         strncpy(classNames->name, ce->metaClass->getClassName(), kIOStatisticsClassNameLength);
1048                         classNames++;
1049                 }
1050         }
1051
1052         return sizeof(IOStatisticsClassName) * registeredClasses;
1053 }
1054
1055 uint32_t
1056 IOStatistics::copyWorkLoopStatistics(IOStatisticsWorkLoop *stats)
1057 {
1058         KextNode *ke;
1059         IOWorkLoopCounter *wlc;
1060         IOWorkLoopDependency *dependentNode;
1061         uint32_t size, accumulatedSize = 0;
1062
1063         RB_FOREACH(ke, KextTree, &kextHead) {
1064                 SLIST_FOREACH(wlc, &ke->workLoopList, link) {
1065                         stats->kextLoadTag = ke->loadTag;
1066                         stats->attachedEventSources = wlc->attachedEventSources;
1067                         stats->timeOnGate = wlc->timeOnGate;
1068                         stats->dependentKexts = 0;
1069                         RB_FOREACH(dependentNode, IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead) {
1070                                 stats->dependentKextLoadTags[stats->dependentKexts] = dependentNode->loadTag;
1071                                 stats->dependentKexts++;
1072                         }
1073
1074                         size = sizeof(IOStatisticsWorkLoop) + (sizeof(uint32_t) * stats->dependentKexts);
1075
1076                         accumulatedSize += size;
1077                         stats = (IOStatisticsWorkLoop*)((void*)((char*)stats + size));
1078                 }
1079         }
1080
1081         return accumulatedSize;
1082 }
1083
1084 uint32_t
1085 IOStatistics::copyUserClientStatistics(IOStatisticsUserClientHeader *stats, uint32_t loadTag)
1086 {
1087         KextNode *sought, *found = NULL;
1088         uint32_t procs = 0;
1089         IOUserClientProcessEntry *processEntry;
1090
1091         RB_FOREACH(sought, KextTree, &kextHead) {
1092                 if (sought->loadTag == loadTag) {
1093                         found = sought;
1094                         break;
1095                 }
1096         }
1097
1098         if (!found) {
1099                 return 0;
1100         }
1101
1102         TAILQ_FOREACH(processEntry, &found->userClientCallList, link) {
1103                 strncpy(stats->userClientCalls[procs].processName, processEntry->processName, kIOStatisticsProcessNameLength);
1104                 stats->userClientCalls[procs].pid = processEntry->pid;
1105                 stats->userClientCalls[procs].calls = processEntry->calls;
1106                 stats->processes++;
1107                 procs++;
1108         }
1109
1110         return sizeof(IOStatisticsUserClientCall) * stats->processes;
1111 }
1112
1113 void
1114 IOStatistics::storeUserClientCallInfo(IOUserClient *userClient, IOUserClientCounter *counter)
1115 {
1116         OSString *ossUserClientCreator = NULL;
1117         int32_t pid = -1;
1118         KextNode *parentKext;
1119         IOUserClientProcessEntry *entry, *nextEntry, *prevEntry = NULL;
1120         uint32_t count = 0;
1121         const char *ptr = NULL;
1122         OSObject *obj;
1123
1124         /* TODO: see if this can be more efficient */
1125         obj = userClient->copyProperty("IOUserClientCreator",
1126             gIOServicePlane,
1127             kIORegistryIterateRecursively | kIORegistryIterateParents);
1128
1129         if (!obj) {
1130                 goto err_nounlock;
1131         }
1132
1133         ossUserClientCreator = OSDynamicCast(OSString, obj);
1134
1135         if (ossUserClientCreator) {
1136                 uint32_t len, lenIter = 0;
1137
1138                 ptr = ossUserClientCreator->getCStringNoCopy();
1139                 len = ossUserClientCreator->getLength();
1140
1141                 while ((*ptr != ' ') && (lenIter < len)) {
1142                         ptr++;
1143                         lenIter++;
1144                 }
1145
1146                 if (lenIter < len) {
1147                         ptr++; // Skip the space
1148                         lenIter++;
1149                         pid = 0;
1150                         while ((*ptr != ',') && (lenIter < len)) {
1151                                 pid = pid * 10 + (*ptr - '0');
1152                                 ptr++;
1153                                 lenIter++;
1154                         }
1155
1156                         if (lenIter == len) {
1157                                 pid = -1;
1158                         } else {
1159                                 ptr += 2;
1160                         }
1161                 }
1162         }
1163
1164         if (-1 == pid) {
1165                 goto err_nounlock;
1166         }
1167
1168         IORWLockWrite(lock);
1169
1170         parentKext = counter->parentClass->parentKext;
1171
1172         TAILQ_FOREACH(entry, &parentKext->userClientCallList, link) {
1173                 if (entry->pid == pid) {
1174                         /* Found, so increment count and move to the head */
1175                         entry->calls++;
1176                         if (count) {
1177                                 TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1178                                 break;
1179                         } else {
1180                                 /* At the head already, so increment and return */
1181                                 goto err_unlock;
1182                         }
1183                 }
1184
1185                 count++;
1186         }
1187
1188         if (!entry) {
1189                 if (count == IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS) {
1190                         /* Max elements hit, so reuse the last */
1191                         entry = TAILQ_LAST(&parentKext->userClientCallList, ProcessEntryList);
1192                         TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1193                 } else {
1194                         /* Otherwise, allocate a new entry */
1195                         entry = (IOUserClientProcessEntry*)kalloc(sizeof(IOUserClientProcessEntry));
1196                         if (!entry) {
1197                                 IORWLockUnlock(lock);
1198                                 return;
1199                         }
1200                 }
1201
1202                 strncpy(entry->processName, ptr, kIOStatisticsProcessNameLength);
1203                 entry->pid = pid;
1204                 entry->calls = 1;
1205         }
1206
1207         TAILQ_FOREACH(nextEntry, &parentKext->userClientCallList, link) {
1208                 if (nextEntry->calls <= entry->calls) {
1209                         break;
1210                 }
1211
1212                 prevEntry = nextEntry;
1213         }
1214
1215         if (!prevEntry) {
1216                 TAILQ_INSERT_HEAD(&parentKext->userClientCallList, entry, link);
1217         } else {
1218                 TAILQ_INSERT_AFTER(&parentKext->userClientCallList, prevEntry, entry, link);
1219         }
1220
1221 err_unlock:
1222         IORWLockUnlock(lock);
1223
1224 err_nounlock:
1225         if (obj) {
1226                 obj->release();
1227         }
1228 }
1229
1230 void
1231 IOStatistics::countUserClientCall(IOUserClient *client)
1232 {
1233         IOUserClient::ExpansionData *data;
1234         IOUserClientCounter *counter;
1235
1236         /* Guard against an uninitialized client object - <rdar://problem/8577946> */
1237         if (!(data = client->reserved)) {
1238                 return;
1239         }
1240
1241         if ((counter = data->counter)) {
1242                 storeUserClientCallInfo(client, counter);
1243                 OSIncrementAtomic(&counter->clientCalls);
1244         }
1245 }
1246
1247 KextNode *
1248 IOStatistics::getKextNodeFromBacktrace(boolean_t write)
1249 {
1250         const uint32_t btMin = 3;
1251
1252         void *bt[16];
1253         unsigned btCount = sizeof(bt) / sizeof(bt[0]);
1254         vm_offset_t *scanAddr = NULL;
1255         uint32_t i;
1256         KextNode *found = NULL, *ke = NULL;
1257
1258         /*
1259          * Gathering the backtrace is a significant source of
1260          * overhead. OSBacktrace does many safety checks that
1261          * are not needed in this situation.
1262          */
1263         btCount = backtrace((uintptr_t*)bt, btCount, NULL);
1264
1265         if (write) {
1266                 IORWLockWrite(lock);
1267         } else {
1268                 IORWLockRead(lock);
1269         }
1270
1271         /* Ignore first levels */
1272         scanAddr = (vm_offset_t *)&bt[btMin - 1];
1273
1274         for (i = btMin - 1; i < btCount; i++, scanAddr++) {
1275                 ke = RB_ROOT(&kextAddressHead);
1276                 while (ke) {
1277                         if (*scanAddr < ke->address) {
1278                                 ke = RB_LEFT(ke, addressLink);
1279                         } else {
1280                                 if ((*scanAddr < ke->address_end) && (*scanAddr >= ke->address)) {
1281                                         if (!ke->kext->isKernelComponent()) {
1282                                                 return ke;
1283                                         } else {
1284                                                 found = ke;
1285                                         }
1286                                 }
1287                                 ke = RB_RIGHT(ke, addressLink);
1288                         }
1289                 }
1290         }
1291
1292         if (!found) {
1293                 IORWLockUnlock(lock);
1294         }
1295
1296         return found;
1297 }
1298
1299 void
1300 IOStatistics::releaseKextNode(KextNode *node)
1301 {
1302 #pragma unused(node)
1303         IORWLockUnlock(lock);
1304 }
1305
1306 /* IOLib allocations */
1307 void
1308 IOStatistics::countAlloc(uint32_t index, vm_size_t size)
1309 {
1310         KextNode *ke;
1311
1312         if (!enabled) {
1313                 return;
1314         }
1315
1316         ke = getKextNodeFromBacktrace(FALSE);
1317         if (ke) {
1318                 OSAddAtomic(size, &ke->memoryCounters[index]);
1319                 releaseKextNode(ke);
1320         }
1321 }
1322
1323 #endif /* IOKITSTATS */