iokit/Kernel/IOStatistics.cpp

   1 /*
   2  * Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/sysctl.h>
  30 #include <kern/backtrace.h>
  31 #include <kern/host.h>
  32 #include <kern/zalloc.h>
  33
  34 #include <IOKit/system.h>
  35 #include <libkern/c++/OSKext.h>
  36 #include <libkern/OSAtomic.h>
  37
  38 #include <IOKit/IOStatisticsPrivate.h>
  39 #include <IOKit/IOUserClient.h>
  40 #include <IOKit/IOEventSource.h>
  41 #include <IOKit/IOKitDebug.h>
  42
  43 #if IOKITSTATS
  44 bool IOStatistics::enabled = false;
  45
  46 uint32_t IOStatistics::sequenceID = 0;
  47
  48 uint32_t IOStatistics::lastClassIndex = 0;
  49 uint32_t IOStatistics::lastKextIndex = 0;
  50
  51 uint32_t IOStatistics::loadedKexts = 0;
  52 uint32_t IOStatistics::registeredClasses = 0;
  53 uint32_t IOStatistics::registeredCounters = 0;
  54 uint32_t IOStatistics::registeredWorkloops = 0;
  55
  56 uint32_t IOStatistics::attachedEventSources = 0;
  57
  58 IOWorkLoopDependency *IOStatistics::nextWorkLoopDependency = NULL;
  59
  60 /* Logging */
  61
  62 #define LOG_LEVEL 0
  63
  64 #define LOG(level, format, ...) \
  65 do { \
  66         if (level <= LOG_LEVEL) \
  67                 printf(format, ##__VA_ARGS__); \
  68 } while (0)
  69
  70 /* Locks */
  71
  72 IORWLock *IOStatistics::lock = NULL;
  73
  74 /* Kext tree */
  75
  76 KextNode *IOStatistics::kextHint = NULL;
  77
  78 IOStatistics::KextTreeHead IOStatistics::kextHead = RB_INITIALIZER(&IOStatistics::kextHead);
  79
  80 int IOStatistics::kextNodeCompare(KextNode *e1, KextNode *e2)
  81 {
  82     if (e1->kext < e2->kext)
  83         return -1;
  84     else if (e1->kext > e2->kext)
  85         return 1;
  86     else
  87         return 0;
  88 }
  89
  90 RB_GENERATE(IOStatistics::KextTree, KextNode, link, kextNodeCompare);
  91
  92 /* Kext tree ordered by address */
  93
  94 IOStatistics::KextAddressTreeHead IOStatistics::kextAddressHead = RB_INITIALIZER(&IOStatistics::kextAddressHead);
  95
  96 int IOStatistics::kextAddressNodeCompare(KextNode *e1, KextNode *e2)
  97 {
  98     if (e1->address < e2->address)
  99         return -1;
 100     else if (e1->address > e2->address)
 101         return 1;
 102     else
 103         return 0;
 104 }
 105
 106 RB_GENERATE(IOStatistics::KextAddressTree, KextNode, addressLink, kextAddressNodeCompare);
 107
 108 /* Class tree */
 109
 110 IOStatistics::ClassTreeHead IOStatistics::classHead = RB_INITIALIZER(&IOStatistics::classHead);
 111
 112 int IOStatistics::classNodeCompare(ClassNode *e1, ClassNode *e2) {
 113     if (e1->metaClass < e2->metaClass)
 114         return -1;
 115     else if (e1->metaClass > e2->metaClass)
 116         return 1;
 117     else
 118         return 0;
 119 }
 120
 121 RB_GENERATE(IOStatistics::ClassTree, ClassNode, tLink, classNodeCompare);
 122
 123 /* Workloop dependencies */
 124
 125 int IOWorkLoopCounter::loadTagCompare(IOWorkLoopDependency *e1, IOWorkLoopDependency *e2) {
 126     if (e1->loadTag < e2->loadTag)
 127         return -1;
 128     else if (e1->loadTag > e2->loadTag)
 129         return 1;
 130     else
 131         return 0;
 132 }
 133
 134 RB_GENERATE(IOWorkLoopCounter::DependencyTree, IOWorkLoopDependency, link, IOWorkLoopCounter::loadTagCompare);
 135
 136 /* sysctl stuff */
 137
 138 static int
 139 oid_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1, int arg2, struct sysctl_req *req)
 140 {
 141         int error = EINVAL;
 142         uint32_t request = arg2;
 143
 144         switch (request)
 145         {
 146                 case kIOStatisticsGeneral:
 147                         error = IOStatistics::getStatistics(req);
 148                         break;
 149                 case kIOStatisticsWorkLoop:
 150                         error = IOStatistics::getWorkLoopStatistics(req);
 151                         break;
 152                 case kIOStatisticsUserClient:
 153                         error = IOStatistics::getUserClientStatistics(req);
 154                         break;
 155                 default:
 156                         break;
 157         }
 158
 159         return error;
 160 }
 161
 162 SYSCTL_NODE(_debug, OID_AUTO, iokit_statistics, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "IOStatistics");
 163
 164 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, general,
 165             CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 166             0, kIOStatisticsGeneral, oid_sysctl, "S", "");
 167
 168 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, workloop,
 169             CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 170             0, kIOStatisticsWorkLoop, oid_sysctl, "S", "");
 171
 172 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, userclient,
 173             CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 174             0, kIOStatisticsUserClient, oid_sysctl, "S", "");
 175
 176 void IOStatistics::initialize()
 177 {
 178         if (enabled) {
 179                 return;
 180         }
 181
 182         /* Only enabled if the boot argument is set. */
 183         if (!(kIOStatistics & gIOKitDebug)) {
 184                 return;
 185         }
 186
 187         sysctl_register_oid(&sysctl__debug_iokit_statistics_general);
 188         sysctl_register_oid(&sysctl__debug_iokit_statistics_workloop);
 189         sysctl_register_oid(&sysctl__debug_iokit_statistics_userclient);
 190
 191         lock = IORWLockAlloc();
 192         if (!lock) {
 193                 return;
 194         }
 195
 196         nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
 197         if (!nextWorkLoopDependency) {
 198                 return;
 199         }
 200
 201         enabled = true;
 202 }
 203
 204 void IOStatistics::onKextLoad(OSKext *kext, kmod_info_t *kmod_info)
 205 {
 206         KextNode *ke;
 207
 208         assert(kext && kmod_info);
 209
 210         if (!enabled) {
 211                 return;
 212         }
 213
 214         LOG(1, "IOStatistics::onKextLoad: %s, tag %d, address 0x%llx, address end 0x%llx\n",
 215                 kext->getIdentifierCString(), kmod_info->id, (uint64_t)kmod_info->address, (uint64_t)(kmod_info->address + kmod_info->size));
 216
 217         ke = (KextNode *)kalloc(sizeof(KextNode));
 218         if (!ke) {
 219                 return;
 220         }
 221
 222         memset(ke, 0, sizeof(KextNode));
 223
 224         ke->kext = kext;
 225         ke->loadTag = kmod_info->id;
 226         ke->address = kmod_info->address;
 227         ke->address_end = kmod_info->address + kmod_info->size;
 228
 229         SLIST_INIT(&ke->classList);
 230         TAILQ_INIT(&ke->userClientCallList);
 231
 232         IORWLockWrite(lock);
 233
 234         RB_INSERT(KextTree, &kextHead, ke);
 235         RB_INSERT(KextAddressTree, &kextAddressHead, ke);
 236
 237         sequenceID++;
 238         loadedKexts++;
 239         lastKextIndex++;
 240
 241         IORWLockUnlock(lock);
 242 }
 243
 244 void IOStatistics::onKextUnload(OSKext *kext)
 245 {
 246         KextNode sought, *found;
 247
 248         assert(kext);
 249
 250         if (!enabled) {
 251                 return;
 252         }
 253
 254         LOG(1, "IOStatistics::onKextUnload: %s\n", kext->getIdentifierCString());
 255
 256         IORWLockWrite(lock);
 257
 258         sought.kext = kext;
 259         found = RB_FIND(KextTree, &kextHead, &sought);
 260         if (found) {
 261                 IOWorkLoopCounter *wlc;
 262                 IOUserClientProcessEntry *uce;
 263
 264                 /* Disconnect workloop counters; cleanup takes place in unregisterWorkLoop() */
 265                 while ((wlc = SLIST_FIRST(&found->workLoopList))) {
 266                         SLIST_REMOVE_HEAD(&found->workLoopList, link);
 267                         wlc->parentKext = NULL;
 268                 }
 269
 270                 /* Free up the user client list */
 271                 while ((uce = TAILQ_FIRST(&found->userClientCallList))) {
 272                         TAILQ_REMOVE(&found->userClientCallList, uce, link);
 273                         kfree(uce, sizeof(IOUserClientProcessEntry));
 274                 }
 275
 276                 /* Remove from kext trees */
 277                 RB_REMOVE(KextTree, &kextHead, found);
 278                 RB_REMOVE(KextAddressTree, &kextAddressHead, found);
 279
 280                 /*
 281                  * Clear a matching kextHint to avoid use after free in
 282                  * onClassAdded() for a class add after a KEXT unload.
 283                  */
 284                 if (found == kextHint) {
 285                         kextHint = NULL;
 286                 }
 287
 288                 /* Finally, free the class node */
 289                 kfree(found, sizeof(KextNode));
 290
 291                 sequenceID++;
 292                 loadedKexts--;
 293         }
 294         else {
 295                 panic("IOStatistics::onKextUnload: cannot find kext: %s", kext->getIdentifierCString());
 296         }
 297
 298         IORWLockUnlock(lock);
 299 }
 300
 301 void IOStatistics::onClassAdded(OSKext *parentKext, OSMetaClass *metaClass)
 302 {
 303         ClassNode *ce;
 304         KextNode soughtKext, *foundKext = NULL;
 305
 306         assert(parentKext && metaClass);
 307
 308         if (!enabled) {
 309                 return;
 310         }
 311
 312         LOG(1, "IOStatistics::onClassAdded: %s\n", metaClass->getClassName());
 313
 314         ce = (ClassNode *)kalloc(sizeof(ClassNode));
 315         if (!ce) {
 316                 return;
 317         }
 318
 319         memset(ce, 0, sizeof(ClassNode));
 320
 321         IORWLockWrite(lock);
 322
 323         /* Hinted? */
 324         if (kextHint && kextHint->kext == parentKext) {
 325                 foundKext = kextHint;
 326         }
 327         else {
 328                 soughtKext.kext = parentKext;
 329                 foundKext = RB_FIND(KextTree, &kextHead, &soughtKext);
 330         }
 331
 332         if (foundKext) {
 333                 ClassNode soughtClass, *foundClass = NULL;
 334                 const OSMetaClass *superClass;
 335
 336                 ce->metaClass = metaClass;
 337                 ce->classID = lastClassIndex++;
 338                 ce->parentKext = foundKext;
 339
 340                 /* Has superclass? */
 341                 superClass = ce->metaClass->getSuperClass();
 342                 if (superClass) {
 343                         soughtClass.metaClass = superClass;
 344                         foundClass = RB_FIND(ClassTree, &classHead, &soughtClass);
 345                 }
 346                 ce->superClassID = foundClass ? foundClass->classID : (uint32_t)(-1);
 347
 348                 SLIST_INIT(&ce->counterList);
 349                 SLIST_INIT(&ce->userClientList);
 350
 351                 RB_INSERT(ClassTree, &classHead, ce);
 352                 SLIST_INSERT_HEAD(&foundKext->classList, ce, lLink);
 353
 354                 foundKext->classes++;
 355
 356                 kextHint = foundKext;
 357
 358                 sequenceID++;
 359                 registeredClasses++;
 360         }
 361         else {
 362                 panic("IOStatistics::onClassAdded: cannot find parent kext: %s", parentKext->getIdentifierCString());
 363         }
 364
 365         IORWLockUnlock(lock);
 366 }
 367
 368 void IOStatistics::onClassRemoved(OSKext *parentKext, OSMetaClass *metaClass)
 369 {
 370         ClassNode sought, *found;
 371
 372         assert(parentKext && metaClass);
 373
 374         if (!enabled) {
 375                 return;
 376         }
 377
 378         LOG(1, "IOStatistics::onClassRemoved: %s\n", metaClass->getClassName());
 379
 380         IORWLockWrite(lock);
 381
 382         sought.metaClass = metaClass;
 383         found = RB_FIND(ClassTree, &classHead, &sought);
 384         if (found) {
 385                 IOEventSourceCounter *esc;
 386                 IOUserClientCounter *ucc;
 387
 388                 /* Free up the list of counters */
 389                 while ((esc = SLIST_FIRST(&found->counterList))) {
 390                         SLIST_REMOVE_HEAD(&found->counterList, link);
 391                         kfree(esc, sizeof(IOEventSourceCounter));
 392                 }
 393
 394                 /* Free up the user client list */
 395                 while ((ucc = SLIST_FIRST(&found->userClientList))) {
 396                         SLIST_REMOVE_HEAD(&found->userClientList, link);
 397                         kfree(ucc, sizeof(IOUserClientCounter));
 398                 }
 399
 400                 /* Remove from class tree */
 401                 RB_REMOVE(ClassTree, &classHead, found);
 402
 403                 /* Remove from parent */
 404                 SLIST_REMOVE(&found->parentKext->classList, found, ClassNode, lLink);
 405
 406                 /* Finally, free the class node */
 407                 kfree(found, sizeof(ClassNode));
 408
 409                 sequenceID++;
 410                 registeredClasses--;
 411         }
 412         else {
 413                 panic("IOStatistics::onClassRemoved: cannot find class: %s", metaClass->getClassName());
 414         }
 415
 416         IORWLockUnlock(lock);
 417 }
 418
 419 IOEventSourceCounter *IOStatistics::registerEventSource(OSObject *inOwner)
 420 {
 421         IOEventSourceCounter *counter = NULL;
 422         ClassNode sought, *found = NULL;
 423         boolean_t createDummyCounter = FALSE;
 424
 425         assert(inOwner);
 426
 427         if (!enabled) {
 428                 return NULL;
 429         }
 430
 431         counter = (IOEventSourceCounter*)kalloc(sizeof(IOEventSourceCounter));
 432         if (!counter) {
 433                 return NULL;
 434         }
 435
 436         memset(counter, 0, sizeof(IOEventSourceCounter));
 437
 438         IORWLockWrite(lock);
 439
 440         /* Workaround for <rdar://problem/7158117> - create a dummy counter when inOwner is bad.
 441          * We use retainCount here as our best indication that the pointer is awry.
 442          */
 443         if (inOwner->retainCount > 0xFFFFFF) {
 444                 kprintf("IOStatistics::registerEventSource - bad metaclass %p\n", inOwner);
 445                 createDummyCounter = TRUE;
 446         }
 447         else {
 448                 sought.metaClass = inOwner->getMetaClass();
 449                 found = RB_FIND(ClassTree, &classHead, &sought);
 450         }
 451
 452         if (found) {
 453                 counter->parentClass = found;
 454                 SLIST_INSERT_HEAD(&found->counterList, counter, link);
 455                 registeredCounters++;
 456         }
 457
 458         if (!(createDummyCounter || found)) {
 459                 panic("IOStatistics::registerEventSource: cannot find parent class: %s", inOwner->getMetaClass()->getClassName());
 460         }
 461
 462         IORWLockUnlock(lock);
 463
 464         return counter;
 465 }
 466
 467 void IOStatistics::unregisterEventSource(IOEventSourceCounter *counter)
 468 {
 469         if (!counter) {
 470                 return;
 471         }
 472
 473         IORWLockWrite(lock);
 474
 475         if (counter->parentClass) {
 476                 SLIST_REMOVE(&counter->parentClass->counterList, counter, IOEventSourceCounter, link);
 477                 registeredCounters--;
 478         }
 479         kfree(counter, sizeof(IOEventSourceCounter));
 480
 481         IORWLockUnlock(lock);
 482 }
 483
 484 IOWorkLoopCounter* IOStatistics::registerWorkLoop(IOWorkLoop *workLoop)
 485 {
 486         IOWorkLoopCounter *counter = NULL;
 487         KextNode *found;
 488
 489         assert(workLoop);
 490
 491         if (!enabled) {
 492                 return NULL;
 493         }
 494
 495         counter = (IOWorkLoopCounter*)kalloc(sizeof(IOWorkLoopCounter));
 496         if (!counter) {
 497                 return NULL;
 498         }
 499
 500         memset(counter, 0, sizeof(IOWorkLoopCounter));
 501
 502         found = getKextNodeFromBacktrace(TRUE);
 503         if (!found) {
 504                 panic("IOStatistics::registerWorkLoop: cannot find parent kext");
 505         }
 506
 507         counter->parentKext = found;
 508         counter->workLoop = workLoop;
 509         RB_INIT(&counter->dependencyHead);
 510         SLIST_INSERT_HEAD(&found->workLoopList, counter, link);
 511         registeredWorkloops++;
 512
 513         releaseKextNode(found);
 514
 515         return counter;
 516 }
 517
 518 void IOStatistics::unregisterWorkLoop(IOWorkLoopCounter *counter)
 519 {
 520         if (!counter) {
 521                 return;
 522         }
 523
 524         IORWLockWrite(lock);
 525         if (counter->parentKext) {
 526                 SLIST_REMOVE(&counter->parentKext->workLoopList, counter, IOWorkLoopCounter, link);
 527         }
 528         kfree(counter, sizeof(IOWorkLoopCounter));
 529         registeredWorkloops--;
 530
 531         IORWLockUnlock(lock);
 532 }
 533
 534 IOUserClientCounter *IOStatistics::registerUserClient(IOUserClient *userClient)
 535 {
 536         ClassNode sought, *found;
 537         IOUserClientCounter *counter = NULL;
 538
 539         assert(userClient);
 540
 541         if (!enabled) {
 542                 return NULL;
 543         }
 544
 545         counter = (IOUserClientCounter*)kalloc(sizeof(IOUserClientCounter));
 546         if (!counter) {
 547                 return NULL;
 548         }
 549
 550         memset(counter, 0, sizeof(IOUserClientCounter));
 551
 552         IORWLockWrite(lock);
 553
 554         sought.metaClass = userClient->getMetaClass();
 555
 556         found = RB_FIND(ClassTree, &classHead, &sought);
 557         if (found) {
 558                 counter->parentClass = found;
 559                 SLIST_INSERT_HEAD(&found->userClientList, counter, link);
 560         }
 561         else {
 562                 panic("IOStatistics::registerUserClient: cannot find parent class: %s", sought.metaClass->getClassName());
 563         }
 564
 565         IORWLockUnlock(lock);
 566
 567         return counter;
 568 }
 569
 570 void IOStatistics::unregisterUserClient(IOUserClientCounter *counter)
 571 {
 572         if (!counter) {
 573                 return;
 574         }
 575
 576         IORWLockWrite(lock);
 577
 578         SLIST_REMOVE(&counter->parentClass->userClientList, counter, IOUserClientCounter, link);
 579         kfree(counter, sizeof(IOUserClientCounter));
 580
 581         IORWLockUnlock(lock);
 582 }
 583
 584 void IOStatistics::attachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
 585 {
 586         if (!wlc) {
 587         return;
 588         }
 589
 590         IORWLockWrite(lock);
 591
 592         if (!nextWorkLoopDependency) {
 593                 return;
 594         }
 595
 596         attachedEventSources++;
 597         wlc->attachedEventSources++;
 598
 599         /* Track the kext dependency */
 600         nextWorkLoopDependency->loadTag = esc->parentClass->parentKext->loadTag;
 601         if (NULL == RB_INSERT(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, nextWorkLoopDependency)) {
 602                 nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
 603         }
 604
 605         IORWLockUnlock(lock);
 606 }
 607
 608 void IOStatistics::detachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
 609 {
 610         IOWorkLoopDependency sought, *found;
 611
 612         if (!wlc) {
 613                 return;
 614         }
 615
 616         IORWLockWrite(lock);
 617
 618         attachedEventSources--;
 619         wlc->attachedEventSources--;
 620
 621         sought.loadTag = esc->parentClass->parentKext->loadTag;
 622
 623         found = RB_FIND(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, &sought);
 624         if (found) {
 625                 RB_REMOVE(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, found);
 626                 kfree(found, sizeof(IOWorkLoopDependency));
 627         }
 628
 629         IORWLockUnlock(lock);
 630 }
 631
 632 int IOStatistics::getStatistics(sysctl_req *req)
 633 {
 634         int error;
 635         uint32_t calculatedSize, size;
 636         char *buffer, *ptr;
 637         IOStatisticsHeader *header;
 638
 639         assert(IOStatistics::enabled && req);
 640
 641         IORWLockRead(IOStatistics::lock);
 642
 643         /* Work out how much we need to allocate. IOStatisticsKext is of variable size. */
 644         calculatedSize = sizeof(IOStatisticsHeader) +
 645                                          sizeof(IOStatisticsGlobal) +
 646                                         (sizeof(IOStatisticsKext) * loadedKexts) + (sizeof(uint32_t) * registeredClasses) +
 647                                         (sizeof(IOStatisticsMemory) * loadedKexts) +
 648                                         (sizeof(IOStatisticsClass) * registeredClasses) +
 649                                         (sizeof(IOStatisticsCounter) * registeredClasses) +
 650                                         (sizeof(IOStatisticsKextIdentifier) * loadedKexts) +
 651                                         (sizeof(IOStatisticsClassName) * registeredClasses);
 652
 653         /* Size request? */
 654         if (req->oldptr == USER_ADDR_NULL) {
 655                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 656                 goto exit;
 657         }
 658
 659         /* Read only */
 660         if (req->newptr != USER_ADDR_NULL) {
 661                 error = EPERM;
 662                 goto exit;
 663         }
 664
 665         buffer = (char*)kalloc(calculatedSize);
 666         if (!buffer) {
 667                 error = ENOMEM;
 668                 goto exit;
 669         }
 670
 671         memset(buffer, 0, calculatedSize);
 672
 673         ptr = buffer;
 674
 675         header = (IOStatisticsHeader*)((void*)ptr);
 676
 677         header->sig = IOSTATISTICS_SIG;
 678         header->ver = IOSTATISTICS_VER;
 679
 680         header->seq = sequenceID;
 681
 682         ptr += sizeof(IOStatisticsHeader);
 683
 684         /* Global data - seq, timers, interrupts, etc) */
 685         header->globalStatsOffset = sizeof(IOStatisticsHeader);
 686         size = copyGlobalStatistics((IOStatisticsGlobal*)((void*)ptr));
 687         ptr += size;
 688
 689         /* Kext statistics */
 690         header->kextStatsOffset = header->globalStatsOffset + size;
 691         size = copyKextStatistics((IOStatisticsKext*)((void*)ptr));
 692         ptr += size;
 693
 694         /* Memory allocation info */
 695         header->memoryStatsOffset = header->kextStatsOffset + size;
 696         size = copyMemoryStatistics((IOStatisticsMemory*)((void*)ptr));
 697         ptr += size;
 698
 699         /* Class statistics */
 700         header->classStatsOffset = header->memoryStatsOffset + size;
 701         size = copyClassStatistics((IOStatisticsClass*)((void*)ptr));
 702         ptr += size;
 703
 704         /* Dynamic class counter data */
 705         header->counterStatsOffset = header->classStatsOffset + size;
 706         size = copyCounterStatistics((IOStatisticsCounter*)((void*)ptr));
 707         ptr += size;
 708
 709         /* Kext identifiers */
 710         header->kextIdentifiersOffset = header->counterStatsOffset + size;
 711         size = copyKextIdentifiers((IOStatisticsKextIdentifier*)((void*)ptr));
 712         ptr += size;
 713
 714         /* Class names */
 715         header->classNamesOffset = header->kextIdentifiersOffset + size;
 716         size = copyClassNames((IOStatisticsClassName*)ptr);
 717         ptr += size;
 718
 719         LOG(2, "IOStatistics::getStatistics - calculatedSize 0x%x, kexts 0x%x, classes 0x%x.\n",
 720                 calculatedSize, loadedKexts, registeredClasses);
 721
 722         assert( (uint32_t)(ptr - buffer) == calculatedSize );
 723
 724         error = SYSCTL_OUT(req, buffer, calculatedSize);
 725
 726         kfree(buffer, calculatedSize);
 727
 728 exit:
 729         IORWLockUnlock(IOStatistics::lock);
 730         return error;
 731 }
 732
 733 int IOStatistics::getWorkLoopStatistics(sysctl_req *req)
 734 {
 735         int error;
 736         uint32_t calculatedSize, size;
 737         char *buffer;
 738         IOStatisticsWorkLoopHeader *header;
 739
 740         assert(IOStatistics::enabled && req);
 741
 742         IORWLockRead(IOStatistics::lock);
 743
 744         /* Approximate how much we need to allocate (worse case estimate) */
 745         calculatedSize = sizeof(IOStatisticsWorkLoop) * registeredWorkloops +
 746                                          sizeof(uint32_t) * attachedEventSources;
 747
 748         /* Size request? */
 749         if (req->oldptr == USER_ADDR_NULL) {
 750                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 751                 goto exit;
 752         }
 753
 754         /* Read only */
 755         if (req->newptr != USER_ADDR_NULL) {
 756                 error = EPERM;
 757                 goto exit;
 758         }
 759
 760         buffer = (char*)kalloc(calculatedSize);
 761         if (!buffer) {
 762                 error = ENOMEM;
 763                 goto exit;
 764         }
 765
 766         header = (IOStatisticsWorkLoopHeader*)((void*)buffer);
 767
 768         header->sig = IOSTATISTICS_SIG_WORKLOOP;
 769         header->ver = IOSTATISTICS_VER;
 770
 771         header->seq = sequenceID;
 772
 773         header->workloopCount = registeredWorkloops;
 774
 775         size = copyWorkLoopStatistics(&header->workLoopStats);
 776
 777         LOG(2, "IOStatistics::getWorkLoopStatistics: calculatedSize %d, size %d\n", calculatedSize, size);
 778
 779         assert( size <= calculatedSize );
 780
 781         error = SYSCTL_OUT(req, buffer, size);
 782
 783         kfree(buffer, calculatedSize);
 784
 785 exit:
 786         IORWLockUnlock(IOStatistics::lock);
 787         return error;
 788 }
 789
 790 int IOStatistics::getUserClientStatistics(sysctl_req *req)
 791 {
 792         int error;
 793         uint32_t calculatedSize, size;
 794         char *buffer;
 795         uint32_t requestedLoadTag = 0;
 796         IOStatisticsUserClientHeader *header;
 797
 798         assert(IOStatistics::enabled && req);
 799
 800         IORWLockRead(IOStatistics::lock);
 801
 802         /* Work out how much we need to allocate */
 803         calculatedSize = sizeof(IOStatisticsUserClientHeader) +
 804                                          sizeof(IOStatisticsUserClientCall) * IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS * loadedKexts;
 805
 806         /* Size request? */
 807         if (req->oldptr == USER_ADDR_NULL) {
 808                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 809                 goto exit;
 810         }
 811
 812         /* Kext request (potentially) valid? */
 813         if (!req->newptr || req->newlen < sizeof(requestedLoadTag)) {
 814                 error = EINVAL;
 815                 goto exit;
 816         }
 817
 818         SYSCTL_IN(req, &requestedLoadTag, sizeof(requestedLoadTag));
 819
 820         LOG(2, "IOStatistics::getUserClientStatistics - requesting kext w/load tag: %d\n", requestedLoadTag);
 821
 822         buffer = (char*)kalloc(calculatedSize);
 823         if (!buffer) {
 824                 error = ENOMEM;
 825                 goto exit;
 826         }
 827
 828         header = (IOStatisticsUserClientHeader*)((void*)buffer);
 829
 830         header->sig = IOSTATISTICS_SIG_USERCLIENT;
 831         header->ver = IOSTATISTICS_VER;
 832
 833         header->seq = sequenceID;
 834
 835         header->processes = 0;
 836
 837         size = copyUserClientStatistics(header, requestedLoadTag);
 838
 839         assert((sizeof(IOStatisticsUserClientHeader) + size) <= calculatedSize);
 840
 841         if (size) {
 842                 error = SYSCTL_OUT(req, buffer, sizeof(IOStatisticsUserClientHeader) + size);
 843         }
 844         else {
 845                 error = EINVAL;
 846         }
 847
 848         kfree(buffer, calculatedSize);
 849
 850 exit:
 851         IORWLockUnlock(IOStatistics::lock);
 852         return error;
 853 }
 854
 855 uint32_t IOStatistics::copyGlobalStatistics(IOStatisticsGlobal *stats)
 856 {
 857         stats->kextCount = loadedKexts;
 858         stats->classCount = registeredClasses;
 859         stats->workloops = registeredWorkloops;
 860
 861         return sizeof(IOStatisticsGlobal);
 862 }
 863
 864 uint32_t IOStatistics::copyKextStatistics(IOStatisticsKext *stats)
 865 {
 866         KextNode *ke;
 867         ClassNode *ce;
 868         uint32_t index = 0;
 869
 870         RB_FOREACH(ke, KextTree, &kextHead) {
 871                 stats->loadTag = ke->loadTag;
 872                 ke->kext->getSizeInfo(&stats->loadSize, &stats->wiredSize);
 873
 874                 stats->classes = ke->classes;
 875
 876                 /* Append indices of owned classes */
 877                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 878                         stats->classIndexes[index++] = ce->classID;
 879                 }
 880
 881                 stats = (IOStatisticsKext *)((void*)((char*)stats + sizeof(IOStatisticsKext) + (ke->classes * sizeof(uint32_t))));
 882         }
 883
 884         return (sizeof(IOStatisticsKext) * loadedKexts + sizeof(uint32_t) * registeredClasses);
 885 }
 886
 887 uint32_t IOStatistics::copyMemoryStatistics(IOStatisticsMemory *stats)
 888 {
 889         KextNode *ke;
 890
 891         RB_FOREACH(ke, KextTree, &kextHead) {
 892                 stats->allocatedSize = ke->memoryCounters[kIOStatisticsMalloc];
 893                 stats->freedSize = ke->memoryCounters[kIOStatisticsFree];
 894                 stats->allocatedAlignedSize = ke->memoryCounters[kIOStatisticsMallocAligned];
 895                 stats->freedAlignedSize = ke->memoryCounters[kIOStatisticsFreeAligned];
 896                 stats->allocatedContiguousSize = ke->memoryCounters[kIOStatisticsMallocContiguous];
 897                 stats->freedContiguousSize = ke->memoryCounters[kIOStatisticsFreeContiguous];
 898                 stats->allocatedPageableSize = ke->memoryCounters[kIOStatisticsMallocPageable];
 899                 stats->freedPageableSize = ke->memoryCounters[kIOStatisticsFreePageable];
 900                 stats++;
 901         }
 902
 903         return (sizeof(IOStatisticsMemory) * loadedKexts);
 904 }
 905
 906 uint32_t IOStatistics::copyClassStatistics(IOStatisticsClass *stats)
 907 {
 908         KextNode *ke;
 909         ClassNode *ce;
 910
 911         RB_FOREACH(ke, KextTree, &kextHead) {
 912                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 913                         stats->classID = ce->classID;
 914                         stats->superClassID = ce->superClassID;
 915                         stats->classSize = ce->metaClass->getClassSize();
 916
 917                         stats++;
 918                 }
 919         }
 920
 921         return sizeof(IOStatisticsClass) * registeredClasses;
 922 }
 923
 924 uint32_t IOStatistics::copyCounterStatistics(IOStatisticsCounter *stats)
 925 {
 926         KextNode *ke;
 927         ClassNode *ce;
 928
 929         RB_FOREACH(ke, KextTree, &kextHead) {
 930                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 931                         IOUserClientCounter *userClientCounter;
 932                         IOEventSourceCounter *counter;
 933
 934                         stats->classID = ce->classID;
 935                         stats->classInstanceCount = ce->metaClass->getInstanceCount();
 936
 937                         IOStatisticsUserClients *uc = &stats->userClientStatistics;
 938
 939                         /* User client counters */
 940                         SLIST_FOREACH(userClientCounter, &ce->userClientList, link) {
 941                                 uc->clientCalls += userClientCounter->clientCalls;
 942                                 uc->created++;
 943                         }
 944
 945                         IOStatisticsInterruptEventSources *iec = &stats->interruptEventSourceStatistics;
 946                         IOStatisticsInterruptEventSources *fiec = &stats->filterInterruptEventSourceStatistics;
 947                         IOStatisticsTimerEventSources *tec = &stats->timerEventSourceStatistics;
 948                         IOStatisticsCommandGates *cgc = &stats->commandGateStatistics;
 949                         IOStatisticsCommandQueues *cqc = &stats->commandQueueStatistics;
 950                         IOStatisticsDerivedEventSources *dec = &stats->derivedEventSourceStatistics;
 951
 952                         /* Event source counters */
 953                         SLIST_FOREACH(counter, &ce->counterList, link) {
 954                                 switch (counter->type) {
 955                                         case kIOStatisticsInterruptEventSourceCounter:
 956                                                 iec->created++;
 957                                                 iec->produced += counter->u.interrupt.produced;
 958                                                 iec->checksForWork += counter->u.interrupt.checksForWork;
 959                                                 break;
 960                                         case kIOStatisticsFilterInterruptEventSourceCounter:
 961                                                 fiec->created++;
 962                                                 fiec->produced += counter->u.filter.produced;
 963                                                 fiec->checksForWork += counter->u.filter.checksForWork;
 964                                                 break;
 965                                         case kIOStatisticsTimerEventSourceCounter:
 966                                                 tec->created++;
 967                                                 tec->timeouts += counter->u.timer.timeouts;
 968                                                 tec->checksForWork += counter->u.timer.checksForWork;
 969                                                 tec->timeOnGate += counter->timeOnGate;
 970                                                 tec->closeGateCalls += counter->closeGateCalls;
 971                                                 tec->openGateCalls += counter->openGateCalls;
 972                                                 break;
 973                                         case kIOStatisticsCommandGateCounter:
 974                                                 cgc->created++;
 975                                                 cgc->timeOnGate += counter->timeOnGate;
 976                                                 cgc->actionCalls += counter->u.commandGate.actionCalls;
 977                                                 break;
 978                                         case kIOStatisticsCommandQueueCounter:
 979                                                 cqc->created++;
 980                                                 cqc->actionCalls += counter->u.commandQueue.actionCalls;
 981                                                 break;
 982                                         case kIOStatisticsDerivedEventSourceCounter:
 983                                                 dec->created++;
 984                                                 dec->timeOnGate += counter->timeOnGate;
 985                                                 dec->closeGateCalls += counter->closeGateCalls;
 986                                                 dec->openGateCalls += counter->openGateCalls;
 987                                                 break;
 988                                         default:
 989                                                 break;
 990                                 }
 991                         }
 992
 993                         stats++;
 994                 }
 995         }
 996
 997         return sizeof(IOStatisticsCounter) * registeredClasses;
 998 }
 999
1000 uint32_t IOStatistics::copyKextIdentifiers(IOStatisticsKextIdentifier *kextIDs)
1001 {
1002         KextNode *ke;
1003
1004         RB_FOREACH(ke, KextTree, &kextHead) {
1005                 strncpy(kextIDs->identifier, ke->kext->getIdentifierCString(), kIOStatisticsDriverNameLength);
1006                 kextIDs++;
1007         }
1008
1009         return (sizeof(IOStatisticsKextIdentifier) * loadedKexts);
1010 }
1011
1012 uint32_t IOStatistics::copyClassNames(IOStatisticsClassName *classNames)
1013 {
1014         KextNode *ke;
1015         ClassNode *ce;
1016
1017         RB_FOREACH(ke, KextTree, &kextHead) {
1018                 SLIST_FOREACH(ce, &ke->classList, lLink) {
1019                         strncpy(classNames->name, ce->metaClass->getClassName(), kIOStatisticsClassNameLength);
1020                         classNames++;
1021                 }
1022         }
1023
1024         return (sizeof(IOStatisticsClassName) * registeredClasses);
1025 }
1026
1027 uint32_t IOStatistics::copyWorkLoopStatistics(IOStatisticsWorkLoop *stats)
1028 {
1029         KextNode *ke;
1030         IOWorkLoopCounter *wlc;
1031         IOWorkLoopDependency *dependentNode;
1032         uint32_t size, accumulatedSize = 0;
1033
1034         RB_FOREACH(ke, KextTree, &kextHead) {
1035                 SLIST_FOREACH(wlc, &ke->workLoopList, link) {
1036                         stats->kextLoadTag = ke->loadTag;
1037                         stats->attachedEventSources = wlc->attachedEventSources;
1038                         stats->timeOnGate = wlc->timeOnGate;
1039                         stats->dependentKexts = 0;
1040                         RB_FOREACH(dependentNode, IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead) {
1041                                 stats->dependentKextLoadTags[stats->dependentKexts] = dependentNode->loadTag;
1042                                 stats->dependentKexts++;
1043                         }
1044
1045                         size = sizeof(IOStatisticsWorkLoop) + (sizeof(uint32_t) * stats->dependentKexts);
1046
1047                         accumulatedSize += size;
1048                         stats = (IOStatisticsWorkLoop*)((void*)((char*)stats + size));
1049                 }
1050         }
1051
1052         return accumulatedSize;
1053 }
1054
1055 uint32_t IOStatistics::copyUserClientStatistics(IOStatisticsUserClientHeader *stats, uint32_t loadTag)
1056 {
1057         KextNode *sought, *found = NULL;
1058         uint32_t procs = 0;
1059         IOUserClientProcessEntry *processEntry;
1060
1061         RB_FOREACH(sought, KextTree, &kextHead) {
1062                 if (sought->loadTag == loadTag) {
1063                         found = sought;
1064                         break;
1065                 }
1066         }
1067
1068         if (!found) {
1069                 return 0;
1070         }
1071
1072         TAILQ_FOREACH(processEntry, &found->userClientCallList, link) {
1073                 strncpy(stats->userClientCalls[procs].processName, processEntry->processName, kIOStatisticsProcessNameLength);
1074                 stats->userClientCalls[procs].pid = processEntry->pid;
1075                 stats->userClientCalls[procs].calls = processEntry->calls;
1076                 stats->processes++;
1077                 procs++;
1078         }
1079
1080         return sizeof(IOStatisticsUserClientCall) * stats->processes;
1081 }
1082
1083 void IOStatistics::storeUserClientCallInfo(IOUserClient *userClient, IOUserClientCounter *counter)
1084 {
1085         OSString *ossUserClientCreator = NULL;
1086         int32_t pid = -1;
1087         KextNode *parentKext;
1088         IOUserClientProcessEntry *entry, *nextEntry, *prevEntry = NULL;
1089         uint32_t count = 0;
1090         const char *ptr = NULL;
1091         OSObject *obj;
1092
1093         /* TODO: see if this can be more efficient */
1094         obj = userClient->copyProperty("IOUserClientCreator",
1095                                         gIOServicePlane,
1096                                         kIORegistryIterateRecursively | kIORegistryIterateParents);
1097
1098         if (!obj)
1099                 goto err_nounlock;
1100
1101         ossUserClientCreator = OSDynamicCast(OSString, obj);
1102
1103         if (ossUserClientCreator) {
1104                 uint32_t len, lenIter = 0;
1105
1106                 ptr = ossUserClientCreator->getCStringNoCopy();
1107                 len = ossUserClientCreator->getLength();
1108
1109                 while ((*ptr != ' ') && (lenIter < len)) {
1110                         ptr++;
1111                         lenIter++;
1112                 }
1113
1114                 if (lenIter < len) {
1115                         ptr++; // Skip the space
1116                         lenIter++;
1117                         pid = 0;
1118                         while ( (*ptr != ',') && (lenIter < len)) {
1119                                 pid = pid*10 + (*ptr - '0');
1120                                 ptr++;
1121                                 lenIter++;
1122                         }
1123
1124                         if(lenIter == len) {
1125                                 pid = -1;
1126                         } else {
1127                                 ptr += 2;
1128                         }
1129                 }
1130         }
1131
1132         if (-1 == pid)
1133                 goto err_nounlock;
1134
1135         IORWLockWrite(lock);
1136
1137         parentKext = counter->parentClass->parentKext;
1138
1139         TAILQ_FOREACH(entry, &parentKext->userClientCallList, link) {
1140                 if (entry->pid == pid) {
1141                         /* Found, so increment count and move to the head */
1142                         entry->calls++;
1143                         if (count) {
1144                                 TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1145                                 break;
1146                         }
1147                         else {
1148                                 /* At the head already, so increment and return */
1149                                 goto err_unlock;
1150                         }
1151                 }
1152
1153                 count++;
1154         }
1155
1156         if (!entry) {
1157                 if (count == IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS) {
1158                         /* Max elements hit, so reuse the last */
1159                         entry = TAILQ_LAST(&parentKext->userClientCallList, ProcessEntryList);
1160                         TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1161                 }
1162                 else {
1163                         /* Otherwise, allocate a new entry */
1164                         entry = (IOUserClientProcessEntry*)kalloc(sizeof(IOUserClientProcessEntry));
1165                         if (!entry) {
1166                             IORWLockUnlock(lock);
1167                                 return;
1168                         }
1169                 }
1170
1171                 strncpy(entry->processName, ptr, kIOStatisticsProcessNameLength);
1172                 entry->pid = pid;
1173                 entry->calls = 1;
1174         }
1175
1176         TAILQ_FOREACH(nextEntry, &parentKext->userClientCallList, link) {
1177                 if (nextEntry->calls <= entry->calls)
1178                         break;
1179
1180                 prevEntry = nextEntry;
1181         }
1182
1183         if (!prevEntry)
1184                 TAILQ_INSERT_HEAD(&parentKext->userClientCallList, entry, link);
1185         else
1186                 TAILQ_INSERT_AFTER(&parentKext->userClientCallList, prevEntry, entry, link);
1187
1188 err_unlock:
1189         IORWLockUnlock(lock);
1190
1191 err_nounlock:
1192         if (obj)
1193                 obj->release();
1194 }
1195
1196 void IOStatistics::countUserClientCall(IOUserClient *client) {
1197         IOUserClient::ExpansionData *data;
1198         IOUserClientCounter *counter;
1199
1200         /* Guard against an uninitialized client object - <rdar://problem/8577946> */
1201         if (!(data = client->reserved)) {
1202                 return;
1203         }
1204
1205         if ((counter = data->counter)) {
1206                 storeUserClientCallInfo(client, counter);
1207                 OSIncrementAtomic(&counter->clientCalls);
1208         }
1209 }
1210
1211 KextNode *IOStatistics::getKextNodeFromBacktrace(boolean_t write) {
1212         const uint32_t btMin = 3;
1213
1214         void *bt[16];
1215         unsigned btCount = sizeof(bt) / sizeof(bt[0]);
1216         vm_offset_t *scanAddr = NULL;
1217         uint32_t i;
1218         KextNode *found = NULL, *ke = NULL;
1219
1220         /*
1221          * Gathering the backtrace is a significant source of
1222          * overhead. OSBacktrace does many safety checks that
1223          * are not needed in this situation.
1224          */
1225         btCount = backtrace((uintptr_t*)bt, btCount);
1226
1227         if (write) {
1228                 IORWLockWrite(lock);
1229         } else {
1230                 IORWLockRead(lock);
1231         }
1232
1233         /* Ignore first levels */
1234         scanAddr = (vm_offset_t *)&bt[btMin - 1];
1235
1236         for (i = btMin - 1; i < btCount; i++, scanAddr++) {
1237                 ke = RB_ROOT(&kextAddressHead);
1238                 while (ke) {
1239                         if (*scanAddr < ke->address) {
1240                                 ke = RB_LEFT(ke, addressLink);
1241                         }
1242                         else {
1243                                 if ((*scanAddr < ke->address_end) && (*scanAddr >= ke->address)) {
1244                                         if (!ke->kext->isKernelComponent()) {
1245                                                 return ke;
1246                                         } else {
1247                                                 found = ke;
1248                                         }
1249                                 }
1250                                 ke = RB_RIGHT(ke, addressLink);
1251                         }
1252                 }
1253         }
1254
1255         if (!found) {
1256                 IORWLockUnlock(lock);
1257         }
1258
1259         return found;
1260 }
1261
1262 void IOStatistics::releaseKextNode(KextNode *node) {
1263 #pragma unused(node)
1264         IORWLockUnlock(lock);
1265 }
1266
1267 /* IOLib allocations */
1268 void IOStatistics::countAlloc(uint32_t index, vm_size_t size) {
1269         KextNode *ke;
1270
1271         if (!enabled) {
1272                 return;
1273         }
1274
1275         ke = getKextNodeFromBacktrace(FALSE);
1276         if (ke) {
1277                 OSAddAtomic(size, &ke->memoryCounters[index]);
1278                 releaseKextNode(ke);
1279         }
1280 }
1281
1282 #endif /* IOKITSTATS */