iokit/Kernel/IOStatistics.cpp

   1 /*
   2  * Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/sysctl.h>
  30 #include <kern/backtrace.h>
  31 #include <kern/host.h>
  32 #include <kern/zalloc.h>
  33
  34 #include <IOKit/system.h>
  35 #include <libkern/c++/OSKext.h>
  36 #include <libkern/OSAtomic.h>
  37
  38 #include <IOKit/IOStatisticsPrivate.h>
  39 #include <IOKit/IOUserClient.h>
  40 #include <IOKit/IOEventSource.h>
  41 #include <IOKit/IOKitDebug.h>
  42
  43 #if IOKITSTATS
  44 bool IOStatistics::enabled = false;
  45
  46 uint32_t IOStatistics::sequenceID = 0;
  47
  48 uint32_t IOStatistics::lastClassIndex = 0;
  49 uint32_t IOStatistics::lastKextIndex = 0;
  50
  51 uint32_t IOStatistics::loadedKexts = 0;
  52 uint32_t IOStatistics::registeredClasses = 0;
  53 uint32_t IOStatistics::registeredCounters = 0;
  54 uint32_t IOStatistics::registeredWorkloops = 0;
  55
  56 uint32_t IOStatistics::attachedEventSources = 0;
  57
  58 IOWorkLoopDependency *IOStatistics::nextWorkLoopDependency = NULL;
  59
  60 /* Logging */
  61
  62 #define LOG_LEVEL 0
  63
  64 #define LOG(level, format, ...) \
  65 do { \
  66         if (level <= LOG_LEVEL) \
  67                 printf(format, ##__VA_ARGS__); \
  68 } while (0)
  69
  70 /* Locks */
  71
  72 IORWLock *IOStatistics::lock = NULL;
  73
  74 /* Kext tree */
  75
  76 KextNode *IOStatistics::kextHint = NULL;
  77
  78 IOStatistics::KextTreeHead IOStatistics::kextHead = RB_INITIALIZER(&IOStatistics::kextHead);
  79
  80 int IOStatistics::kextNodeCompare(KextNode *e1, KextNode *e2)
  81 {
  82     if (e1->kext < e2->kext)
  83         return -1;
  84     else if (e1->kext > e2->kext)
  85         return 1;
  86     else
  87         return 0;
  88 }
  89
  90 RB_GENERATE(IOStatistics::KextTree, KextNode, link, kextNodeCompare);
  91
  92 /* Kext tree ordered by address */
  93
  94 IOStatistics::KextAddressTreeHead IOStatistics::kextAddressHead = RB_INITIALIZER(&IOStatistics::kextAddressHead);
  95
  96 int IOStatistics::kextAddressNodeCompare(KextNode *e1, KextNode *e2)
  97 {
  98     if (e1->address < e2->address)
  99         return -1;
 100     else if (e1->address > e2->address)
 101         return 1;
 102     else
 103         return 0;
 104 }
 105
 106 RB_GENERATE(IOStatistics::KextAddressTree, KextNode, addressLink, kextAddressNodeCompare);
 107
 108 /* Class tree */
 109
 110 IOStatistics::ClassTreeHead IOStatistics::classHead = RB_INITIALIZER(&IOStatistics::classHead);
 111
 112 int IOStatistics::classNodeCompare(ClassNode *e1, ClassNode *e2) {
 113     if (e1->metaClass < e2->metaClass)
 114         return -1;
 115     else if (e1->metaClass > e2->metaClass)
 116         return 1;
 117     else
 118         return 0;
 119 }
 120
 121 RB_GENERATE(IOStatistics::ClassTree, ClassNode, tLink, classNodeCompare);
 122
 123 /* Workloop dependencies */
 124
 125 int IOWorkLoopCounter::loadTagCompare(IOWorkLoopDependency *e1, IOWorkLoopDependency *e2) {
 126     if (e1->loadTag < e2->loadTag)
 127         return -1;
 128     else if (e1->loadTag > e2->loadTag)
 129         return 1;
 130     else
 131         return 0;
 132 }
 133
 134 RB_GENERATE(IOWorkLoopCounter::DependencyTree, IOWorkLoopDependency, link, IOWorkLoopCounter::loadTagCompare);
 135
 136 /* sysctl stuff */
 137
 138 static int
 139 oid_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1, int arg2, struct sysctl_req *req)
 140 {
 141         int error = EINVAL;
 142         uint32_t request = arg2;
 143
 144         switch (request)
 145         {
 146                 case kIOStatisticsGeneral:
 147                         error = IOStatistics::getStatistics(req);
 148                         break;
 149                 case kIOStatisticsWorkLoop:
 150                         error = IOStatistics::getWorkLoopStatistics(req);
 151                         break;
 152                 case kIOStatisticsUserClient:
 153                         error = IOStatistics::getUserClientStatistics(req);
 154                         break;
 155                 default:
 156                         break;
 157         }
 158
 159         return error;
 160 }
 161
 162 SYSCTL_NODE(_debug, OID_AUTO, iokit_statistics, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "IOStatistics");
 163
 164 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, general,
 165             CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 166             0, kIOStatisticsGeneral, oid_sysctl, "S", "");
 167
 168 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, workloop,
 169             CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 170             0, kIOStatisticsWorkLoop, oid_sysctl, "S", "");
 171
 172 static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, userclient,
 173             CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
 174             0, kIOStatisticsUserClient, oid_sysctl, "S", "");
 175
 176 void IOStatistics::initialize()
 177 {
 178         if (enabled) {
 179                 return;
 180         }
 181
 182         /* Only enabled if the boot argument is set. */
 183         if (!(kIOStatistics & gIOKitDebug)) {
 184                 return;
 185         }
 186
 187         sysctl_register_oid(&sysctl__debug_iokit_statistics_general);
 188         sysctl_register_oid(&sysctl__debug_iokit_statistics_workloop);
 189         sysctl_register_oid(&sysctl__debug_iokit_statistics_userclient);
 190
 191         lock = IORWLockAlloc();
 192         if (!lock) {
 193                 return;
 194         }
 195
 196         nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
 197         if (!nextWorkLoopDependency) {
 198                 return;
 199         }
 200
 201         enabled = true;
 202 }
 203
 204 void IOStatistics::onKextLoad(OSKext *kext, kmod_info_t *kmod_info)
 205 {
 206         KextNode *ke;
 207
 208         assert(kext && kmod_info);
 209
 210         if (!enabled) {
 211                 return;
 212         }
 213
 214         LOG(1, "IOStatistics::onKextLoad: %s, tag %d, address 0x%llx, address end 0x%llx\n",
 215                 kext->getIdentifierCString(), kmod_info->id, (uint64_t)kmod_info->address, (uint64_t)(kmod_info->address + kmod_info->size));
 216
 217         ke = (KextNode *)kalloc(sizeof(KextNode));
 218         if (!ke) {
 219                 return;
 220         }
 221
 222         memset(ke, 0, sizeof(KextNode));
 223
 224         ke->kext = kext;
 225         ke->loadTag = kmod_info->id;
 226         ke->address = kmod_info->address;
 227         ke->address_end = kmod_info->address + kmod_info->size;
 228
 229         SLIST_INIT(&ke->classList);
 230         TAILQ_INIT(&ke->userClientCallList);
 231
 232         IORWLockWrite(lock);
 233
 234         RB_INSERT(KextTree, &kextHead, ke);
 235         RB_INSERT(KextAddressTree, &kextAddressHead, ke);
 236
 237         sequenceID++;
 238         loadedKexts++;
 239         lastKextIndex++;
 240
 241         IORWLockUnlock(lock);
 242 }
 243
 244 void IOStatistics::onKextUnload(OSKext *kext)
 245 {
 246         KextNode sought, *found;
 247
 248         assert(kext);
 249
 250         if (!enabled) {
 251                 return;
 252         }
 253
 254         LOG(1, "IOStatistics::onKextUnload: %s\n", kext->getIdentifierCString());
 255
 256         IORWLockWrite(lock);
 257
 258         sought.kext = kext;
 259         found = RB_FIND(KextTree, &kextHead, &sought);
 260         if (found) {
 261                 IOWorkLoopCounter *wlc;
 262                 IOUserClientProcessEntry *uce;
 263
 264                 /* Disconnect workloop counters; cleanup takes place in unregisterWorkLoop() */
 265                 while ((wlc = SLIST_FIRST(&found->workLoopList))) {
 266                         SLIST_REMOVE_HEAD(&found->workLoopList, link);
 267                         wlc->parentKext = NULL;
 268                 }
 269
 270                 /* Free up the user client list */
 271                 while ((uce = TAILQ_FIRST(&found->userClientCallList))) {
 272                         TAILQ_REMOVE(&found->userClientCallList, uce, link);
 273                         kfree(uce, sizeof(IOUserClientProcessEntry));
 274                 }
 275
 276                 /* Remove from kext trees */
 277                 RB_REMOVE(KextTree, &kextHead, found);
 278                 RB_REMOVE(KextAddressTree, &kextAddressHead, found);
 279
 280                 /*
 281                  * Clear a matching kextHint to avoid use after free in
 282                  * onClassAdded() for a class add after a KEXT unload.
 283                  */
 284                 if (found == kextHint) {
 285                         kextHint = NULL;
 286                 }
 287
 288                 /* Finally, free the class node */
 289                 kfree(found, sizeof(KextNode));
 290
 291                 sequenceID++;
 292                 loadedKexts--;
 293         }
 294         else {
 295                 panic("IOStatistics::onKextUnload: cannot find kext: %s", kext->getIdentifierCString());
 296         }
 297
 298         IORWLockUnlock(lock);
 299 }
 300
 301 void IOStatistics::onClassAdded(OSKext *parentKext, OSMetaClass *metaClass)
 302 {
 303         ClassNode *ce;
 304         KextNode soughtKext, *foundKext = NULL;
 305
 306         assert(parentKext && metaClass);
 307
 308         if (!enabled) {
 309                 return;
 310         }
 311
 312         LOG(1, "IOStatistics::onClassAdded: %s\n", metaClass->getClassName());
 313
 314         ce = (ClassNode *)kalloc(sizeof(ClassNode));
 315         if (!ce) {
 316                 return;
 317         }
 318
 319         memset(ce, 0, sizeof(ClassNode));
 320
 321         IORWLockWrite(lock);
 322
 323         /* Hinted? */
 324         if (kextHint && kextHint->kext == parentKext) {
 325                 foundKext = kextHint;
 326         }
 327         else {
 328                 soughtKext.kext = parentKext;
 329                 foundKext = RB_FIND(KextTree, &kextHead, &soughtKext);
 330         }
 331
 332         if (foundKext) {
 333                 ClassNode soughtClass, *foundClass = NULL;
 334                 const OSMetaClass *superClass;
 335
 336                 ce->metaClass = metaClass;
 337                 ce->classID = lastClassIndex++;
 338                 ce->parentKext = foundKext;
 339
 340                 /* Has superclass? */
 341                 superClass = ce->metaClass->getSuperClass();
 342                 if (superClass) {
 343                         soughtClass.metaClass = superClass;
 344                         foundClass = RB_FIND(ClassTree, &classHead, &soughtClass);
 345                 }
 346                 ce->superClassID = foundClass ? foundClass->classID : (uint32_t)(-1);
 347
 348                 SLIST_INIT(&ce->counterList);
 349                 SLIST_INIT(&ce->userClientList);
 350
 351                 RB_INSERT(ClassTree, &classHead, ce);
 352                 SLIST_INSERT_HEAD(&foundKext->classList, ce, lLink);
 353
 354                 foundKext->classes++;
 355
 356                 kextHint = foundKext;
 357
 358                 sequenceID++;
 359                 registeredClasses++;
 360         }
 361         else {
 362                 panic("IOStatistics::onClassAdded: cannot find parent kext: %s", parentKext->getIdentifierCString());
 363         }
 364
 365         IORWLockUnlock(lock);
 366 }
 367
 368 void IOStatistics::onClassRemoved(OSKext *parentKext, OSMetaClass *metaClass)
 369 {
 370         ClassNode sought, *found;
 371
 372         assert(parentKext && metaClass);
 373
 374         if (!enabled) {
 375                 return;
 376         }
 377
 378         LOG(1, "IOStatistics::onClassRemoved: %s\n", metaClass->getClassName());
 379
 380         IORWLockWrite(lock);
 381
 382         sought.metaClass = metaClass;
 383         found = RB_FIND(ClassTree, &classHead, &sought);
 384         if (found) {
 385                 IOEventSourceCounter *esc;
 386                 IOUserClientCounter *ucc;
 387
 388                 /* Free up the list of counters */
 389                 while ((esc = SLIST_FIRST(&found->counterList))) {
 390                         SLIST_REMOVE_HEAD(&found->counterList, link);
 391                         kfree(esc, sizeof(IOEventSourceCounter));
 392                 }
 393
 394                 /* Free up the user client list */
 395                 while ((ucc = SLIST_FIRST(&found->userClientList))) {
 396                         SLIST_REMOVE_HEAD(&found->userClientList, link);
 397                         kfree(ucc, sizeof(IOUserClientCounter));
 398                 }
 399
 400                 /* Remove from class tree */
 401                 RB_REMOVE(ClassTree, &classHead, found);
 402
 403                 /* Remove from parent */
 404                 SLIST_REMOVE(&found->parentKext->classList, found, ClassNode, lLink);
 405
 406                 /* Finally, free the class node */
 407                 kfree(found, sizeof(ClassNode));
 408
 409                 sequenceID++;
 410                 registeredClasses--;
 411         }
 412         else {
 413                 panic("IOStatistics::onClassRemoved: cannot find class: %s", metaClass->getClassName());
 414         }
 415
 416         IORWLockUnlock(lock);
 417 }
 418
 419 IOEventSourceCounter *IOStatistics::registerEventSource(OSObject *inOwner)
 420 {
 421         IOEventSourceCounter *counter = NULL;
 422         ClassNode sought, *found = NULL;
 423         boolean_t createDummyCounter = FALSE;
 424
 425         assert(inOwner);
 426
 427         if (!enabled) {
 428                 return NULL;
 429         }
 430
 431         counter = (IOEventSourceCounter*)kalloc(sizeof(IOEventSourceCounter));
 432         if (!counter) {
 433                 return NULL;
 434         }
 435
 436         memset(counter, 0, sizeof(IOEventSourceCounter));
 437
 438         IORWLockWrite(lock);
 439
 440         /* Workaround for <rdar://problem/7158117> - create a dummy counter when inOwner is bad.
 441          * We use retainCount here as our best indication that the pointer is awry.
 442          */
 443         if (inOwner->retainCount > 0xFFFFFF) {
 444                 kprintf("IOStatistics::registerEventSource - bad metaclass %p\n", inOwner);
 445                 createDummyCounter = TRUE;
 446         }
 447         else {
 448                 sought.metaClass = inOwner->getMetaClass();
 449                 found = RB_FIND(ClassTree, &classHead, &sought);
 450         }
 451
 452         if (found) {
 453                 counter->parentClass = found;
 454                 SLIST_INSERT_HEAD(&found->counterList, counter, link);
 455                 registeredCounters++;
 456         }
 457
 458         if (!(createDummyCounter || found)) {
 459                 panic("IOStatistics::registerEventSource: cannot find parent class: %s", inOwner->getMetaClass()->getClassName());
 460         }
 461
 462         IORWLockUnlock(lock);
 463
 464         return counter;
 465 }
 466
 467 void IOStatistics::unregisterEventSource(IOEventSourceCounter *counter)
 468 {
 469         if (!counter) {
 470                 return;
 471         }
 472
 473         IORWLockWrite(lock);
 474
 475         if (counter->parentClass) {
 476                 SLIST_REMOVE(&counter->parentClass->counterList, counter, IOEventSourceCounter, link);
 477                 registeredCounters--;
 478         }
 479         kfree(counter, sizeof(IOEventSourceCounter));
 480
 481         IORWLockUnlock(lock);
 482 }
 483
 484 IOWorkLoopCounter* IOStatistics::registerWorkLoop(IOWorkLoop *workLoop)
 485 {
 486         IOWorkLoopCounter *counter = NULL;
 487         KextNode *found;
 488
 489         assert(workLoop);
 490
 491         if (!enabled) {
 492                 return NULL;
 493         }
 494
 495         counter = (IOWorkLoopCounter*)kalloc(sizeof(IOWorkLoopCounter));
 496         if (!counter) {
 497                 return NULL;
 498         }
 499
 500         memset(counter, 0, sizeof(IOWorkLoopCounter));
 501
 502         found = getKextNodeFromBacktrace(TRUE);
 503         if (!found) {
 504                 panic("IOStatistics::registerWorkLoop: cannot find parent kext");
 505         }
 506
 507         counter->parentKext = found;
 508         counter->workLoop = workLoop;
 509         RB_INIT(&counter->dependencyHead);
 510         SLIST_INSERT_HEAD(&found->workLoopList, counter, link);
 511         registeredWorkloops++;
 512
 513         releaseKextNode(found);
 514
 515         return counter;
 516 }
 517
 518 void IOStatistics::unregisterWorkLoop(IOWorkLoopCounter *counter)
 519 {
 520         if (!counter) {
 521                 return;
 522         }
 523
 524         IORWLockWrite(lock);
 525         if (counter->parentKext) {
 526                 SLIST_REMOVE(&counter->parentKext->workLoopList, counter, IOWorkLoopCounter, link);
 527         }
 528         kfree(counter, sizeof(IOWorkLoopCounter));
 529         registeredWorkloops--;
 530
 531         IORWLockUnlock(lock);
 532 }
 533
 534 IOUserClientCounter *IOStatistics::registerUserClient(IOUserClient *userClient)
 535 {
 536         ClassNode sought, *found;
 537         IOUserClientCounter *counter = NULL;
 538
 539         assert(userClient);
 540
 541         if (!enabled) {
 542                 return NULL;
 543         }
 544
 545         counter = (IOUserClientCounter*)kalloc(sizeof(IOUserClientCounter));
 546         if (!counter) {
 547                 return NULL;
 548         }
 549
 550         memset(counter, 0, sizeof(IOUserClientCounter));
 551
 552         IORWLockWrite(lock);
 553
 554         sought.metaClass = userClient->getMetaClass();
 555
 556         found = RB_FIND(ClassTree, &classHead, &sought);
 557         if (found) {
 558                 counter->parentClass = found;
 559                 SLIST_INSERT_HEAD(&found->userClientList, counter, link);
 560         }
 561         else {
 562                 panic("IOStatistics::registerUserClient: cannot find parent class: %s", sought.metaClass->getClassName());
 563         }
 564
 565         IORWLockUnlock(lock);
 566
 567         return counter;
 568 }
 569
 570 void IOStatistics::unregisterUserClient(IOUserClientCounter *counter)
 571 {
 572         if (!counter) {
 573                 return;
 574         }
 575
 576         IORWLockWrite(lock);
 577
 578         SLIST_REMOVE(&counter->parentClass->userClientList, counter, IOUserClientCounter, link);
 579         kfree(counter, sizeof(IOUserClientCounter));
 580
 581         IORWLockUnlock(lock);
 582 }
 583
 584 void IOStatistics::attachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
 585 {
 586         if (!wlc) {
 587         return;
 588         }
 589
 590         IORWLockWrite(lock);
 591
 592         if (!nextWorkLoopDependency) {
 593                 return;
 594         }
 595
 596         attachedEventSources++;
 597         wlc->attachedEventSources++;
 598
 599         /* Track the kext dependency */
 600         nextWorkLoopDependency->loadTag = esc->parentClass->parentKext->loadTag;
 601         if (NULL == RB_INSERT(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, nextWorkLoopDependency)) {
 602                 nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
 603         }
 604
 605         IORWLockUnlock(lock);
 606 }
 607
 608 void IOStatistics::detachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
 609 {
 610         IOWorkLoopDependency sought, *found;
 611
 612         if (!wlc) {
 613                 return;
 614         }
 615
 616         IORWLockWrite(lock);
 617
 618         attachedEventSources--;
 619         wlc->attachedEventSources--;
 620
 621         sought.loadTag = esc->parentClass->parentKext->loadTag;
 622
 623         found = RB_FIND(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, &sought);
 624         if (found) {
 625                 RB_REMOVE(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, found);
 626                 kfree(found, sizeof(IOWorkLoopDependency));
 627         }
 628
 629         IORWLockUnlock(lock);
 630 }
 631
 632 int IOStatistics::getStatistics(sysctl_req *req)
 633 {
 634         int error;
 635         uint32_t calculatedSize, size;
 636         char *buffer, *ptr;
 637         IOStatisticsHeader *header;
 638
 639         assert(IOStatistics::enabled && req);
 640
 641         IORWLockRead(IOStatistics::lock);
 642
 643         /* Work out how much we need to allocate. IOStatisticsKext is of variable size. */
 644         calculatedSize = sizeof(IOStatisticsHeader) +
 645                                          sizeof(IOStatisticsGlobal) +
 646                                         (sizeof(IOStatisticsKext) * loadedKexts) + (sizeof(uint32_t) * registeredClasses) +
 647                                         (sizeof(IOStatisticsMemory) * loadedKexts) +
 648                                         (sizeof(IOStatisticsClass) * registeredClasses) +
 649                                         (sizeof(IOStatisticsCounter) * registeredClasses) +
 650                                         (sizeof(IOStatisticsKextIdentifier) * loadedKexts) +
 651                                         (sizeof(IOStatisticsClassName) * registeredClasses);
 652
 653         /* Size request? */
 654         if (req->oldptr == USER_ADDR_NULL) {
 655                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 656                 goto exit;
 657         }
 658
 659         /* Read only */
 660         if (req->newptr != USER_ADDR_NULL) {
 661                 error = EPERM;
 662                 goto exit;
 663         }
 664
 665         buffer = (char*)kalloc(calculatedSize);
 666         if (!buffer) {
 667                 error = ENOMEM;
 668                 goto exit;
 669         }
 670
 671         memset(buffer, 0, calculatedSize);
 672
 673         ptr = buffer;
 674
 675         header = (IOStatisticsHeader*)((void*)ptr);
 676
 677         header->sig = IOSTATISTICS_SIG;
 678         header->ver = IOSTATISTICS_VER;
 679
 680         header->seq = sequenceID;
 681
 682         ptr += sizeof(IOStatisticsHeader);
 683
 684         /* Global data - seq, timers, interrupts, etc) */
 685         header->globalStatsOffset = sizeof(IOStatisticsHeader);
 686         size = copyGlobalStatistics((IOStatisticsGlobal*)((void*)ptr));
 687         ptr += size;
 688
 689         /* Kext statistics */
 690         header->kextStatsOffset = header->globalStatsOffset + size;
 691         size = copyKextStatistics((IOStatisticsKext*)((void*)ptr));
 692         ptr += size;
 693
 694         /* Memory allocation info */
 695         header->memoryStatsOffset = header->kextStatsOffset + size;
 696         size = copyMemoryStatistics((IOStatisticsMemory*)((void*)ptr));
 697         ptr += size;
 698
 699         /* Class statistics */
 700         header->classStatsOffset = header->memoryStatsOffset + size;
 701         size = copyClassStatistics((IOStatisticsClass*)((void*)ptr));
 702         ptr += size;
 703
 704         /* Dynamic class counter data */
 705         header->counterStatsOffset = header->classStatsOffset + size;
 706         size = copyCounterStatistics((IOStatisticsCounter*)((void*)ptr));
 707         ptr += size;
 708
 709         /* Kext identifiers */
 710         header->kextIdentifiersOffset = header->counterStatsOffset + size;
 711         size = copyKextIdentifiers((IOStatisticsKextIdentifier*)((void*)ptr));
 712         ptr += size;
 713
 714         /* Class names */
 715         header->classNamesOffset = header->kextIdentifiersOffset + size;
 716         size = copyClassNames((IOStatisticsClassName*)ptr);
 717         ptr += size;
 718
 719         LOG(2, "IOStatistics::getStatistics - calculatedSize 0x%x, kexts 0x%x, classes 0x%x.\n",
 720                 calculatedSize, loadedKexts, registeredClasses);
 721
 722         assert( (uint32_t)(ptr - buffer) == calculatedSize );
 723
 724         error = SYSCTL_OUT(req, buffer, calculatedSize);
 725
 726         kfree(buffer, calculatedSize);
 727
 728 exit:
 729         IORWLockUnlock(IOStatistics::lock);
 730         return error;
 731 }
 732
 733 int IOStatistics::getWorkLoopStatistics(sysctl_req *req)
 734 {
 735         int error;
 736         uint32_t calculatedSize, size;
 737         char *buffer;
 738         IOStatisticsWorkLoopHeader *header;
 739
 740         assert(IOStatistics::enabled && req);
 741
 742         IORWLockRead(IOStatistics::lock);
 743
 744         /* Approximate how much we need to allocate (worse case estimate) */
 745         calculatedSize = sizeof(IOStatisticsWorkLoop) * registeredWorkloops +
 746                                          sizeof(uint32_t) * attachedEventSources;
 747
 748         /* Size request? */
 749         if (req->oldptr == USER_ADDR_NULL) {
 750                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 751                 goto exit;
 752         }
 753
 754         /* Read only */
 755         if (req->newptr != USER_ADDR_NULL) {
 756                 error = EPERM;
 757                 goto exit;
 758         }
 759
 760         buffer = (char*)kalloc(calculatedSize);
 761         if (!buffer) {
 762                 error = ENOMEM;
 763                 goto exit;
 764         }
 765         memset(buffer, 0, calculatedSize);
 766         header = (IOStatisticsWorkLoopHeader*)((void*)buffer);
 767
 768         header->sig = IOSTATISTICS_SIG_WORKLOOP;
 769         header->ver = IOSTATISTICS_VER;
 770
 771         header->seq = sequenceID;
 772
 773         header->workloopCount = registeredWorkloops;
 774
 775         size = copyWorkLoopStatistics(&header->workLoopStats);
 776
 777         LOG(2, "IOStatistics::getWorkLoopStatistics: calculatedSize %d, size %d\n", calculatedSize, size);
 778
 779         assert( size <= calculatedSize );
 780
 781         error = SYSCTL_OUT(req, buffer, size);
 782
 783         kfree(buffer, calculatedSize);
 784
 785 exit:
 786         IORWLockUnlock(IOStatistics::lock);
 787         return error;
 788 }
 789
 790 int IOStatistics::getUserClientStatistics(sysctl_req *req)
 791 {
 792         int error;
 793         uint32_t calculatedSize, size;
 794         char *buffer;
 795         uint32_t requestedLoadTag = 0;
 796         IOStatisticsUserClientHeader *header;
 797
 798         assert(IOStatistics::enabled && req);
 799
 800         IORWLockRead(IOStatistics::lock);
 801
 802         /* Work out how much we need to allocate */
 803         calculatedSize = sizeof(IOStatisticsUserClientHeader) +
 804                                          sizeof(IOStatisticsUserClientCall) * IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS * loadedKexts;
 805
 806         /* Size request? */
 807         if (req->oldptr == USER_ADDR_NULL) {
 808                 error = SYSCTL_OUT(req, NULL, calculatedSize);
 809                 goto exit;
 810         }
 811
 812         /* Kext request (potentially) valid? */
 813         if (!req->newptr || req->newlen < sizeof(requestedLoadTag)) {
 814                 error = EINVAL;
 815                 goto exit;
 816         }
 817
 818         error = SYSCTL_IN(req, &requestedLoadTag, sizeof(requestedLoadTag));
 819         if (error) {
 820                 goto exit;
 821         }
 822
 823         LOG(2, "IOStatistics::getUserClientStatistics - requesting kext w/load tag: %d\n", requestedLoadTag);
 824
 825         buffer = (char*)kalloc(calculatedSize);
 826         if (!buffer) {
 827                 error = ENOMEM;
 828                 goto exit;
 829         }
 830         memset(buffer, 0, calculatedSize);
 831         header = (IOStatisticsUserClientHeader*)((void*)buffer);
 832
 833         header->sig = IOSTATISTICS_SIG_USERCLIENT;
 834         header->ver = IOSTATISTICS_VER;
 835
 836         header->seq = sequenceID;
 837
 838         header->processes = 0;
 839
 840         size = copyUserClientStatistics(header, requestedLoadTag);
 841
 842         assert((sizeof(IOStatisticsUserClientHeader) + size) <= calculatedSize);
 843
 844         if (size) {
 845                 error = SYSCTL_OUT(req, buffer, sizeof(IOStatisticsUserClientHeader) + size);
 846         }
 847         else {
 848                 error = EINVAL;
 849         }
 850
 851         kfree(buffer, calculatedSize);
 852
 853 exit:
 854         IORWLockUnlock(IOStatistics::lock);
 855         return error;
 856 }
 857
 858 uint32_t IOStatistics::copyGlobalStatistics(IOStatisticsGlobal *stats)
 859 {
 860         stats->kextCount = loadedKexts;
 861         stats->classCount = registeredClasses;
 862         stats->workloops = registeredWorkloops;
 863
 864         return sizeof(IOStatisticsGlobal);
 865 }
 866
 867 uint32_t IOStatistics::copyKextStatistics(IOStatisticsKext *stats)
 868 {
 869         KextNode *ke;
 870         ClassNode *ce;
 871         uint32_t index = 0;
 872
 873         RB_FOREACH(ke, KextTree, &kextHead) {
 874                 stats->loadTag = ke->loadTag;
 875                 ke->kext->getSizeInfo(&stats->loadSize, &stats->wiredSize);
 876
 877                 stats->classes = ke->classes;
 878
 879                 /* Append indices of owned classes */
 880                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 881                         stats->classIndexes[index++] = ce->classID;
 882                 }
 883
 884                 stats = (IOStatisticsKext *)((void*)((char*)stats + sizeof(IOStatisticsKext) + (ke->classes * sizeof(uint32_t))));
 885         }
 886
 887         return (sizeof(IOStatisticsKext) * loadedKexts + sizeof(uint32_t) * registeredClasses);
 888 }
 889
 890 uint32_t IOStatistics::copyMemoryStatistics(IOStatisticsMemory *stats)
 891 {
 892         KextNode *ke;
 893
 894         RB_FOREACH(ke, KextTree, &kextHead) {
 895                 stats->allocatedSize = ke->memoryCounters[kIOStatisticsMalloc];
 896                 stats->freedSize = ke->memoryCounters[kIOStatisticsFree];
 897                 stats->allocatedAlignedSize = ke->memoryCounters[kIOStatisticsMallocAligned];
 898                 stats->freedAlignedSize = ke->memoryCounters[kIOStatisticsFreeAligned];
 899                 stats->allocatedContiguousSize = ke->memoryCounters[kIOStatisticsMallocContiguous];
 900                 stats->freedContiguousSize = ke->memoryCounters[kIOStatisticsFreeContiguous];
 901                 stats->allocatedPageableSize = ke->memoryCounters[kIOStatisticsMallocPageable];
 902                 stats->freedPageableSize = ke->memoryCounters[kIOStatisticsFreePageable];
 903                 stats++;
 904         }
 905
 906         return (sizeof(IOStatisticsMemory) * loadedKexts);
 907 }
 908
 909 uint32_t IOStatistics::copyClassStatistics(IOStatisticsClass *stats)
 910 {
 911         KextNode *ke;
 912         ClassNode *ce;
 913
 914         RB_FOREACH(ke, KextTree, &kextHead) {
 915                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 916                         stats->classID = ce->classID;
 917                         stats->superClassID = ce->superClassID;
 918                         stats->classSize = ce->metaClass->getClassSize();
 919
 920                         stats++;
 921                 }
 922         }
 923
 924         return sizeof(IOStatisticsClass) * registeredClasses;
 925 }
 926
 927 uint32_t IOStatistics::copyCounterStatistics(IOStatisticsCounter *stats)
 928 {
 929         KextNode *ke;
 930         ClassNode *ce;
 931
 932         RB_FOREACH(ke, KextTree, &kextHead) {
 933                 SLIST_FOREACH(ce, &ke->classList, lLink) {
 934                         IOUserClientCounter *userClientCounter;
 935                         IOEventSourceCounter *counter;
 936
 937                         stats->classID = ce->classID;
 938                         stats->classInstanceCount = ce->metaClass->getInstanceCount();
 939
 940                         IOStatisticsUserClients *uc = &stats->userClientStatistics;
 941
 942                         /* User client counters */
 943                         SLIST_FOREACH(userClientCounter, &ce->userClientList, link) {
 944                                 uc->clientCalls += userClientCounter->clientCalls;
 945                                 uc->created++;
 946                         }
 947
 948                         IOStatisticsInterruptEventSources *iec = &stats->interruptEventSourceStatistics;
 949                         IOStatisticsInterruptEventSources *fiec = &stats->filterInterruptEventSourceStatistics;
 950                         IOStatisticsTimerEventSources *tec = &stats->timerEventSourceStatistics;
 951                         IOStatisticsCommandGates *cgc = &stats->commandGateStatistics;
 952                         IOStatisticsCommandQueues *cqc = &stats->commandQueueStatistics;
 953                         IOStatisticsDerivedEventSources *dec = &stats->derivedEventSourceStatistics;
 954
 955                         /* Event source counters */
 956                         SLIST_FOREACH(counter, &ce->counterList, link) {
 957                                 switch (counter->type) {
 958                                         case kIOStatisticsInterruptEventSourceCounter:
 959                                                 iec->created++;
 960                                                 iec->produced += counter->u.interrupt.produced;
 961                                                 iec->checksForWork += counter->u.interrupt.checksForWork;
 962                                                 break;
 963                                         case kIOStatisticsFilterInterruptEventSourceCounter:
 964                                                 fiec->created++;
 965                                                 fiec->produced += counter->u.filter.produced;
 966                                                 fiec->checksForWork += counter->u.filter.checksForWork;
 967                                                 break;
 968                                         case kIOStatisticsTimerEventSourceCounter:
 969                                                 tec->created++;
 970                                                 tec->timeouts += counter->u.timer.timeouts;
 971                                                 tec->checksForWork += counter->u.timer.checksForWork;
 972                                                 tec->timeOnGate += counter->timeOnGate;
 973                                                 tec->closeGateCalls += counter->closeGateCalls;
 974                                                 tec->openGateCalls += counter->openGateCalls;
 975                                                 break;
 976                                         case kIOStatisticsCommandGateCounter:
 977                                                 cgc->created++;
 978                                                 cgc->timeOnGate += counter->timeOnGate;
 979                                                 cgc->actionCalls += counter->u.commandGate.actionCalls;
 980                                                 break;
 981                                         case kIOStatisticsCommandQueueCounter:
 982                                                 cqc->created++;
 983                                                 cqc->actionCalls += counter->u.commandQueue.actionCalls;
 984                                                 break;
 985                                         case kIOStatisticsDerivedEventSourceCounter:
 986                                                 dec->created++;
 987                                                 dec->timeOnGate += counter->timeOnGate;
 988                                                 dec->closeGateCalls += counter->closeGateCalls;
 989                                                 dec->openGateCalls += counter->openGateCalls;
 990                                                 break;
 991                                         default:
 992                                                 break;
 993                                 }
 994                         }
 995
 996                         stats++;
 997                 }
 998         }
 999
1000         return sizeof(IOStatisticsCounter) * registeredClasses;
1001 }
1002
1003 uint32_t IOStatistics::copyKextIdentifiers(IOStatisticsKextIdentifier *kextIDs)
1004 {
1005         KextNode *ke;
1006
1007         RB_FOREACH(ke, KextTree, &kextHead) {
1008                 strncpy(kextIDs->identifier, ke->kext->getIdentifierCString(), kIOStatisticsDriverNameLength);
1009                 kextIDs++;
1010         }
1011
1012         return (sizeof(IOStatisticsKextIdentifier) * loadedKexts);
1013 }
1014
1015 uint32_t IOStatistics::copyClassNames(IOStatisticsClassName *classNames)
1016 {
1017         KextNode *ke;
1018         ClassNode *ce;
1019
1020         RB_FOREACH(ke, KextTree, &kextHead) {
1021                 SLIST_FOREACH(ce, &ke->classList, lLink) {
1022                         strncpy(classNames->name, ce->metaClass->getClassName(), kIOStatisticsClassNameLength);
1023                         classNames++;
1024                 }
1025         }
1026
1027         return (sizeof(IOStatisticsClassName) * registeredClasses);
1028 }
1029
1030 uint32_t IOStatistics::copyWorkLoopStatistics(IOStatisticsWorkLoop *stats)
1031 {
1032         KextNode *ke;
1033         IOWorkLoopCounter *wlc;
1034         IOWorkLoopDependency *dependentNode;
1035         uint32_t size, accumulatedSize = 0;
1036
1037         RB_FOREACH(ke, KextTree, &kextHead) {
1038                 SLIST_FOREACH(wlc, &ke->workLoopList, link) {
1039                         stats->kextLoadTag = ke->loadTag;
1040                         stats->attachedEventSources = wlc->attachedEventSources;
1041                         stats->timeOnGate = wlc->timeOnGate;
1042                         stats->dependentKexts = 0;
1043                         RB_FOREACH(dependentNode, IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead) {
1044                                 stats->dependentKextLoadTags[stats->dependentKexts] = dependentNode->loadTag;
1045                                 stats->dependentKexts++;
1046                         }
1047
1048                         size = sizeof(IOStatisticsWorkLoop) + (sizeof(uint32_t) * stats->dependentKexts);
1049
1050                         accumulatedSize += size;
1051                         stats = (IOStatisticsWorkLoop*)((void*)((char*)stats + size));
1052                 }
1053         }
1054
1055         return accumulatedSize;
1056 }
1057
1058 uint32_t IOStatistics::copyUserClientStatistics(IOStatisticsUserClientHeader *stats, uint32_t loadTag)
1059 {
1060         KextNode *sought, *found = NULL;
1061         uint32_t procs = 0;
1062         IOUserClientProcessEntry *processEntry;
1063
1064         RB_FOREACH(sought, KextTree, &kextHead) {
1065                 if (sought->loadTag == loadTag) {
1066                         found = sought;
1067                         break;
1068                 }
1069         }
1070
1071         if (!found) {
1072                 return 0;
1073         }
1074
1075         TAILQ_FOREACH(processEntry, &found->userClientCallList, link) {
1076                 strncpy(stats->userClientCalls[procs].processName, processEntry->processName, kIOStatisticsProcessNameLength);
1077                 stats->userClientCalls[procs].pid = processEntry->pid;
1078                 stats->userClientCalls[procs].calls = processEntry->calls;
1079                 stats->processes++;
1080                 procs++;
1081         }
1082
1083         return sizeof(IOStatisticsUserClientCall) * stats->processes;
1084 }
1085
1086 void IOStatistics::storeUserClientCallInfo(IOUserClient *userClient, IOUserClientCounter *counter)
1087 {
1088         OSString *ossUserClientCreator = NULL;
1089         int32_t pid = -1;
1090         KextNode *parentKext;
1091         IOUserClientProcessEntry *entry, *nextEntry, *prevEntry = NULL;
1092         uint32_t count = 0;
1093         const char *ptr = NULL;
1094         OSObject *obj;
1095
1096         /* TODO: see if this can be more efficient */
1097         obj = userClient->copyProperty("IOUserClientCreator",
1098                                         gIOServicePlane,
1099                                         kIORegistryIterateRecursively | kIORegistryIterateParents);
1100
1101         if (!obj)
1102                 goto err_nounlock;
1103
1104         ossUserClientCreator = OSDynamicCast(OSString, obj);
1105
1106         if (ossUserClientCreator) {
1107                 uint32_t len, lenIter = 0;
1108
1109                 ptr = ossUserClientCreator->getCStringNoCopy();
1110                 len = ossUserClientCreator->getLength();
1111
1112                 while ((*ptr != ' ') && (lenIter < len)) {
1113                         ptr++;
1114                         lenIter++;
1115                 }
1116
1117                 if (lenIter < len) {
1118                         ptr++; // Skip the space
1119                         lenIter++;
1120                         pid = 0;
1121                         while ( (*ptr != ',') && (lenIter < len)) {
1122                                 pid = pid*10 + (*ptr - '0');
1123                                 ptr++;
1124                                 lenIter++;
1125                         }
1126
1127                         if(lenIter == len) {
1128                                 pid = -1;
1129                         } else {
1130                                 ptr += 2;
1131                         }
1132                 }
1133         }
1134
1135         if (-1 == pid)
1136                 goto err_nounlock;
1137
1138         IORWLockWrite(lock);
1139
1140         parentKext = counter->parentClass->parentKext;
1141
1142         TAILQ_FOREACH(entry, &parentKext->userClientCallList, link) {
1143                 if (entry->pid == pid) {
1144                         /* Found, so increment count and move to the head */
1145                         entry->calls++;
1146                         if (count) {
1147                                 TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1148                                 break;
1149                         }
1150                         else {
1151                                 /* At the head already, so increment and return */
1152                                 goto err_unlock;
1153                         }
1154                 }
1155
1156                 count++;
1157         }
1158
1159         if (!entry) {
1160                 if (count == IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS) {
1161                         /* Max elements hit, so reuse the last */
1162                         entry = TAILQ_LAST(&parentKext->userClientCallList, ProcessEntryList);
1163                         TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1164                 }
1165                 else {
1166                         /* Otherwise, allocate a new entry */
1167                         entry = (IOUserClientProcessEntry*)kalloc(sizeof(IOUserClientProcessEntry));
1168                         if (!entry) {
1169                             IORWLockUnlock(lock);
1170                                 return;
1171                         }
1172                 }
1173
1174                 strncpy(entry->processName, ptr, kIOStatisticsProcessNameLength);
1175                 entry->pid = pid;
1176                 entry->calls = 1;
1177         }
1178
1179         TAILQ_FOREACH(nextEntry, &parentKext->userClientCallList, link) {
1180                 if (nextEntry->calls <= entry->calls)
1181                         break;
1182
1183                 prevEntry = nextEntry;
1184         }
1185
1186         if (!prevEntry)
1187                 TAILQ_INSERT_HEAD(&parentKext->userClientCallList, entry, link);
1188         else
1189                 TAILQ_INSERT_AFTER(&parentKext->userClientCallList, prevEntry, entry, link);
1190
1191 err_unlock:
1192         IORWLockUnlock(lock);
1193
1194 err_nounlock:
1195         if (obj)
1196                 obj->release();
1197 }
1198
1199 void IOStatistics::countUserClientCall(IOUserClient *client) {
1200         IOUserClient::ExpansionData *data;
1201         IOUserClientCounter *counter;
1202
1203         /* Guard against an uninitialized client object - <rdar://problem/8577946> */
1204         if (!(data = client->reserved)) {
1205                 return;
1206         }
1207
1208         if ((counter = data->counter)) {
1209                 storeUserClientCallInfo(client, counter);
1210                 OSIncrementAtomic(&counter->clientCalls);
1211         }
1212 }
1213
1214 KextNode *IOStatistics::getKextNodeFromBacktrace(boolean_t write) {
1215         const uint32_t btMin = 3;
1216
1217         void *bt[16];
1218         unsigned btCount = sizeof(bt) / sizeof(bt[0]);
1219         vm_offset_t *scanAddr = NULL;
1220         uint32_t i;
1221         KextNode *found = NULL, *ke = NULL;
1222
1223         /*
1224          * Gathering the backtrace is a significant source of
1225          * overhead. OSBacktrace does many safety checks that
1226          * are not needed in this situation.
1227          */
1228         btCount = backtrace((uintptr_t*)bt, btCount);
1229
1230         if (write) {
1231                 IORWLockWrite(lock);
1232         } else {
1233                 IORWLockRead(lock);
1234         }
1235
1236         /* Ignore first levels */
1237         scanAddr = (vm_offset_t *)&bt[btMin - 1];
1238
1239         for (i = btMin - 1; i < btCount; i++, scanAddr++) {
1240                 ke = RB_ROOT(&kextAddressHead);
1241                 while (ke) {
1242                         if (*scanAddr < ke->address) {
1243                                 ke = RB_LEFT(ke, addressLink);
1244                         }
1245                         else {
1246                                 if ((*scanAddr < ke->address_end) && (*scanAddr >= ke->address)) {
1247                                         if (!ke->kext->isKernelComponent()) {
1248                                                 return ke;
1249                                         } else {
1250                                                 found = ke;
1251                                         }
1252                                 }
1253                                 ke = RB_RIGHT(ke, addressLink);
1254                         }
1255                 }
1256         }
1257
1258         if (!found) {
1259                 IORWLockUnlock(lock);
1260         }
1261
1262         return found;
1263 }
1264
1265 void IOStatistics::releaseKextNode(KextNode *node) {
1266 #pragma unused(node)
1267         IORWLockUnlock(lock);
1268 }
1269
1270 /* IOLib allocations */
1271 void IOStatistics::countAlloc(uint32_t index, vm_size_t size) {
1272         KextNode *ke;
1273
1274         if (!enabled) {
1275                 return;
1276         }
1277
1278         ke = getKextNodeFromBacktrace(FALSE);
1279         if (ke) {
1280                 OSAddAtomic(size, &ke->memoryCounters[index]);
1281                 releaseKextNode(ke);
1282         }
1283 }
1284
1285 #endif /* IOKITSTATS */