libkern/c++/OSRuntime.cpp

   1 /*
   2  * Copyright (c) 2000,2008-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * Copyright (c) 1997 Apple Inc.
  30  *
  31  */
  32 #include <libkern/c++/OSMetaClass.h>
  33 #include <libkern/c++/OSKext.h>
  34 #include <libkern/c++/OSLib.h>
  35 #include <libkern/c++/OSSymbol.h>
  36 #include <IOKit/IOKitDebug.h>
  37
  38 #include <sys/cdefs.h>
  39 #if defined(HAS_APPLE_PAC)
  40 #include <ptrauth.h>
  41 #define PTRAUTH_STRIP_STRUCTOR(x)       ((uintptr_t) ptrauth_strip(ptrauth_nop_cast(void *, (x)), ptrauth_key_function_pointer))
  42 #else  /* defined(HAS_APPLE_PAC) */
  43 #define PTRAUTH_STRIP_STRUCTOR(x)       ((uintptr_t) (x))
  44 #endif /* !defined(HAS_APPLE_PAC) */
  45
  46 __BEGIN_DECLS
  47
  48 #include <string.h>
  49 #include <mach/mach_types.h>
  50 #include <libkern/kernel_mach_header.h>
  51 #include <libkern/prelink.h>
  52 #include <stdarg.h>
  53
  54 #if KASAN
  55 #include <san/kasan.h>
  56 #endif
  57
  58 #if PRAGMA_MARK
  59 #pragma mark Constants &c.
  60 #endif /* PRAGMA_MARK */
  61 OSKextLogSpec kOSRuntimeLogSpec =
  62     kOSKextLogErrorLevel |
  63     kOSKextLogLoadFlag |
  64     kOSKextLogKextBookkeepingFlag;
  65
  66 #if PRAGMA_MARK
  67 #pragma mark Logging Bootstrap
  68 #endif /* PRAGMA_MARK */
  69 /*********************************************************************
  70 * kern_os Logging Bootstrap
  71 *
  72 * We can't call in to OSKext until the kernel's C++ environment is up
  73 * and running, so let's mask those references with a check variable.
  74 * We print unconditionally if C++ isn't up, but if that's the case
  75 * we've generally hit a serious error in kernel init!
  76 *********************************************************************/
  77 static bool gKernelCPPInitialized = false;
  78
  79 #define OSRuntimeLog(kext, flags, format, args ...)            \
  80     do {                                                      \
  81         if (gKernelCPPInitialized) {                          \
  82             OSKextLog((kext), (flags), (format), ## args);  \
  83         } else {                                              \
  84             printf((format), ## args);                        \
  85         }                                                     \
  86     } while (0)
  87
  88 #if PRAGMA_MARK
  89 #pragma mark Libkern Init
  90 #endif /* PRAGMA_MARK */
  91 /*********************************************************************
  92 * Libkern Init
  93 *********************************************************************/
  94
  95 #if __GNUC__ >= 3
  96 void __dead2
  97 __cxa_pure_virtual( void )
  98 {
  99         panic("%s", __FUNCTION__);
 100 }
 101 #else
 102 void __dead2
 103 __pure_virtual( void )
 104 {
 105         panic("%s", __FUNCTION__);
 106 }
 107 #endif
 108
 109 extern lck_grp_t * IOLockGroup;
 110 extern kmod_info_t g_kernel_kmod_info;
 111
 112 enum {
 113         kOSSectionNamesDefault     = 0,
 114         kOSSectionNamesBuiltinKext = 1,
 115         kOSSectionNamesCount       = 2,
 116 };
 117 enum {
 118         kOSSectionNameInitializer = 0,
 119         kOSSectionNameFinalizer   = 1,
 120         kOSSectionNameCount       = 2
 121 };
 122
 123 static const char *
 124     gOSStructorSectionNames[kOSSectionNamesCount][kOSSectionNameCount] = {
 125         { SECT_MODINITFUNC, SECT_MODTERMFUNC },
 126         { kBuiltinInitSection, kBuiltinTermSection }
 127 };
 128
 129 void
 130 OSlibkernInit(void)
 131 {
 132         // This must be called before calling OSRuntimeInitializeCPP.
 133         OSMetaClassBase::initialize();
 134
 135         g_kernel_kmod_info.address = (vm_address_t) &_mh_execute_header;
 136         if (kOSReturnSuccess != OSRuntimeInitializeCPP(NULL)) {
 137                 // &g_kernel_kmod_info, gOSSectionNamesStandard, 0, 0)) {
 138                 panic("OSRuntime: C++ runtime failed to initialize.");
 139         }
 140
 141         gKernelCPPInitialized = true;
 142
 143         return;
 144 }
 145
 146 __END_DECLS
 147
 148 #if PRAGMA_MARK
 149 #pragma mark C++ Runtime Load/Unload
 150 #endif /* PRAGMA_MARK */
 151 /*********************************************************************
 152 * kern_os C++ Runtime Load/Unload
 153 *********************************************************************/
 154
 155 typedef void (*structor_t)(void);
 156
 157 static bool
 158 OSRuntimeCallStructorsInSection(
 159         OSKext                   * theKext,
 160         kmod_info_t              * kmodInfo,
 161         void                     * metaHandle,
 162         kernel_segment_command_t * segment,
 163         const char               * sectionName,
 164         uintptr_t                  textStart,
 165         uintptr_t                  textEnd)
 166 {
 167         kernel_section_t * section;
 168         bool result = TRUE;
 169
 170         for (section = firstsect(segment);
 171             section != NULL;
 172             section = nextsect(segment, section)) {
 173                 if (strncmp(section->sectname, sectionName, sizeof(section->sectname) - 1)) {
 174                         continue;
 175                 }
 176                 if (section->size == 0) {
 177                         continue;
 178                 }
 179
 180                 structor_t * structors = (structor_t *)section->addr;
 181                 if (!structors) {
 182                         continue;
 183                 }
 184
 185                 structor_t structor;
 186                 uintptr_t value;
 187                 unsigned long num_structors = section->size / sizeof(structor_t);
 188                 unsigned int hit_null_structor = 0;
 189                 unsigned long firstIndex = 0;
 190
 191                 if (textStart) {
 192                         // bsearch for any in range
 193                         unsigned long baseIdx;
 194                         unsigned long lim;
 195                         firstIndex = num_structors;
 196                         for (lim = num_structors, baseIdx = 0; lim; lim >>= 1) {
 197                                 structor = structors[baseIdx + (lim >> 1)];
 198                                 if (!structor) {
 199                                         panic("%s: null structor", kmodInfo->name);
 200                                 }
 201                                 value = PTRAUTH_STRIP_STRUCTOR(structor);
 202                                 if ((value >= textStart) && (value < textEnd)) {
 203                                         firstIndex = (baseIdx + (lim >> 1));
 204                                         // scan back for the first in range
 205                                         for (; firstIndex; firstIndex--) {
 206                                                 structor = structors[firstIndex - 1];
 207                                                 value = PTRAUTH_STRIP_STRUCTOR(structor);
 208                                                 if ((value < textStart) || (value >= textEnd)) {
 209                                                         break;
 210                                                 }
 211                                         }
 212                                         break;
 213                                 }
 214                                 if (textStart > value) {
 215                                         // move right
 216                                         baseIdx += (lim >> 1) + 1;
 217                                         lim--;
 218                                 }
 219                                 // else move left
 220                         }
 221                         baseIdx = (baseIdx + (lim >> 1));
 222                 }
 223                 for (;
 224                     (firstIndex < num_structors)
 225                     && (!metaHandle || OSMetaClass::checkModLoad(metaHandle));
 226                     firstIndex++) {
 227                         if ((structor = structors[firstIndex])) {
 228                                 value = PTRAUTH_STRIP_STRUCTOR(structor);
 229                                 if ((textStart && (value < textStart))
 230                                     || (textEnd && (value >= textEnd))) {
 231                                         break;
 232                                 }
 233                                 (*structor)();
 234                         } else if (!hit_null_structor) {
 235                                 hit_null_structor = 1;
 236                                 OSRuntimeLog(theKext, kOSRuntimeLogSpec,
 237                                     "Null structor in kext %s segment %s!",
 238                                     kmodInfo->name, section->segname);
 239                         }
 240                 }
 241                 if (metaHandle) {
 242                         result = OSMetaClass::checkModLoad(metaHandle);
 243                 }
 244                 break;
 245         } /* for (section...) */
 246         return result;
 247 }
 248
 249 /*********************************************************************
 250 *********************************************************************/
 251 kern_return_t
 252 OSRuntimeFinalizeCPP(
 253         OSKext                   * theKext)
 254 {
 255         kern_return_t              result = KMOD_RETURN_FAILURE;
 256         void                     * metaHandle = NULL;// do not free
 257         kernel_mach_header_t     * header;
 258         kernel_segment_command_t * segment;
 259         kmod_info_t              * kmodInfo;
 260         const char              ** sectionNames;
 261         uintptr_t                  textStart;
 262         uintptr_t                  textEnd;
 263
 264         textStart    = 0;
 265         textEnd      = 0;
 266         sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault];
 267         if (theKext) {
 268                 if (!theKext->isCPPInitialized()) {
 269                         result = KMOD_RETURN_SUCCESS;
 270                         goto finish;
 271                 }
 272                 kmodInfo = theKext->kmod_info;
 273                 if (!kmodInfo || !kmodInfo->address) {
 274                         result = kOSKextReturnInvalidArgument;
 275                         goto finish;
 276                 }
 277                 header = (kernel_mach_header_t *)kmodInfo->address;
 278                 if (theKext->flags.builtin) {
 279                         header       = (kernel_mach_header_t *)g_kernel_kmod_info.address;
 280                         textStart    = kmodInfo->address;
 281                         textEnd      = textStart + kmodInfo->size;
 282                         sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext];
 283                 }
 284         } else {
 285                 kmodInfo = &g_kernel_kmod_info;
 286                 header   = (kernel_mach_header_t *)kmodInfo->address;
 287         }
 288
 289         /* OSKext checks for this condition now, but somebody might call
 290          * this function directly (the symbol is exported....).
 291          */
 292         if (OSMetaClass::modHasInstance(kmodInfo->name)) {
 293                 // xxx - Don't log under errors? this is more of an info thing
 294                 OSRuntimeLog(theKext, kOSRuntimeLogSpec,
 295                     "Can't tear down kext %s C++; classes have instances:",
 296                     kmodInfo->name);
 297                 OSKext::reportOSMetaClassInstances(kmodInfo->name, kOSRuntimeLogSpec);
 298                 result = kOSMetaClassHasInstances;
 299                 goto finish;
 300         }
 301
 302         /* Tell the meta class system that we are starting to unload.
 303          * metaHandle isn't actually needed on the finalize path,
 304          * so we don't check it here, even though OSMetaClass::postModLoad() will
 305          * return a failure (it only does actual work on the init path anyhow).
 306          */
 307         metaHandle = OSMetaClass::preModLoad(kmodInfo->name);
 308
 309         OSSymbol::checkForPageUnload((void *)kmodInfo->address,
 310             (void *)(kmodInfo->address + kmodInfo->size));
 311
 312         header = (kernel_mach_header_t *)kmodInfo->address;
 313         segment = firstsegfromheader(header);
 314
 315         for (segment = firstsegfromheader(header);
 316             segment != NULL;
 317             segment = nextsegfromheader(header, segment)) {
 318                 OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment,
 319                     sectionNames[kOSSectionNameFinalizer], textStart, textEnd);
 320         }
 321
 322         (void)OSMetaClass::postModLoad(metaHandle);
 323
 324         if (theKext) {
 325                 theKext->setCPPInitialized(false);
 326         }
 327         result = KMOD_RETURN_SUCCESS;
 328 finish:
 329         return result;
 330 }
 331
 332 #if defined(HAS_APPLE_PAC)
 333 #if !KASAN
 334 /*
 335  * Place this function in __KLD,__text on non-kasan builds so it gets unmapped
 336  * after CTRR lockdown.
 337  */
 338 __attribute__((noinline, section("__KLD,__text")))
 339 #endif
 340 static void
 341 OSRuntimeSignStructorsInSegment(kernel_segment_command_t *segment)
 342 {
 343         kernel_section_t         * section;
 344         structor_t               * structors;
 345         volatile structor_t                 structor;
 346         size_t                     idx, num_structors;
 347
 348         for (section = firstsect(segment);
 349             section != NULL;
 350             section = nextsect(segment, section)) {
 351                 if ((S_MOD_INIT_FUNC_POINTERS != (SECTION_TYPE & section->flags))
 352                     && (S_MOD_TERM_FUNC_POINTERS != (SECTION_TYPE & section->flags))) {
 353                         continue;
 354                 }
 355                 structors = (structor_t *)section->addr;
 356                 if (!structors) {
 357                         continue;
 358                 }
 359                 num_structors = section->size / sizeof(structor_t);
 360                 for (idx = 0; idx < num_structors; idx++) {
 361                         structor = structors[idx];
 362                         if (NULL == structor) {
 363                                 continue;
 364                         }
 365                         structor = ptrauth_strip(structor, ptrauth_key_function_pointer);
 366                         structor = ptrauth_sign_unauthenticated(structor, ptrauth_key_function_pointer, ptrauth_function_pointer_type_discriminator(void (*)(void)));
 367                         structors[idx] = structor;
 368                 }
 369         } /* for (section...) */
 370 }
 371 #endif
 372
 373 /*********************************************************************
 374 *********************************************************************/
 375 void
 376 OSRuntimeSignStructors(
 377         kernel_mach_header_t * header __unused)
 378 {
 379 #if defined(HAS_APPLE_PAC)
 380
 381         kernel_segment_command_t * segment;
 382
 383         for (segment = firstsegfromheader(header);
 384             segment != NULL;
 385             segment = nextsegfromheader(header, segment)) {
 386                 OSRuntimeSignStructorsInSegment(segment);
 387         } /* for (segment...) */
 388 #endif /* !defined(XXX) && defined(HAS_APPLE_PAC) */
 389 }
 390
 391 /*********************************************************************
 392 *********************************************************************/
 393 void
 394 OSRuntimeSignStructorsInFileset(
 395         kernel_mach_header_t * fileset_header __unused)
 396 {
 397 #if defined(HAS_APPLE_PAC)
 398         struct load_command *lc;
 399
 400         lc = (struct load_command *)((uintptr_t)fileset_header + sizeof(*fileset_header));
 401         for (uint32_t i = 0; i < fileset_header->ncmds; i++,
 402             lc = (struct load_command *)((uintptr_t)lc + lc->cmdsize)) {
 403                 if (lc->cmd == LC_FILESET_ENTRY) {
 404                         struct fileset_entry_command *fse;
 405                         kernel_mach_header_t *mh;
 406
 407                         fse = (struct fileset_entry_command *)(uintptr_t)lc;
 408                         mh = (kernel_mach_header_t *)((uintptr_t)fse->vmaddr);
 409                         OSRuntimeSignStructors(mh);
 410                 } else if (lc->cmd == LC_SEGMENT_64) {
 411                         /*
 412                          * Slide/adjust all LC_SEGMENT_64 commands in the fileset
 413                          * (and any sections in those segments)
 414                          */
 415                         kernel_segment_command_t *seg;
 416                         seg = (kernel_segment_command_t *)(uintptr_t)lc;
 417                         OSRuntimeSignStructorsInSegment(seg);
 418                 }
 419         }
 420
 421 #endif /* defined(HAS_APPLE_PAC) */
 422 }
 423
 424 /*********************************************************************
 425 *********************************************************************/
 426 kern_return_t
 427 OSRuntimeInitializeCPP(
 428         OSKext                   * theKext)
 429 {
 430         kern_return_t              result          = KMOD_RETURN_FAILURE;
 431         kernel_mach_header_t     * header          = NULL;
 432         void                     * metaHandle      = NULL;// do not free
 433         bool                       load_success    = true;
 434         kernel_segment_command_t * segment         = NULL;// do not free
 435         kernel_segment_command_t * failure_segment = NULL; // do not free
 436         kmod_info_t             *  kmodInfo;
 437         const char              ** sectionNames;
 438         uintptr_t                  textStart;
 439         uintptr_t                  textEnd;
 440
 441         textStart    = 0;
 442         textEnd      = 0;
 443         sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault];
 444         if (theKext) {
 445                 if (theKext->isCPPInitialized()) {
 446                         result = KMOD_RETURN_SUCCESS;
 447                         goto finish;
 448                 }
 449
 450                 kmodInfo = theKext->kmod_info;
 451                 if (!kmodInfo || !kmodInfo->address) {
 452                         result = kOSKextReturnInvalidArgument;
 453                         goto finish;
 454                 }
 455                 header = (kernel_mach_header_t *)kmodInfo->address;
 456
 457                 if (theKext->flags.builtin) {
 458                         header       = (kernel_mach_header_t *)g_kernel_kmod_info.address;
 459                         textStart    = kmodInfo->address;
 460                         textEnd      = textStart + kmodInfo->size;
 461                         sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext];
 462                 }
 463         } else {
 464                 kmodInfo = &g_kernel_kmod_info;
 465                 header   = (kernel_mach_header_t *)kmodInfo->address;
 466         }
 467
 468         /* Tell the meta class system that we are starting the load
 469          */
 470         metaHandle = OSMetaClass::preModLoad(kmodInfo->name);
 471         assert(metaHandle);
 472         if (!metaHandle) {
 473                 goto finish;
 474         }
 475
 476         /* NO GOTO PAST HERE. */
 477
 478         /* Scan the header for all constructor sections, in any
 479          * segment, and invoke the constructors within those sections.
 480          */
 481         for (segment = firstsegfromheader(header);
 482             segment != NULL && load_success;
 483             segment = nextsegfromheader(header, segment)) {
 484                 /* Record the current segment in the event of a failure.
 485                  */
 486                 failure_segment = segment;
 487                 load_success = OSRuntimeCallStructorsInSection(
 488                         theKext, kmodInfo, metaHandle, segment,
 489                         sectionNames[kOSSectionNameInitializer],
 490                         textStart, textEnd);
 491         } /* for (segment...) */
 492
 493         /* We failed so call all of the destructors. We must do this before
 494          * calling OSMetaClass::postModLoad() as the OSMetaClass destructors
 495          * will alter state (in the metaHandle) used by that function.
 496          */
 497         if (!load_success) {
 498                 /* Scan the header for all destructor sections, in any
 499                  * segment, and invoke the constructors within those sections.
 500                  */
 501                 for (segment = firstsegfromheader(header);
 502                     segment != failure_segment && segment != NULL;
 503                     segment = nextsegfromheader(header, segment)) {
 504                         OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment,
 505                             sectionNames[kOSSectionNameFinalizer], textStart, textEnd);
 506                 } /* for (segment...) */
 507         }
 508
 509         /* Now, regardless of success so far, do the post-init registration
 510          * and cleanup. If we had to call the unloadCPP function, static
 511          * destructors have removed classes from the stalled list so no
 512          * metaclasses will actually be registered.
 513          */
 514         result = OSMetaClass::postModLoad(metaHandle);
 515
 516         /* If we've otherwise been fine up to now, but OSMetaClass::postModLoad()
 517          * fails (typically due to a duplicate class), tear down all the C++
 518          * stuff from the kext. This isn't necessary for libkern/OSMetaClass stuff,
 519          * but may be necessary for other C++ code. We ignore the return value
 520          * because it's only a fail when there are existing instances of libkern
 521          * classes, and there had better not be any created on the C++ init path.
 522          */
 523         if (load_success && result != KMOD_RETURN_SUCCESS) {
 524                 (void)OSRuntimeFinalizeCPP(theKext); //kmodInfo, sectionNames, textStart, textEnd);
 525         }
 526
 527         if (theKext && load_success && result == KMOD_RETURN_SUCCESS) {
 528                 theKext->setCPPInitialized(true);
 529         }
 530 finish:
 531         return result;
 532 }
 533
 534 /*********************************************************************
 535 *   Unload a kernel segment.
 536 *********************************************************************/
 537
 538 void
 539 OSRuntimeUnloadCPPForSegment(
 540         kernel_segment_command_t * segment)
 541 {
 542         OSRuntimeCallStructorsInSection(NULL, &g_kernel_kmod_info, NULL, segment,
 543             gOSStructorSectionNames[kOSSectionNamesDefault][kOSSectionNameFinalizer], 0, 0);
 544 }
 545
 546 #if PRAGMA_MARK
 547 #pragma mark C++ Allocators & Deallocators
 548 #endif /* PRAGMA_MARK */
 549 /*********************************************************************
 550 * C++ Allocators & Deallocators
 551 *********************************************************************/
 552 void *
 553 operator new(size_t size)
 554 {
 555         assert(size);
 556         return kheap_alloc_tag_bt(KERN_OS_MALLOC, size,
 557                    (zalloc_flags_t) (Z_WAITOK | Z_ZERO), VM_KERN_MEMORY_LIBKERN);
 558 }
 559
 560 void
 561 operator delete(void * addr)
 562 #if __cplusplus >= 201103L
 563 noexcept
 564 #endif
 565 {
 566         kheap_free_addr(KERN_OS_MALLOC, addr);
 567         return;
 568 }
 569
 570 void *
 571 operator new[](unsigned long sz)
 572 {
 573         return kheap_alloc_tag_bt(KERN_OS_MALLOC, sz,
 574                    (zalloc_flags_t) (Z_WAITOK | Z_ZERO), VM_KERN_MEMORY_LIBKERN);
 575 }
 576
 577 void
 578 operator delete[](void * ptr)
 579 #if __cplusplus >= 201103L
 580 noexcept
 581 #endif
 582 {
 583         if (ptr) {
 584 #if KASAN
 585                 /*
 586                  * Unpoison the C++ array cookie inserted (but not removed) by the
 587                  * compiler on new[].
 588                  */
 589                 kasan_unpoison_cxx_array_cookie(ptr);
 590 #endif
 591                 kheap_free_addr(KERN_OS_MALLOC, ptr);
 592         }
 593         return;
 594 }
 595
 596 #if __cplusplus >= 201103L
 597
 598 void
 599 operator delete(void * addr, size_t sz) noexcept
 600 {
 601         kheap_free(KERN_OS_MALLOC, addr, sz);
 602 }
 603
 604 void
 605 operator delete[](void * addr, size_t sz) noexcept
 606 {
 607         if (addr) {
 608                 kheap_free(KERN_OS_MALLOC, addr, sz);
 609         }
 610 }
 611
 612 #endif /* __cplusplus >= 201103L */
 613
 614 /* PR-6481964 - The compiler is going to check for size overflows in calls to
 615  * new[], and if there is an overflow, it will call __throw_length_error.
 616  * This is an unrecoverable error by the C++ standard, so we must panic here.
 617  *
 618  * We have to put the function inside the std namespace because of how the
 619  * compiler expects the name to be mangled.
 620  */
 621 namespace std {
 622 void __dead2
 623 __throw_length_error(const char *msg __unused)
 624 {
 625         panic("Size of array created by new[] has overflowed");
 626 }
 627 };