libkern/c++/OSRuntime.cpp

   1 /*
   2  * Copyright (c) 2000,2008-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * Copyright (c) 1997 Apple Inc.
  30  *
  31  */
  32 #include <libkern/c++/OSMetaClass.h>
  33 #include <libkern/c++/OSKext.h>
  34 #include <libkern/c++/OSLib.h>
  35 #include <libkern/c++/OSSymbol.h>
  36 #include <IOKit/IOKitDebug.h>
  37
  38 #include <sys/cdefs.h>
  39 #if defined(HAS_APPLE_PAC)
  40 #include <ptrauth.h>
  41 #define PTRAUTH_STRIP_STRUCTOR(x)       ((uintptr_t) ptrauth_strip(ptrauth_nop_cast(void *, (x)), ptrauth_key_function_pointer))
  42 #else  /* defined(HAS_APPLE_PAC) */
  43 #define PTRAUTH_STRIP_STRUCTOR(x)       ((uintptr_t) (x))
  44 #endif /* !defined(HAS_APPLE_PAC) */
  45
  46 __BEGIN_DECLS
  47
  48 #include <string.h>
  49 #include <mach/mach_types.h>
  50 #include <libkern/kernel_mach_header.h>
  51 #include <libkern/prelink.h>
  52 #include <stdarg.h>
  53
  54 #if KASAN
  55 #include <san/kasan.h>
  56 #endif
  57
  58 #if PRAGMA_MARK
  59 #pragma mark Constants &c.
  60 #endif /* PRAGMA_MARK */
  61 OSKextLogSpec kOSRuntimeLogSpec =
  62     kOSKextLogErrorLevel |
  63     kOSKextLogLoadFlag |
  64     kOSKextLogKextBookkeepingFlag;
  65
  66 #if PRAGMA_MARK
  67 #pragma mark Logging Bootstrap
  68 #endif /* PRAGMA_MARK */
  69 /*********************************************************************
  70 * kern_os Logging Bootstrap
  71 *
  72 * We can't call in to OSKext until the kernel's C++ environment is up
  73 * and running, so let's mask those references with a check variable.
  74 * We print unconditionally if C++ isn't up, but if that's the case
  75 * we've generally hit a serious error in kernel init!
  76 *********************************************************************/
  77 static bool gKernelCPPInitialized = false;
  78
  79 #define OSRuntimeLog(kext, flags, format, args ...)            \
  80     do {                                                      \
  81         if (gKernelCPPInitialized) {                          \
  82             OSKextLog((kext), (flags), (format), ## args);  \
  83         } else {                                              \
  84             printf((format), ## args);                        \
  85         }                                                     \
  86     } while (0)
  87
  88 #if PRAGMA_MARK
  89 #pragma mark Libkern Init
  90 #endif /* PRAGMA_MARK */
  91 /*********************************************************************
  92 * Libkern Init
  93 *********************************************************************/
  94
  95 #if __GNUC__ >= 3
  96 void __dead2
  97 __cxa_pure_virtual( void )
  98 {
  99         panic("%s", __FUNCTION__);
 100 }
 101 #else
 102 void __dead2
 103 __pure_virtual( void )
 104 {
 105         panic("%s", __FUNCTION__);
 106 }
 107 #endif
 108
 109 extern lck_grp_t * IOLockGroup;
 110 extern kmod_info_t g_kernel_kmod_info;
 111
 112 enum {
 113         kOSSectionNamesDefault     = 0,
 114         kOSSectionNamesBuiltinKext = 1,
 115         kOSSectionNamesCount       = 2,
 116 };
 117 enum {
 118         kOSSectionNameInitializer = 0,
 119         kOSSectionNameFinalizer   = 1,
 120         kOSSectionNameCount       = 2
 121 };
 122
 123 static const char *
 124     gOSStructorSectionNames[kOSSectionNamesCount][kOSSectionNameCount] = {
 125         { SECT_MODINITFUNC, SECT_MODTERMFUNC },
 126         { kBuiltinInitSection, kBuiltinTermSection }
 127 };
 128
 129 void
 130 OSlibkernInit(void)
 131 {
 132         // This must be called before calling OSRuntimeInitializeCPP.
 133         OSMetaClassBase::initialize();
 134
 135         g_kernel_kmod_info.address = (vm_address_t) &_mh_execute_header;
 136         if (kOSReturnSuccess != OSRuntimeInitializeCPP(NULL)) {
 137                 // &g_kernel_kmod_info, gOSSectionNamesStandard, 0, 0)) {
 138                 panic("OSRuntime: C++ runtime failed to initialize.");
 139         }
 140
 141         gKernelCPPInitialized = true;
 142
 143         return;
 144 }
 145
 146 __END_DECLS
 147
 148 #if PRAGMA_MARK
 149 #pragma mark C++ Runtime Load/Unload
 150 #endif /* PRAGMA_MARK */
 151 /*********************************************************************
 152 * kern_os C++ Runtime Load/Unload
 153 *********************************************************************/
 154
 155 typedef void (*structor_t)(void);
 156
 157 static bool
 158 OSRuntimeCallStructorsInSection(
 159         OSKext                   * theKext,
 160         kmod_info_t              * kmodInfo,
 161         void                     * metaHandle,
 162         kernel_segment_command_t * segment,
 163         const char               * sectionName,
 164         uintptr_t                  textStart,
 165         uintptr_t                  textEnd)
 166 {
 167         kernel_section_t * section;
 168         bool result = TRUE;
 169
 170         for (section = firstsect(segment);
 171             section != NULL;
 172             section = nextsect(segment, section)) {
 173                 if (strncmp(section->sectname, sectionName, sizeof(section->sectname) - 1)) {
 174                         continue;
 175                 }
 176                 if (section->size == 0) {
 177                         continue;
 178                 }
 179
 180                 structor_t * structors = (structor_t *)section->addr;
 181                 if (!structors) {
 182                         continue;
 183                 }
 184
 185                 structor_t structor;
 186                 uintptr_t value;
 187                 unsigned long num_structors = section->size / sizeof(structor_t);
 188                 unsigned int hit_null_structor = 0;
 189                 unsigned long firstIndex = 0;
 190
 191                 if (textStart) {
 192                         // bsearch for any in range
 193                         unsigned long baseIdx;
 194                         unsigned long lim;
 195                         firstIndex = num_structors;
 196                         for (lim = num_structors, baseIdx = 0; lim; lim >>= 1) {
 197                                 structor = structors[baseIdx + (lim >> 1)];
 198                                 if (!structor) {
 199                                         panic("%s: null structor", kmodInfo->name);
 200                                 }
 201                                 value = PTRAUTH_STRIP_STRUCTOR(structor);
 202                                 if ((value >= textStart) && (value < textEnd)) {
 203                                         firstIndex = (baseIdx + (lim >> 1));
 204                                         // scan back for the first in range
 205                                         for (; firstIndex; firstIndex--) {
 206                                                 structor = structors[firstIndex - 1];
 207                                                 value = PTRAUTH_STRIP_STRUCTOR(structor);
 208                                                 if ((value < textStart) || (value >= textEnd)) {
 209                                                         break;
 210                                                 }
 211                                         }
 212                                         break;
 213                                 }
 214                                 if (textStart > value) {
 215                                         // move right
 216                                         baseIdx += (lim >> 1) + 1;
 217                                         lim--;
 218                                 }
 219                                 // else move left
 220                         }
 221                         baseIdx = (baseIdx + (lim >> 1));
 222                 }
 223                 for (;
 224                     (firstIndex < num_structors)
 225                     && (!metaHandle || OSMetaClass::checkModLoad(metaHandle));
 226                     firstIndex++) {
 227                         if ((structor = structors[firstIndex])) {
 228                                 value = PTRAUTH_STRIP_STRUCTOR(structor);
 229                                 if ((textStart && (value < textStart))
 230                                     || (textEnd && (value >= textEnd))) {
 231                                         break;
 232                                 }
 233                                 (*structor)();
 234                         } else if (!hit_null_structor) {
 235                                 hit_null_structor = 1;
 236                                 OSRuntimeLog(theKext, kOSRuntimeLogSpec,
 237                                     "Null structor in kext %s segment %s!",
 238                                     kmodInfo->name, section->segname);
 239                         }
 240                 }
 241                 if (metaHandle) {
 242                         result = OSMetaClass::checkModLoad(metaHandle);
 243                 }
 244                 break;
 245         } /* for (section...) */
 246         return result;
 247 }
 248
 249 /*********************************************************************
 250 *********************************************************************/
 251 kern_return_t
 252 OSRuntimeFinalizeCPP(
 253         OSKext                   * theKext)
 254 {
 255         kern_return_t              result = KMOD_RETURN_FAILURE;
 256         void                     * metaHandle = NULL;// do not free
 257         kernel_mach_header_t     * header;
 258         kernel_segment_command_t * segment;
 259         kmod_info_t              * kmodInfo;
 260         const char              ** sectionNames;
 261         uintptr_t                  textStart;
 262         uintptr_t                  textEnd;
 263
 264         textStart    = 0;
 265         textEnd      = 0;
 266         sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault];
 267         if (theKext) {
 268                 if (!theKext->isCPPInitialized()) {
 269                         result = KMOD_RETURN_SUCCESS;
 270                         goto finish;
 271                 }
 272                 kmodInfo = theKext->kmod_info;
 273                 if (!kmodInfo || !kmodInfo->address) {
 274                         result = kOSKextReturnInvalidArgument;
 275                         goto finish;
 276                 }
 277                 header = (kernel_mach_header_t *)kmodInfo->address;
 278                 if (theKext->flags.builtin) {
 279                         header       = (kernel_mach_header_t *)g_kernel_kmod_info.address;
 280                         textStart    = kmodInfo->address;
 281                         textEnd      = textStart + kmodInfo->size;
 282                         sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext];
 283                 }
 284         } else {
 285                 kmodInfo = &g_kernel_kmod_info;
 286                 header   = (kernel_mach_header_t *)kmodInfo->address;
 287         }
 288
 289         /* OSKext checks for this condition now, but somebody might call
 290          * this function directly (the symbol is exported....).
 291          */
 292         if (OSMetaClass::modHasInstance(kmodInfo->name)) {
 293                 // xxx - Don't log under errors? this is more of an info thing
 294                 OSRuntimeLog(theKext, kOSRuntimeLogSpec,
 295                     "Can't tear down kext %s C++; classes have instances:",
 296                     kmodInfo->name);
 297                 OSKext::reportOSMetaClassInstances(kmodInfo->name, kOSRuntimeLogSpec);
 298                 result = kOSMetaClassHasInstances;
 299                 goto finish;
 300         }
 301
 302         /* Tell the meta class system that we are starting to unload.
 303          * metaHandle isn't actually needed on the finalize path,
 304          * so we don't check it here, even though OSMetaClass::postModLoad() will
 305          * return a failure (it only does actual work on the init path anyhow).
 306          */
 307         metaHandle = OSMetaClass::preModLoad(kmodInfo->name);
 308
 309         OSSymbol::checkForPageUnload((void *)kmodInfo->address,
 310             (void *)(kmodInfo->address + kmodInfo->size));
 311
 312         header = (kernel_mach_header_t *)kmodInfo->address;
 313         segment = firstsegfromheader(header);
 314
 315         for (segment = firstsegfromheader(header);
 316             segment != NULL;
 317             segment = nextsegfromheader(header, segment)) {
 318                 OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment,
 319                     sectionNames[kOSSectionNameFinalizer], textStart, textEnd);
 320         }
 321
 322         (void)OSMetaClass::postModLoad(metaHandle);
 323
 324         if (theKext) {
 325                 theKext->setCPPInitialized(false);
 326         }
 327         result = KMOD_RETURN_SUCCESS;
 328 finish:
 329         return result;
 330 }
 331
 332 #if defined(HAS_APPLE_PAC)
 333 static inline void
 334 OSRuntimeSignStructorsInSegment(kernel_segment_command_t *segment)
 335 {
 336         kernel_section_t         * section;
 337         structor_t               * structors;
 338         volatile structor_t                 structor;
 339         size_t                     idx, num_structors;
 340
 341         for (section = firstsect(segment);
 342             section != NULL;
 343             section = nextsect(segment, section)) {
 344                 if ((S_MOD_INIT_FUNC_POINTERS != (SECTION_TYPE & section->flags))
 345                     && (S_MOD_TERM_FUNC_POINTERS != (SECTION_TYPE & section->flags))) {
 346                         continue;
 347                 }
 348                 structors = (structor_t *)section->addr;
 349                 if (!structors) {
 350                         continue;
 351                 }
 352                 num_structors = section->size / sizeof(structor_t);
 353                 for (idx = 0; idx < num_structors; idx++) {
 354                         structor = structors[idx];
 355                         if (NULL == structor) {
 356                                 continue;
 357                         }
 358                         structor = ptrauth_strip(structor, ptrauth_key_function_pointer);
 359                         structor = ptrauth_sign_unauthenticated(structor, ptrauth_key_function_pointer, ptrauth_function_pointer_type_discriminator(void (*)(void)));
 360                         structors[idx] = structor;
 361                 }
 362         } /* for (section...) */
 363 }
 364 #endif
 365
 366 /*********************************************************************
 367 *********************************************************************/
 368 void
 369 OSRuntimeSignStructors(
 370         kernel_mach_header_t * header __unused)
 371 {
 372 #if defined(HAS_APPLE_PAC)
 373
 374         kernel_segment_command_t * segment;
 375
 376         for (segment = firstsegfromheader(header);
 377             segment != NULL;
 378             segment = nextsegfromheader(header, segment)) {
 379                 OSRuntimeSignStructorsInSegment(segment);
 380         } /* for (segment...) */
 381 #endif /* !defined(XXX) && defined(HAS_APPLE_PAC) */
 382 }
 383
 384 /*********************************************************************
 385 *********************************************************************/
 386 void
 387 OSRuntimeSignStructorsInFileset(
 388         kernel_mach_header_t * fileset_header __unused)
 389 {
 390 #if defined(HAS_APPLE_PAC)
 391         struct load_command *lc;
 392
 393         lc = (struct load_command *)((uintptr_t)fileset_header + sizeof(*fileset_header));
 394         for (uint32_t i = 0; i < fileset_header->ncmds; i++,
 395             lc = (struct load_command *)((uintptr_t)lc + lc->cmdsize)) {
 396                 if (lc->cmd == LC_FILESET_ENTRY) {
 397                         struct fileset_entry_command *fse;
 398                         kernel_mach_header_t *mh;
 399
 400                         fse = (struct fileset_entry_command *)(uintptr_t)lc;
 401                         mh = (kernel_mach_header_t *)((uintptr_t)fse->vmaddr);
 402                         OSRuntimeSignStructors(mh);
 403                 } else if (lc->cmd == LC_SEGMENT_64) {
 404                         /*
 405                          * Slide/adjust all LC_SEGMENT_64 commands in the fileset
 406                          * (and any sections in those segments)
 407                          */
 408                         kernel_segment_command_t *seg;
 409                         seg = (kernel_segment_command_t *)(uintptr_t)lc;
 410                         OSRuntimeSignStructorsInSegment(seg);
 411                 }
 412         }
 413
 414 #endif /* defined(HAS_APPLE_PAC) */
 415 }
 416
 417 /*********************************************************************
 418 *********************************************************************/
 419 kern_return_t
 420 OSRuntimeInitializeCPP(
 421         OSKext                   * theKext)
 422 {
 423         kern_return_t              result          = KMOD_RETURN_FAILURE;
 424         kernel_mach_header_t     * header          = NULL;
 425         void                     * metaHandle      = NULL;// do not free
 426         bool                       load_success    = true;
 427         kernel_segment_command_t * segment         = NULL;// do not free
 428         kernel_segment_command_t * failure_segment = NULL; // do not free
 429         kmod_info_t             *  kmodInfo;
 430         const char              ** sectionNames;
 431         uintptr_t                  textStart;
 432         uintptr_t                  textEnd;
 433
 434         textStart    = 0;
 435         textEnd      = 0;
 436         sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault];
 437         if (theKext) {
 438                 if (theKext->isCPPInitialized()) {
 439                         result = KMOD_RETURN_SUCCESS;
 440                         goto finish;
 441                 }
 442
 443                 kmodInfo = theKext->kmod_info;
 444                 if (!kmodInfo || !kmodInfo->address) {
 445                         result = kOSKextReturnInvalidArgument;
 446                         goto finish;
 447                 }
 448                 header = (kernel_mach_header_t *)kmodInfo->address;
 449
 450                 if (theKext->flags.builtin) {
 451                         header       = (kernel_mach_header_t *)g_kernel_kmod_info.address;
 452                         textStart    = kmodInfo->address;
 453                         textEnd      = textStart + kmodInfo->size;
 454                         sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext];
 455                 }
 456         } else {
 457                 kmodInfo = &g_kernel_kmod_info;
 458                 header   = (kernel_mach_header_t *)kmodInfo->address;
 459         }
 460
 461         /* Tell the meta class system that we are starting the load
 462          */
 463         metaHandle = OSMetaClass::preModLoad(kmodInfo->name);
 464         assert(metaHandle);
 465         if (!metaHandle) {
 466                 goto finish;
 467         }
 468
 469         /* NO GOTO PAST HERE. */
 470
 471         /* Scan the header for all constructor sections, in any
 472          * segment, and invoke the constructors within those sections.
 473          */
 474         for (segment = firstsegfromheader(header);
 475             segment != NULL && load_success;
 476             segment = nextsegfromheader(header, segment)) {
 477                 /* Record the current segment in the event of a failure.
 478                  */
 479                 failure_segment = segment;
 480                 load_success = OSRuntimeCallStructorsInSection(
 481                         theKext, kmodInfo, metaHandle, segment,
 482                         sectionNames[kOSSectionNameInitializer],
 483                         textStart, textEnd);
 484         } /* for (segment...) */
 485
 486         /* We failed so call all of the destructors. We must do this before
 487          * calling OSMetaClass::postModLoad() as the OSMetaClass destructors
 488          * will alter state (in the metaHandle) used by that function.
 489          */
 490         if (!load_success) {
 491                 /* Scan the header for all destructor sections, in any
 492                  * segment, and invoke the constructors within those sections.
 493                  */
 494                 for (segment = firstsegfromheader(header);
 495                     segment != failure_segment && segment != NULL;
 496                     segment = nextsegfromheader(header, segment)) {
 497                         OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment,
 498                             sectionNames[kOSSectionNameFinalizer], textStart, textEnd);
 499                 } /* for (segment...) */
 500         }
 501
 502         /* Now, regardless of success so far, do the post-init registration
 503          * and cleanup. If we had to call the unloadCPP function, static
 504          * destructors have removed classes from the stalled list so no
 505          * metaclasses will actually be registered.
 506          */
 507         result = OSMetaClass::postModLoad(metaHandle);
 508
 509         /* If we've otherwise been fine up to now, but OSMetaClass::postModLoad()
 510          * fails (typically due to a duplicate class), tear down all the C++
 511          * stuff from the kext. This isn't necessary for libkern/OSMetaClass stuff,
 512          * but may be necessary for other C++ code. We ignore the return value
 513          * because it's only a fail when there are existing instances of libkern
 514          * classes, and there had better not be any created on the C++ init path.
 515          */
 516         if (load_success && result != KMOD_RETURN_SUCCESS) {
 517                 (void)OSRuntimeFinalizeCPP(theKext); //kmodInfo, sectionNames, textStart, textEnd);
 518         }
 519
 520         if (theKext && load_success && result == KMOD_RETURN_SUCCESS) {
 521                 theKext->setCPPInitialized(true);
 522         }
 523 finish:
 524         return result;
 525 }
 526
 527 /*********************************************************************
 528 *   Unload a kernel segment.
 529 *********************************************************************/
 530
 531 void
 532 OSRuntimeUnloadCPPForSegment(
 533         kernel_segment_command_t * segment)
 534 {
 535         OSRuntimeCallStructorsInSection(NULL, &g_kernel_kmod_info, NULL, segment,
 536             gOSStructorSectionNames[kOSSectionNamesDefault][kOSSectionNameFinalizer], 0, 0);
 537 }
 538
 539 #if PRAGMA_MARK
 540 #pragma mark C++ Allocators & Deallocators
 541 #endif /* PRAGMA_MARK */
 542 /*********************************************************************
 543 * C++ Allocators & Deallocators
 544 *********************************************************************/
 545 void *
 546 operator new(size_t size)
 547 {
 548         assert(size);
 549         return kheap_alloc_tag_bt(KERN_OS_MALLOC, size,
 550                    (zalloc_flags_t) (Z_WAITOK | Z_ZERO), VM_KERN_MEMORY_LIBKERN);
 551 }
 552
 553 void
 554 operator delete(void * addr)
 555 #if __cplusplus >= 201103L
 556 noexcept
 557 #endif
 558 {
 559         kheap_free_addr(KERN_OS_MALLOC, addr);
 560         return;
 561 }
 562
 563 void *
 564 operator new[](unsigned long sz)
 565 {
 566         return kheap_alloc_tag_bt(KERN_OS_MALLOC, sz,
 567                    (zalloc_flags_t) (Z_WAITOK | Z_ZERO), VM_KERN_MEMORY_LIBKERN);
 568 }
 569
 570 void
 571 operator delete[](void * ptr)
 572 #if __cplusplus >= 201103L
 573 noexcept
 574 #endif
 575 {
 576         if (ptr) {
 577 #if KASAN
 578                 /*
 579                  * Unpoison the C++ array cookie inserted (but not removed) by the
 580                  * compiler on new[].
 581                  */
 582                 kasan_unpoison_cxx_array_cookie(ptr);
 583 #endif
 584                 kheap_free_addr(KERN_OS_MALLOC, ptr);
 585         }
 586         return;
 587 }
 588
 589 #if __cplusplus >= 201103L
 590
 591 void
 592 operator delete(void * addr, size_t sz) noexcept
 593 {
 594         kheap_free(KERN_OS_MALLOC, addr, sz);
 595 }
 596
 597 void
 598 operator delete[](void * addr, size_t sz) noexcept
 599 {
 600         if (addr) {
 601                 kheap_free(KERN_OS_MALLOC, addr, sz);
 602         }
 603 }
 604
 605 #endif /* __cplusplus >= 201103L */
 606
 607 /* PR-6481964 - The compiler is going to check for size overflows in calls to
 608  * new[], and if there is an overflow, it will call __throw_length_error.
 609  * This is an unrecoverable error by the C++ standard, so we must panic here.
 610  *
 611  * We have to put the function inside the std namespace because of how the
 612  * compiler expects the name to be mangled.
 613  */
 614 namespace std {
 615 void __dead2
 616 __throw_length_error(const char *msg __unused)
 617 {
 618         panic("Size of array created by new[] has overflowed");
 619 }
 620 };