libkern/c++/OSRuntime.cpp

   1 /*
   2  * Copyright (c) 2000,2008-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * Copyright (c) 1997 Apple Inc.
  30  *
  31  */
  32 #include <libkern/c++/OSMetaClass.h>
  33 #include <libkern/c++/OSKext.h>
  34 #include <libkern/c++/OSLib.h>
  35 #include <libkern/c++/OSSymbol.h>
  36 #include <IOKit/IOKitDebug.h>
  37
  38 #include <sys/cdefs.h>
  39
  40 __BEGIN_DECLS
  41
  42 #include <string.h>
  43 #include <mach/mach_types.h>
  44 #include <libkern/kernel_mach_header.h>
  45 #include <libkern/prelink.h>
  46 #include <stdarg.h>
  47
  48 #if KASAN
  49 #include <san/kasan.h>
  50 #endif
  51
  52 #if PRAGMA_MARK
  53 #pragma mark Constants &c.
  54 #endif /* PRAGMA_MARK */
  55 OSKextLogSpec kOSRuntimeLogSpec =
  56     kOSKextLogErrorLevel |
  57     kOSKextLogLoadFlag |
  58     kOSKextLogKextBookkeepingFlag;
  59
  60 #if PRAGMA_MARK
  61 #pragma mark Logging Bootstrap
  62 #endif /* PRAGMA_MARK */
  63 /*********************************************************************
  64 * kern_os Logging Bootstrap
  65 *
  66 * We can't call in to OSKext until the kernel's C++ environment is up
  67 * and running, so let's mask those references with a check variable.
  68 * We print unconditionally if C++ isn't up, but if that's the case
  69 * we've generally hit a serious error in kernel init!
  70 *********************************************************************/
  71 static bool gKernelCPPInitialized = false;
  72
  73 #define OSRuntimeLog(kext, flags, format, args...)            \
  74     do {                                                      \
  75         if (gKernelCPPInitialized) {                          \
  76             OSKextLog((kext), (flags), (format), ## args);  \
  77         } else {                                              \
  78             printf((format), ## args);                        \
  79         }                                                     \
  80     } while (0)
  81
  82 #if PRAGMA_MARK
  83 #pragma mark kern_os Allocator Package
  84 #endif /* PRAGMA_MARK */
  85 /*********************************************************************
  86 * kern_os Allocator Package
  87 *********************************************************************/
  88
  89 /*********************************************************************
  90 *********************************************************************/
  91 #if OSALLOCDEBUG
  92 extern int debug_iomalloc_size;
  93 #endif
  94
  95 /*********************************************************************
  96 *********************************************************************/
  97 void *
  98 kern_os_malloc(size_t size)
  99 {
 100         void *mem;
 101         if (size == 0) {
 102                 return NULL;
 103         }
 104
 105         mem = kallocp_tag_bt((vm_size_t *)&size, VM_KERN_MEMORY_LIBKERN);
 106         if (!mem) {
 107                 return NULL;
 108         }
 109
 110 #if OSALLOCDEBUG
 111         OSAddAtomic(size, &debug_iomalloc_size);
 112 #endif
 113
 114         bzero(mem, size);
 115
 116         return mem;
 117 }
 118
 119 /*********************************************************************
 120 *********************************************************************/
 121 void
 122 kern_os_free(void * addr)
 123 {
 124         size_t size;
 125         size = kalloc_size(addr);
 126 #if OSALLOCDEBUG
 127         OSAddAtomic(-size, &debug_iomalloc_size);
 128 #endif
 129
 130         kfree_addr(addr);
 131 }
 132
 133 /*********************************************************************
 134 *********************************************************************/
 135 void *
 136 kern_os_realloc(
 137         void   * addr,
 138         size_t   nsize)
 139 {
 140         void            *nmem;
 141         size_t          osize;
 142
 143         if (!addr) {
 144                 return kern_os_malloc(nsize);
 145         }
 146
 147         osize = kalloc_size(addr);
 148         if (nsize == osize) {
 149                 return addr;
 150         }
 151
 152         if (nsize == 0) {
 153                 kfree_addr(addr);
 154                 return NULL;
 155         }
 156
 157         nmem = kallocp_tag_bt((vm_size_t *)&nsize, VM_KERN_MEMORY_LIBKERN);
 158         if (!nmem) {
 159                 kfree_addr(addr);
 160                 return NULL;
 161         }
 162
 163 #if OSALLOCDEBUG
 164         OSAddAtomic((nsize - osize), &debug_iomalloc_size);
 165 #endif
 166
 167         if (nsize > osize) {
 168                 (void)memset((char *)nmem + osize, 0, nsize - osize);
 169         }
 170         (void)memcpy(nmem, addr, (nsize > osize) ? osize : nsize);
 171         kfree_addr(addr);
 172
 173         return nmem;
 174 }
 175
 176 #if PRAGMA_MARK
 177 #pragma mark Libkern Init
 178 #endif /* PRAGMA_MARK */
 179 /*********************************************************************
 180 * Libkern Init
 181 *********************************************************************/
 182
 183 #if __GNUC__ >= 3
 184 void __dead2
 185 __cxa_pure_virtual( void )
 186 {
 187         panic("%s", __FUNCTION__);
 188 }
 189 #else
 190 void __dead2
 191 __pure_virtual( void )
 192 {
 193         panic("%s", __FUNCTION__);
 194 }
 195 #endif
 196
 197 extern lck_grp_t * IOLockGroup;
 198 extern kmod_info_t g_kernel_kmod_info;
 199
 200 enum {
 201         kOSSectionNamesDefault     = 0,
 202         kOSSectionNamesBuiltinKext = 1,
 203         kOSSectionNamesCount       = 2,
 204 };
 205 enum {
 206         kOSSectionNameInitializer = 0,
 207         kOSSectionNameFinalizer   = 1,
 208         kOSSectionNameCount       = 2
 209 };
 210
 211 static const char *
 212     gOSStructorSectionNames[kOSSectionNamesCount][kOSSectionNameCount] = {
 213         { SECT_MODINITFUNC, SECT_MODTERMFUNC },
 214         { kBuiltinInitSection, kBuiltinTermSection }
 215 };
 216
 217 void
 218 OSlibkernInit(void)
 219 {
 220         // This must be called before calling OSRuntimeInitializeCPP.
 221         OSMetaClassBase::initialize();
 222
 223         g_kernel_kmod_info.address = (vm_address_t) &_mh_execute_header;
 224         if (kOSReturnSuccess != OSRuntimeInitializeCPP(NULL)) {
 225                 // &g_kernel_kmod_info, gOSSectionNamesStandard, 0, 0)) {
 226                 panic("OSRuntime: C++ runtime failed to initialize.");
 227         }
 228
 229         gKernelCPPInitialized = true;
 230
 231         return;
 232 }
 233
 234 __END_DECLS
 235
 236 #if PRAGMA_MARK
 237 #pragma mark C++ Runtime Load/Unload
 238 #endif /* PRAGMA_MARK */
 239 /*********************************************************************
 240 * kern_os C++ Runtime Load/Unload
 241 *********************************************************************/
 242
 243 #if defined(HAS_APPLE_PAC)
 244 #include <ptrauth.h>
 245 #endif /* defined(HAS_APPLE_PAC) */
 246
 247 typedef void (*structor_t)(void);
 248
 249 static bool
 250 OSRuntimeCallStructorsInSection(
 251         OSKext                   * theKext,
 252         kmod_info_t              * kmodInfo,
 253         void                     * metaHandle,
 254         kernel_segment_command_t * segment,
 255         const char               * sectionName,
 256         uintptr_t                  textStart,
 257         uintptr_t                  textEnd)
 258 {
 259         kernel_section_t * section;
 260         bool result = TRUE;
 261
 262         for (section = firstsect(segment);
 263             section != NULL;
 264             section = nextsect(segment, section)) {
 265                 if (strncmp(section->sectname, sectionName, sizeof(section->sectname) - 1)) {
 266                         continue;
 267                 }
 268
 269                 structor_t * structors = (structor_t *)section->addr;
 270                 if (!structors) {
 271                         continue;
 272                 }
 273
 274                 structor_t structor;
 275                 unsigned int num_structors = section->size / sizeof(structor_t);
 276                 unsigned int hit_null_structor = 0;
 277                 unsigned int firstIndex = 0;
 278
 279                 if (textStart) {
 280                         // bsearch for any in range
 281                         unsigned int baseIdx;
 282                         unsigned int lim;
 283                         uintptr_t value;
 284                         firstIndex = num_structors;
 285                         for (lim = num_structors, baseIdx = 0; lim; lim >>= 1) {
 286                                 value = (uintptr_t) structors[baseIdx + (lim >> 1)];
 287                                 if (!value) {
 288                                         panic("%s: null structor", kmodInfo->name);
 289                                 }
 290                                 if ((value >= textStart) && (value < textEnd)) {
 291                                         firstIndex = (baseIdx + (lim >> 1));
 292                                         // scan back for the first in range
 293                                         for (; firstIndex; firstIndex--) {
 294                                                 value = (uintptr_t) structors[firstIndex - 1];
 295                                                 if ((value < textStart) || (value >= textEnd)) {
 296                                                         break;
 297                                                 }
 298                                         }
 299                                         break;
 300                                 }
 301                                 if (textStart > value) {
 302                                         // move right
 303                                         baseIdx += (lim >> 1) + 1;
 304                                         lim--;
 305                                 }
 306                                 // else move left
 307                         }
 308                         baseIdx = (baseIdx + (lim >> 1));
 309                 }
 310                 for (;
 311                     (firstIndex < num_structors)
 312                     && (!metaHandle || OSMetaClass::checkModLoad(metaHandle));
 313                     firstIndex++) {
 314                         if ((structor = structors[firstIndex])) {
 315                                 if ((textStart && ((uintptr_t) structor < textStart))
 316                                     || (textEnd && ((uintptr_t) structor >= textEnd))) {
 317                                         break;
 318                                 }
 319
 320 #if !defined(XXX) && defined(HAS_APPLE_PAC)
 321                                 structor = __builtin_ptrauth_strip(structor, ptrauth_key_function_pointer);
 322                                 structor = __builtin_ptrauth_sign_unauthenticated(structor, ptrauth_key_function_pointer, 0);
 323 #endif
 324                                 (*structor)();
 325                         } else if (!hit_null_structor) {
 326                                 hit_null_structor = 1;
 327                                 OSRuntimeLog(theKext, kOSRuntimeLogSpec,
 328                                     "Null structor in kext %s segment %s!",
 329                                     kmodInfo->name, section->segname);
 330                         }
 331                 }
 332                 if (metaHandle) {
 333                         result = OSMetaClass::checkModLoad(metaHandle);
 334                 }
 335                 break;
 336         } /* for (section...) */
 337         return result;
 338 }
 339
 340 /*********************************************************************
 341 *********************************************************************/
 342 kern_return_t
 343 OSRuntimeFinalizeCPP(
 344         OSKext                   * theKext)
 345 {
 346         kern_return_t              result = KMOD_RETURN_FAILURE;
 347         void                     * metaHandle = NULL;// do not free
 348         kernel_mach_header_t     * header;
 349         kernel_segment_command_t * segment;
 350         kmod_info_t              * kmodInfo;
 351         const char              ** sectionNames;
 352         uintptr_t                  textStart;
 353         uintptr_t                  textEnd;
 354
 355         textStart    = 0;
 356         textEnd      = 0;
 357         sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault];
 358         if (theKext) {
 359                 if (!theKext->isCPPInitialized()) {
 360                         result = KMOD_RETURN_SUCCESS;
 361                         goto finish;
 362                 }
 363                 kmodInfo = theKext->kmod_info;
 364                 if (!kmodInfo || !kmodInfo->address) {
 365                         result = kOSKextReturnInvalidArgument;
 366                         goto finish;
 367                 }
 368                 header = (kernel_mach_header_t *)kmodInfo->address;
 369                 if (theKext->flags.builtin) {
 370                         header       = (kernel_mach_header_t *)g_kernel_kmod_info.address;
 371                         textStart    = kmodInfo->address;
 372                         textEnd      = textStart + kmodInfo->size;
 373                         sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext];
 374                 }
 375         } else {
 376                 kmodInfo = &g_kernel_kmod_info;
 377                 header   = (kernel_mach_header_t *)kmodInfo->address;
 378         }
 379
 380         /* OSKext checks for this condition now, but somebody might call
 381          * this function directly (the symbol is exported....).
 382          */
 383         if (OSMetaClass::modHasInstance(kmodInfo->name)) {
 384                 // xxx - Don't log under errors? this is more of an info thing
 385                 OSRuntimeLog(theKext, kOSRuntimeLogSpec,
 386                     "Can't tear down kext %s C++; classes have instances:",
 387                     kmodInfo->name);
 388                 OSKext::reportOSMetaClassInstances(kmodInfo->name, kOSRuntimeLogSpec);
 389                 result = kOSMetaClassHasInstances;
 390                 goto finish;
 391         }
 392
 393         /* Tell the meta class system that we are starting to unload.
 394          * metaHandle isn't actually needed on the finalize path,
 395          * so we don't check it here, even though OSMetaClass::postModLoad() will
 396          * return a failure (it only does actual work on the init path anyhow).
 397          */
 398         metaHandle = OSMetaClass::preModLoad(kmodInfo->name);
 399
 400         OSSymbol::checkForPageUnload((void *)kmodInfo->address,
 401             (void *)(kmodInfo->address + kmodInfo->size));
 402
 403         header = (kernel_mach_header_t *)kmodInfo->address;
 404         segment = firstsegfromheader(header);
 405
 406         for (segment = firstsegfromheader(header);
 407             segment != NULL;
 408             segment = nextsegfromheader(header, segment)) {
 409                 OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment,
 410                     sectionNames[kOSSectionNameFinalizer], textStart, textEnd);
 411         }
 412
 413         (void)OSMetaClass::postModLoad(metaHandle);
 414
 415         if (theKext) {
 416                 theKext->setCPPInitialized(false);
 417         }
 418         result = KMOD_RETURN_SUCCESS;
 419 finish:
 420         return result;
 421 }
 422
 423 /*********************************************************************
 424 *********************************************************************/
 425 kern_return_t
 426 OSRuntimeInitializeCPP(
 427         OSKext                   * theKext)
 428 {
 429         kern_return_t              result          = KMOD_RETURN_FAILURE;
 430         kernel_mach_header_t     * header          = NULL;
 431         void                     * metaHandle      = NULL;// do not free
 432         bool                       load_success    = true;
 433         kernel_segment_command_t * segment         = NULL;// do not free
 434         kernel_segment_command_t * failure_segment = NULL; // do not free
 435         kmod_info_t             *  kmodInfo;
 436         const char              ** sectionNames;
 437         uintptr_t                  textStart;
 438         uintptr_t                  textEnd;
 439
 440         textStart    = 0;
 441         textEnd      = 0;
 442         sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault];
 443         if (theKext) {
 444                 if (theKext->isCPPInitialized()) {
 445                         result = KMOD_RETURN_SUCCESS;
 446                         goto finish;
 447                 }
 448
 449                 kmodInfo = theKext->kmod_info;
 450                 if (!kmodInfo || !kmodInfo->address) {
 451                         result = kOSKextReturnInvalidArgument;
 452                         goto finish;
 453                 }
 454                 header = (kernel_mach_header_t *)kmodInfo->address;
 455
 456                 if (theKext->flags.builtin) {
 457                         header       = (kernel_mach_header_t *)g_kernel_kmod_info.address;
 458                         textStart    = kmodInfo->address;
 459                         textEnd      = textStart + kmodInfo->size;
 460                         sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext];
 461                 }
 462         } else {
 463                 kmodInfo = &g_kernel_kmod_info;
 464                 header   = (kernel_mach_header_t *)kmodInfo->address;
 465         }
 466
 467         /* Tell the meta class system that we are starting the load
 468          */
 469         metaHandle = OSMetaClass::preModLoad(kmodInfo->name);
 470         assert(metaHandle);
 471         if (!metaHandle) {
 472                 goto finish;
 473         }
 474
 475         /* NO GOTO PAST HERE. */
 476
 477         /* Scan the header for all constructor sections, in any
 478          * segment, and invoke the constructors within those sections.
 479          */
 480         for (segment = firstsegfromheader(header);
 481             segment != NULL && load_success;
 482             segment = nextsegfromheader(header, segment)) {
 483                 /* Record the current segment in the event of a failure.
 484                  */
 485                 failure_segment = segment;
 486                 load_success = OSRuntimeCallStructorsInSection(
 487                         theKext, kmodInfo, metaHandle, segment,
 488                         sectionNames[kOSSectionNameInitializer],
 489                         textStart, textEnd);
 490         } /* for (segment...) */
 491
 492         /* We failed so call all of the destructors. We must do this before
 493          * calling OSMetaClass::postModLoad() as the OSMetaClass destructors
 494          * will alter state (in the metaHandle) used by that function.
 495          */
 496         if (!load_success) {
 497                 /* Scan the header for all destructor sections, in any
 498                  * segment, and invoke the constructors within those sections.
 499                  */
 500                 for (segment = firstsegfromheader(header);
 501                     segment != failure_segment && segment != NULL;
 502                     segment = nextsegfromheader(header, segment)) {
 503                         OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment,
 504                             sectionNames[kOSSectionNameFinalizer], textStart, textEnd);
 505                 } /* for (segment...) */
 506         }
 507
 508         /* Now, regardless of success so far, do the post-init registration
 509          * and cleanup. If we had to call the unloadCPP function, static
 510          * destructors have removed classes from the stalled list so no
 511          * metaclasses will actually be registered.
 512          */
 513         result = OSMetaClass::postModLoad(metaHandle);
 514
 515         /* If we've otherwise been fine up to now, but OSMetaClass::postModLoad()
 516          * fails (typically due to a duplicate class), tear down all the C++
 517          * stuff from the kext. This isn't necessary for libkern/OSMetaClass stuff,
 518          * but may be necessary for other C++ code. We ignore the return value
 519          * because it's only a fail when there are existing instances of libkern
 520          * classes, and there had better not be any created on the C++ init path.
 521          */
 522         if (load_success && result != KMOD_RETURN_SUCCESS) {
 523                 (void)OSRuntimeFinalizeCPP(theKext); //kmodInfo, sectionNames, textStart, textEnd);
 524         }
 525
 526         if (theKext && load_success && result == KMOD_RETURN_SUCCESS) {
 527                 theKext->setCPPInitialized(true);
 528         }
 529 finish:
 530         return result;
 531 }
 532
 533 /*********************************************************************
 534 *   Unload a kernel segment.
 535 *********************************************************************/
 536
 537 void
 538 OSRuntimeUnloadCPPForSegment(
 539         kernel_segment_command_t * segment)
 540 {
 541         OSRuntimeCallStructorsInSection(NULL, &g_kernel_kmod_info, NULL, segment,
 542             gOSStructorSectionNames[kOSSectionNamesDefault][kOSSectionNameFinalizer], 0, 0);
 543 }
 544
 545 #if PRAGMA_MARK
 546 #pragma mark C++ Allocators & Deallocators
 547 #endif /* PRAGMA_MARK */
 548 /*********************************************************************
 549 * C++ Allocators & Deallocators
 550 *********************************************************************/
 551 void *
 552 operator new(size_t size)
 553 {
 554         void * result;
 555
 556         result = (void *) kern_os_malloc(size);
 557         return result;
 558 }
 559
 560 void
 561 operator delete(void * addr)
 562 #if __cplusplus >= 201103L
 563 noexcept
 564 #endif
 565 {
 566         kern_os_free(addr);
 567         return;
 568 }
 569
 570 void *
 571 operator new[](unsigned long sz)
 572 {
 573         if (sz == 0) {
 574                 sz = 1;
 575         }
 576         return kern_os_malloc(sz);
 577 }
 578
 579 void
 580 operator delete[](void * ptr)
 581 #if __cplusplus >= 201103L
 582 noexcept
 583 #endif
 584 {
 585         if (ptr) {
 586 #if KASAN
 587                 /*
 588                  * Unpoison the C++ array cookie inserted (but not removed) by the
 589                  * compiler on new[].
 590                  */
 591                 kasan_unpoison_cxx_array_cookie(ptr);
 592 #endif
 593                 kern_os_free(ptr);
 594         }
 595         return;
 596 }
 597
 598 #if __cplusplus >= 201103L
 599
 600 void
 601 operator delete(void * addr, size_t sz) noexcept
 602 {
 603 #if OSALLOCDEBUG
 604         OSAddAtomic(-sz, &debug_iomalloc_size);
 605 #endif /* OSALLOCDEBUG */
 606         kfree(addr, sz);
 607 }
 608
 609 void
 610 operator delete[](void * addr, size_t sz) noexcept
 611 {
 612         if (addr) {
 613 #if OSALLOCDEBUG
 614                 OSAddAtomic(-sz, &debug_iomalloc_size);
 615 #endif /* OSALLOCDEBUG */
 616                 kfree(addr, sz);
 617         }
 618 }
 619
 620 #endif /* __cplusplus >= 201103L */
 621
 622 /* PR-6481964 - The compiler is going to check for size overflows in calls to
 623  * new[], and if there is an overflow, it will call __throw_length_error.
 624  * This is an unrecoverable error by the C++ standard, so we must panic here.
 625  *
 626  * We have to put the function inside the std namespace because of how the
 627  * compiler expects the name to be mangled.
 628  */
 629 namespace std {
 630 void __dead2
 631 __throw_length_error(const char *msg __unused)
 632 {
 633         panic("Size of array created by new[] has overflowed");
 634 }
 635 };