libkern/libclosure/runtime.cpp

   1 /*
   2  * runtime.c
   3  * libclosure
   4  *
   5  * Copyright (c) 2008-2010 Apple Inc. All rights reserved.
   6  *
   7  * @APPLE_LLVM_LICENSE_HEADER@
   8  */
   9
  10
  11 #ifndef KERNEL
  12
  13 #include "Block_private.h"
  14 #include <stdio.h>
  15 #include <stdlib.h>
  16 #include <dlfcn.h>
  17 #include <os/assumes.h>
  18
  19 #else /* !KERNEL */
  20
  21 #include <libkern/Block_private.h>
  22 #include <libkern/OSRuntime.h>
  23
  24 #define malloc(s)  kern_os_malloc((s))
  25 #define free(a)    kern_os_free((a))
  26
  27 #endif /* KERNEL */
  28
  29 #include <machine/atomic.h>
  30 #include <string.h>
  31 #include <stdint.h>
  32 #ifndef os_assumes
  33 #define os_assumes(_x) (_x)
  34 #endif
  35 #ifndef os_assert
  36 #define os_assert(_x) assert(_x)
  37 #endif
  38
  39 #if TARGET_OS_WIN32
  40 #define _CRT_SECURE_NO_WARNINGS 1
  41 #include <windows.h>
  42 static __inline bool
  43 OSAtomicCompareAndSwapLong(long oldl, long newl, long volatile *dst)
  44 {
  45         // fixme barrier is overkill -- see objc-os.h
  46         long original = InterlockedCompareExchange(dst, newl, oldl);
  47         return original == oldl;
  48 }
  49
  50 static __inline bool
  51 OSAtomicCompareAndSwapInt(int oldi, int newi, int volatile *dst)
  52 {
  53         // fixme barrier is overkill -- see objc-os.h
  54         int original = InterlockedCompareExchange(dst, newi, oldi);
  55         return original == oldi;
  56 }
  57 #else
  58 #define OSAtomicCompareAndSwapLong(_Old, _New, _Ptr) os_atomic_cmpxchg(_Ptr, _Old, _New, relaxed)
  59 #define OSAtomicCompareAndSwapInt(_Old, _New, _Ptr) os_atomic_cmpxchg(_Ptr, _Old, _New, relaxed)
  60 #endif
  61
  62
  63 /*******************************************************************************
  64  *  Internal Utilities
  65  ********************************************************************************/
  66
  67 static int32_t
  68 latching_incr_int(volatile int32_t *where)
  69 {
  70         while (1) {
  71                 int32_t old_value = *where;
  72                 if ((old_value & BLOCK_REFCOUNT_MASK) == BLOCK_REFCOUNT_MASK) {
  73                         return BLOCK_REFCOUNT_MASK;
  74                 }
  75                 if (OSAtomicCompareAndSwapInt(old_value, old_value + 2, where)) {
  76                         return old_value + 2;
  77                 }
  78         }
  79 }
  80
  81 static bool
  82 latching_incr_int_not_deallocating(volatile int32_t *where)
  83 {
  84         while (1) {
  85                 int32_t old_value = *where;
  86                 if (old_value & BLOCK_DEALLOCATING) {
  87                         // if deallocating we can't do this
  88                         return false;
  89                 }
  90                 if ((old_value & BLOCK_REFCOUNT_MASK) == BLOCK_REFCOUNT_MASK) {
  91                         // if latched, we're leaking this block, and we succeed
  92                         return true;
  93                 }
  94                 if (OSAtomicCompareAndSwapInt(old_value, old_value + 2, where)) {
  95                         // otherwise, we must store a new retained value without the deallocating bit set
  96                         return true;
  97                 }
  98         }
  99 }
 100
 101
 102 // return should_deallocate?
 103 static bool
 104 latching_decr_int_should_deallocate(volatile int32_t *where)
 105 {
 106         while (1) {
 107                 int32_t old_value = *where;
 108                 if ((old_value & BLOCK_REFCOUNT_MASK) == BLOCK_REFCOUNT_MASK) {
 109                         return false; // latched high
 110                 }
 111                 if ((old_value & BLOCK_REFCOUNT_MASK) == 0) {
 112                         return false; // underflow, latch low
 113                 }
 114                 int32_t new_value = old_value - 2;
 115                 bool result = false;
 116                 if ((old_value & (BLOCK_REFCOUNT_MASK | BLOCK_DEALLOCATING)) == 2) {
 117                         new_value = old_value - 1;
 118                         result = true;
 119                 }
 120                 if (OSAtomicCompareAndSwapInt(old_value, new_value, where)) {
 121                         return result;
 122                 }
 123         }
 124 }
 125
 126
 127 /**************************************************************************
 128  *  Framework callback functions and their default implementations.
 129  ***************************************************************************/
 130 #if !TARGET_OS_WIN32
 131 #pragma mark Framework Callback Routines
 132 #endif
 133
 134 static void
 135 _Block_retain_object_default(const void *ptr __unused)
 136 {
 137 }
 138
 139 static void
 140 _Block_release_object_default(const void *ptr __unused)
 141 {
 142 }
 143
 144 static void
 145 _Block_destructInstance_default(const void *aBlock __unused)
 146 {
 147 }
 148
 149 static void (*_Block_retain_object)(const void *ptr) = _Block_retain_object_default;
 150 static void (*_Block_release_object)(const void *ptr) = _Block_release_object_default;
 151 static void (*_Block_destructInstance) (const void *aBlock) = _Block_destructInstance_default;
 152
 153
 154 /**************************************************************************
 155  *  Callback registration from ObjC runtime and CoreFoundation
 156  ***************************************************************************/
 157
 158 void
 159 _Block_use_RR2(const Block_callbacks_RR *callbacks)
 160 {
 161         _Block_retain_object = callbacks->retain;
 162         _Block_release_object = callbacks->release;
 163         _Block_destructInstance = callbacks->destructInstance;
 164 }
 165
 166 /****************************************************************************
 167  *  Accessors for block descriptor fields
 168  *****************************************************************************/
 169 #if 0
 170 static struct Block_descriptor_1 *
 171 _Block_descriptor_1(struct Block_layout *aBlock)
 172 {
 173         return aBlock->descriptor;
 174 }
 175 #endif
 176
 177 static struct Block_descriptor_2 *
 178 _Block_descriptor_2(struct Block_layout *aBlock)
 179 {
 180         if (!(aBlock->flags & BLOCK_HAS_COPY_DISPOSE)) {
 181                 return NULL;
 182         }
 183         uint8_t *desc = (uint8_t *)aBlock->descriptor;
 184         desc += sizeof(struct Block_descriptor_1);
 185         return __IGNORE_WCASTALIGN((struct Block_descriptor_2 *)desc);
 186 }
 187
 188 static struct Block_descriptor_3 *
 189 _Block_descriptor_3(struct Block_layout *aBlock)
 190 {
 191         if (!(aBlock->flags & BLOCK_HAS_SIGNATURE)) {
 192                 return NULL;
 193         }
 194         uint8_t *desc = (uint8_t *)aBlock->descriptor;
 195         desc += sizeof(struct Block_descriptor_1);
 196         if (aBlock->flags & BLOCK_HAS_COPY_DISPOSE) {
 197                 desc += sizeof(struct Block_descriptor_2);
 198         }
 199         return __IGNORE_WCASTALIGN((struct Block_descriptor_3 *)desc);
 200 }
 201
 202 static void
 203 _Block_call_copy_helper(void *result, struct Block_layout *aBlock)
 204 {
 205         struct Block_descriptor_2 *desc = _Block_descriptor_2(aBlock);
 206         if (!desc) {
 207                 return;
 208         }
 209
 210         (*desc->copy)(result, aBlock); // do fixup
 211 }
 212
 213 static void
 214 _Block_call_dispose_helper(struct Block_layout *aBlock)
 215 {
 216         struct Block_descriptor_2 *desc = _Block_descriptor_2(aBlock);
 217         if (!desc) {
 218                 return;
 219         }
 220
 221         (*desc->dispose)(aBlock);
 222 }
 223
 224 /*******************************************************************************
 225  *  Internal Support routines for copying
 226  ********************************************************************************/
 227
 228 #if !TARGET_OS_WIN32
 229 #pragma mark Copy/Release support
 230 #endif
 231
 232 // Copy, or bump refcount, of a block.  If really copying, call the copy helper if present.
 233 void *
 234 _Block_copy(const void *arg)
 235 {
 236         struct Block_layout *aBlock;
 237
 238         if (!arg) {
 239                 return NULL;
 240         }
 241
 242         // The following would be better done as a switch statement
 243         aBlock = (struct Block_layout *)arg;
 244         if (aBlock->flags & BLOCK_NEEDS_FREE) {
 245                 // latches on high
 246                 latching_incr_int(&aBlock->flags);
 247                 return aBlock;
 248         } else if (aBlock->flags & BLOCK_IS_GLOBAL) {
 249                 return aBlock;
 250         } else {
 251                 // Its a stack block.  Make a copy.
 252                 struct Block_layout *result = (typeof(result))malloc(aBlock->descriptor->size);
 253                 if (!result) {
 254                         return NULL;
 255                 }
 256                 memmove(result, aBlock, aBlock->descriptor->size); // bitcopy first
 257 #if __has_feature(ptrauth_calls)
 258                 // Resign the invoke pointer as it uses address authentication.
 259                 result->invoke = aBlock->invoke;
 260 #endif
 261                 // reset refcount
 262                 result->flags &= ~(BLOCK_REFCOUNT_MASK | BLOCK_DEALLOCATING); // XXX not needed
 263                 result->flags |= BLOCK_NEEDS_FREE | 2; // logical refcount 1
 264                 _Block_call_copy_helper(result, aBlock);
 265                 // Set isa last so memory analysis tools see a fully-initialized object.
 266                 result->isa = _NSConcreteMallocBlock;
 267                 return result;
 268         }
 269 }
 270
 271
 272 // Runtime entry points for maintaining the sharing knowledge of byref data blocks.
 273
 274 // A closure has been copied and its fixup routine is asking us to fix up the reference to the shared byref data
 275 // Closures that aren't copied must still work, so everyone always accesses variables after dereferencing the forwarding ptr.
 276 // We ask if the byref pointer that we know about has already been copied to the heap, and if so, increment and return it.
 277 // Otherwise we need to copy it and update the stack forwarding pointer
 278 static struct Block_byref *
 279 _Block_byref_copy(const void *arg)
 280 {
 281         struct Block_byref *src = (struct Block_byref *)arg;
 282
 283         if ((src->forwarding->flags & BLOCK_REFCOUNT_MASK) == 0) {
 284                 // src points to stack
 285                 struct Block_byref *copy = (struct Block_byref *)malloc(src->size);
 286                 copy->isa = NULL;
 287                 // byref value 4 is logical refcount of 2: one for caller, one for stack
 288                 copy->flags = src->flags | BLOCK_BYREF_NEEDS_FREE | 4;
 289                 copy->forwarding = copy; // patch heap copy to point to itself
 290                 src->forwarding = copy; // patch stack to point to heap copy
 291                 copy->size = src->size;
 292
 293                 if (src->flags & BLOCK_BYREF_HAS_COPY_DISPOSE) {
 294                         // Trust copy helper to copy everything of interest
 295                         // If more than one field shows up in a byref block this is wrong XXX
 296                         struct Block_byref_2 *src2 = (struct Block_byref_2 *)(src + 1);
 297                         struct Block_byref_2 *copy2 = (struct Block_byref_2 *)(copy + 1);
 298                         copy2->byref_keep = src2->byref_keep;
 299                         copy2->byref_destroy = src2->byref_destroy;
 300
 301                         if (src->flags & BLOCK_BYREF_LAYOUT_EXTENDED) {
 302                                 struct Block_byref_3 *src3 = (struct Block_byref_3 *)(src2 + 1);
 303                                 struct Block_byref_3 *copy3 = (struct Block_byref_3*)(copy2 + 1);
 304                                 copy3->layout = src3->layout;
 305                         }
 306
 307                         (*src2->byref_keep)(copy, src);
 308                 } else {
 309                         // Bitwise copy.
 310                         // This copy includes Block_byref_3, if any.
 311                         memmove(copy + 1, src + 1, src->size - sizeof(*src));
 312                 }
 313         }
 314         // already copied to heap
 315         else if ((src->forwarding->flags & BLOCK_BYREF_NEEDS_FREE) == BLOCK_BYREF_NEEDS_FREE) {
 316                 latching_incr_int(&src->forwarding->flags);
 317         }
 318
 319         return src->forwarding;
 320 }
 321
 322 static void
 323 _Block_byref_release(const void *arg)
 324 {
 325         struct Block_byref *byref = (struct Block_byref *)arg;
 326
 327         // dereference the forwarding pointer since the compiler isn't doing this anymore (ever?)
 328         byref = byref->forwarding;
 329
 330         if (byref->flags & BLOCK_BYREF_NEEDS_FREE) {
 331                 __assert_only int32_t refcount = byref->flags & BLOCK_REFCOUNT_MASK;
 332                 os_assert(refcount);
 333                 if (latching_decr_int_should_deallocate(&byref->flags)) {
 334                         if (byref->flags & BLOCK_BYREF_HAS_COPY_DISPOSE) {
 335                                 struct Block_byref_2 *byref2 = (struct Block_byref_2 *)(byref + 1);
 336                                 (*byref2->byref_destroy)(byref);
 337                         }
 338                         free(byref);
 339                 }
 340         }
 341 }
 342
 343
 344 /************************************************************
 345  *
 346  * API supporting SPI
 347  * _Block_copy, _Block_release, and (old) _Block_destroy
 348  *
 349  ***********************************************************/
 350
 351 #if !TARGET_OS_WIN32
 352 #pragma mark SPI/API
 353 #endif
 354
 355
 356 // API entry point to release a copied Block
 357 void
 358 _Block_release(const void *arg)
 359 {
 360         struct Block_layout *aBlock = (struct Block_layout *)arg;
 361         if (!aBlock) {
 362                 return;
 363         }
 364         if (aBlock->flags & BLOCK_IS_GLOBAL) {
 365                 return;
 366         }
 367         if (!(aBlock->flags & BLOCK_NEEDS_FREE)) {
 368                 return;
 369         }
 370
 371         if (latching_decr_int_should_deallocate(&aBlock->flags)) {
 372                 _Block_call_dispose_helper(aBlock);
 373                 _Block_destructInstance(aBlock);
 374                 free(aBlock);
 375         }
 376 }
 377
 378 bool
 379 _Block_tryRetain(const void *arg)
 380 {
 381         struct Block_layout *aBlock = (struct Block_layout *)arg;
 382         return latching_incr_int_not_deallocating(&aBlock->flags);
 383 }
 384
 385 bool
 386 _Block_isDeallocating(const void *arg)
 387 {
 388         struct Block_layout *aBlock = (struct Block_layout *)arg;
 389         return (aBlock->flags & BLOCK_DEALLOCATING) != 0;
 390 }
 391
 392
 393 /************************************************************
 394  *
 395  * SPI used by other layers
 396  *
 397  ***********************************************************/
 398
 399 size_t
 400 Block_size(void *aBlock)
 401 {
 402         return ((struct Block_layout *)aBlock)->descriptor->size;
 403 }
 404
 405 bool
 406 _Block_use_stret(void *aBlock)
 407 {
 408         struct Block_layout *layout = (struct Block_layout *)aBlock;
 409
 410         int requiredFlags = BLOCK_HAS_SIGNATURE | BLOCK_USE_STRET;
 411         return (layout->flags & requiredFlags) == requiredFlags;
 412 }
 413
 414 // Checks for a valid signature, not merely the BLOCK_HAS_SIGNATURE bit.
 415 bool
 416 _Block_has_signature(void *aBlock)
 417 {
 418         return _Block_signature(aBlock) ? true : false;
 419 }
 420
 421 const char *
 422 _Block_signature(void *aBlock)
 423 {
 424         struct Block_descriptor_3 *desc3 = _Block_descriptor_3((struct Block_layout *)aBlock);
 425         if (!desc3) {
 426                 return NULL;
 427         }
 428
 429         return desc3->signature;
 430 }
 431
 432 const char *
 433 _Block_layout(void *aBlock)
 434 {
 435         // Don't return extended layout to callers expecting old GC layout
 436         struct Block_layout *layout = (struct Block_layout *)aBlock;
 437         if (layout->flags & BLOCK_HAS_EXTENDED_LAYOUT) {
 438                 return NULL;
 439         }
 440
 441         struct Block_descriptor_3 *desc3 = _Block_descriptor_3((struct Block_layout *)aBlock);
 442         if (!desc3) {
 443                 return NULL;
 444         }
 445
 446         return desc3->layout;
 447 }
 448
 449 const char *
 450 _Block_extended_layout(void *aBlock)
 451 {
 452         // Don't return old GC layout to callers expecting extended layout
 453         struct Block_layout *layout = (struct Block_layout *)aBlock;
 454         if (!(layout->flags & BLOCK_HAS_EXTENDED_LAYOUT)) {
 455                 return NULL;
 456         }
 457
 458         struct Block_descriptor_3 *desc3 = _Block_descriptor_3((struct Block_layout *)aBlock);
 459         if (!desc3) {
 460                 return NULL;
 461         }
 462
 463         // Return empty string (all non-object bytes) instead of NULL
 464         // so callers can distinguish "empty layout" from "no layout".
 465         if (!desc3->layout) {
 466                 return "";
 467         } else {
 468                 return desc3->layout;
 469         }
 470 }
 471
 472 #if !TARGET_OS_WIN32
 473 #pragma mark Compiler SPI entry points
 474 #endif
 475
 476
 477 /*******************************************************
 478  *
 479  *  Entry points used by the compiler - the real API!
 480  *
 481  *
 482  *  A Block can reference four different kinds of things that require help when the Block is copied to the heap.
 483  *  1) C++ stack based objects
 484  *  2) References to Objective-C objects
 485  *  3) Other Blocks
 486  *  4) __block variables
 487  *
 488  *  In these cases helper functions are synthesized by the compiler for use in Block_copy and Block_release, called the copy and dispose helpers.  The copy helper emits a call to the C++ const copy constructor for C++ stack based objects and for the rest calls into the runtime support function _Block_object_assign.  The dispose helper has a call to the C++ destructor for case 1 and a call into _Block_object_dispose for the rest.
 489  *
 490  *  The flags parameter of _Block_object_assign and _Block_object_dispose is set to
 491  * BLOCK_FIELD_IS_OBJECT (3), for the case of an Objective-C Object,
 492  * BLOCK_FIELD_IS_BLOCK (7), for the case of another Block, and
 493  * BLOCK_FIELD_IS_BYREF (8), for the case of a __block variable.
 494  *  If the __block variable is marked weak the compiler also or's in BLOCK_FIELD_IS_WEAK (16)
 495  *
 496  *  So the Block copy/dispose helpers should only ever generate the four flag values of 3, 7, 8, and 24.
 497  *
 498  *  When  a __block variable is either a C++ object, an Objective-C object, or another Block then the compiler also generates copy/dispose helper functions.  Similarly to the Block copy helper, the "__block" copy helper (formerly and still a.k.a. "byref" copy helper) will do a C++ copy constructor (not a const one though!) and the dispose helper will do the destructor.  And similarly the helpers will call into the same two support functions with the same values for objects and Blocks with the additional BLOCK_BYREF_CALLER (128) bit of information supplied.
 499  *
 500  *  So the __block copy/dispose helpers will generate flag values of 3 or 7 for objects and Blocks respectively, with BLOCK_FIELD_IS_WEAK (16) or'ed as appropriate and always 128 or'd in, for the following set of possibilities:
 501  *   __block id                   128+3       (0x83)
 502  *   __block (^Block)             128+7       (0x87)
 503  *   __weak __block id            128+3+16    (0x93)
 504  *   __weak __block (^Block)      128+7+16    (0x97)
 505  *
 506  *
 507  ********************************************************/
 508
 509 //
 510 // When Blocks or Block_byrefs hold objects then their copy routine helpers use this entry point
 511 // to do the assignment.
 512 //
 513 void
 514 _Block_object_assign(void *destArg, const void *object, const int flags)
 515 {
 516         const void **dest = (const void **)destArg;
 517         switch (os_assumes(flags & BLOCK_ALL_COPY_DISPOSE_FLAGS)) {
 518         case BLOCK_FIELD_IS_OBJECT:
 519                 /*******
 520                  *  id object = ...;
 521                  *  [^{ object; } copy];
 522                  ********/
 523
 524                 _Block_retain_object(object);
 525                 *dest = object;
 526                 break;
 527
 528         case BLOCK_FIELD_IS_BLOCK:
 529                 /*******
 530                  *  void (^object)(void) = ...;
 531                  *  [^{ object; } copy];
 532                  ********/
 533
 534                 *dest = _Block_copy(object);
 535                 break;
 536
 537         case BLOCK_FIELD_IS_BYREF | BLOCK_FIELD_IS_WEAK:
 538         case BLOCK_FIELD_IS_BYREF:
 539                 /*******
 540                  *  // copy the onstack __block container to the heap
 541                  *  // Note this __weak is old GC-weak/MRC-unretained.
 542                  *  // ARC-style __weak is handled by the copy helper directly.
 543                  *  __block ... x;
 544                  *  __weak __block ... x;
 545                  *  [^{ x; } copy];
 546                  ********/
 547
 548                 *dest = _Block_byref_copy(object);
 549                 break;
 550
 551         case BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_OBJECT:
 552         case BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_BLOCK:
 553                 /*******
 554                  *  // copy the actual field held in the __block container
 555                  *  // Note this is MRC unretained __block only.
 556                  *  // ARC retained __block is handled by the copy helper directly.
 557                  *  __block id object;
 558                  *  __block void (^object)(void);
 559                  *  [^{ object; } copy];
 560                  ********/
 561
 562                 *dest = object;
 563                 break;
 564
 565         case BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_OBJECT | BLOCK_FIELD_IS_WEAK:
 566         case BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_BLOCK  | BLOCK_FIELD_IS_WEAK:
 567                 /*******
 568                  *  // copy the actual field held in the __block container
 569                  *  // Note this __weak is old GC-weak/MRC-unretained.
 570                  *  // ARC-style __weak is handled by the copy helper directly.
 571                  *  __weak __block id object;
 572                  *  __weak __block void (^object)(void);
 573                  *  [^{ object; } copy];
 574                  ********/
 575
 576                 *dest = object;
 577                 break;
 578
 579         default:
 580                 break;
 581         }
 582 }
 583
 584 // When Blocks or Block_byrefs hold objects their destroy helper routines call this entry point
 585 // to help dispose of the contents
 586 void
 587 _Block_object_dispose(const void *object, const int flags)
 588 {
 589         switch (os_assumes(flags & BLOCK_ALL_COPY_DISPOSE_FLAGS)) {
 590         case BLOCK_FIELD_IS_BYREF | BLOCK_FIELD_IS_WEAK:
 591         case BLOCK_FIELD_IS_BYREF:
 592                 // get rid of the __block data structure held in a Block
 593                 _Block_byref_release(object);
 594                 break;
 595         case BLOCK_FIELD_IS_BLOCK:
 596                 _Block_release(object);
 597                 break;
 598         case BLOCK_FIELD_IS_OBJECT:
 599                 _Block_release_object(object);
 600                 break;
 601         case BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_OBJECT:
 602         case BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_BLOCK:
 603         case BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_OBJECT | BLOCK_FIELD_IS_WEAK:
 604         case BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_BLOCK  | BLOCK_FIELD_IS_WEAK:
 605                 break;
 606         default:
 607                 break;
 608         }
 609 }
 610
 611
 612 // Workaround for <rdar://26015603> dylib with no __DATA segment fails to rebase
 613 __attribute__((used))
 614 static int let_there_be_data = 42;
 615
 616 #undef malloc
 617 #undef free