CFBase.c

   1 /*
   2  * Copyright (c) 2011 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 /*      CFBase.c
  25         Copyright (c) 1998-2011, Apple Inc. All rights reserved.
  26         Responsibility: Christopher Kane
  27 */
  28
  29 #include <CoreFoundation/CFBase.h>
  30 #include "CFInternal.h"
  31 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED || DEPLOYMENT_TARGET_LINUX || DEPLOYMENT_TARGET_FREEBSD
  32     #include <pthread.h>
  33 #endif
  34 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
  35     #include <malloc/malloc.h>
  36     extern size_t malloc_good_size(size_t size);
  37     #include <mach/mach.h>
  38     #include <dlfcn.h>
  39 #endif
  40 #include <stdlib.h>
  41 #include <string.h>
  42
  43 // -------- -------- -------- -------- -------- -------- -------- --------
  44
  45 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
  46 // CFAllocator structure must match struct _malloc_zone_t!
  47 // The first two reserved fields in struct _malloc_zone_t are for us with CFRuntimeBase
  48 #endif
  49
  50
  51 struct __CFAllocator {
  52     CFRuntimeBase _base;
  53 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
  54     size_t      (*size)(struct _malloc_zone_t *zone, const void *ptr); /* returns the size of a block or 0 if not in this zone; must be fast, especially for negative answers */
  55     void        *(*malloc)(struct _malloc_zone_t *zone, size_t size);
  56     void        *(*calloc)(struct _malloc_zone_t *zone, size_t num_items, size_t size); /* same as malloc, but block returned is set to zero */
  57     void        *(*valloc)(struct _malloc_zone_t *zone, size_t size); /* same as malloc, but block returned is set to zero and is guaranteed to be page aligned */
  58     void        (*free)(struct _malloc_zone_t *zone, void *ptr);
  59     void        *(*realloc)(struct _malloc_zone_t *zone, void *ptr, size_t size);
  60     void        (*destroy)(struct _malloc_zone_t *zone); /* zone is destroyed and all memory reclaimed */
  61     const char  *zone_name;
  62
  63     /* Optional batch callbacks; these may be NULL */
  64     unsigned    (*batch_malloc)(struct _malloc_zone_t *zone, size_t size, void **results, unsigned num_requested); /* given a size, returns pointers capable of holding that size; returns the number of pointers allocated (maybe 0 or less than num_requested) */
  65     void        (*batch_free)(struct _malloc_zone_t *zone, void **to_be_freed, unsigned num_to_be_freed); /* frees all the pointers in to_be_freed; note that to_be_freed may be overwritten during the process */
  66
  67     struct malloc_introspection_t       *introspect;
  68     unsigned    version;
  69
  70     /* aligned memory allocation. The callback may be NULL. */
  71         void *(*memalign)(struct _malloc_zone_t *zone, size_t alignment, size_t size);
  72
  73     /* free a pointer known to be in zone and known to have the given size. The callback may be NULL. */
  74     void (*free_definite_size)(struct _malloc_zone_t *zone, void *ptr, size_t size);
  75 #endif
  76     CFAllocatorRef _allocator;
  77     CFAllocatorContext _context;
  78 };
  79
  80 CF_INLINE CFAllocatorRetainCallBack __CFAllocatorGetRetainFunction(const CFAllocatorContext *context) {
  81     CFAllocatorRetainCallBack retval = NULL;
  82         retval = context->retain;
  83     return retval;
  84 }
  85
  86 CF_INLINE CFAllocatorReleaseCallBack __CFAllocatorGetReleaseFunction(const CFAllocatorContext *context) {
  87     CFAllocatorReleaseCallBack retval = NULL;
  88         retval = context->release;
  89     return retval;
  90 }
  91
  92 CF_INLINE CFAllocatorCopyDescriptionCallBack __CFAllocatorGetCopyDescriptionFunction(const CFAllocatorContext *context) {
  93     CFAllocatorCopyDescriptionCallBack retval = NULL;
  94         retval = context->copyDescription;
  95     return retval;
  96 }
  97
  98 CF_INLINE CFAllocatorAllocateCallBack __CFAllocatorGetAllocateFunction(const CFAllocatorContext *context) {
  99     CFAllocatorAllocateCallBack retval = NULL;
 100         retval = context->allocate;
 101     return retval;
 102 }
 103
 104 CF_INLINE CFAllocatorReallocateCallBack __CFAllocatorGetReallocateFunction(const CFAllocatorContext *context) {
 105     CFAllocatorReallocateCallBack retval = NULL;
 106         retval = context->reallocate;
 107     return retval;
 108 }
 109
 110 CF_INLINE CFAllocatorDeallocateCallBack __CFAllocatorGetDeallocateFunction(const CFAllocatorContext *context) {
 111     CFAllocatorDeallocateCallBack retval = NULL;
 112         retval = context->deallocate;
 113     return retval;
 114 }
 115
 116 CF_INLINE CFAllocatorPreferredSizeCallBack __CFAllocatorGetPreferredSizeFunction(const CFAllocatorContext *context) {
 117     CFAllocatorPreferredSizeCallBack retval = NULL;
 118         retval = context->preferredSize;
 119     return retval;
 120 }
 121
 122 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 123
 124 __private_extern__ void __CFAllocatorDeallocate(CFTypeRef cf);
 125
 126 static kern_return_t __CFAllocatorZoneIntrospectNoOp(void) {
 127     return 0;
 128 }
 129
 130 static boolean_t __CFAllocatorZoneIntrospectTrue(void) {
 131     return 1;
 132 }
 133
 134 static size_t __CFAllocatorCustomSize(malloc_zone_t *zone, const void *ptr) {
 135     return 0;
 136
 137     // The only way to implement this with a version 0 allocator would be
 138     // for CFAllocator to keep track of all blocks allocated itself, which
 139     // could be done, but would be bad for performance, so we don't do it.
 140     //    size_t (*size)(struct _malloc_zone_t *zone, const void *ptr);
 141     /* returns the size of a block or 0 if not in this zone;
 142      * must be fast, especially for negative answers */
 143 }
 144
 145 static void *__CFAllocatorCustomMalloc(malloc_zone_t *zone, size_t size) {
 146     CFAllocatorRef allocator = (CFAllocatorRef)zone;
 147     return CFAllocatorAllocate(allocator, size, 0);
 148 }
 149
 150 static void *__CFAllocatorCustomCalloc(malloc_zone_t *zone, size_t num_items, size_t size) {
 151     CFAllocatorRef allocator = (CFAllocatorRef)zone;
 152     void *newptr = CFAllocatorAllocate(allocator, size, 0);
 153     if (newptr) memset(newptr, 0, size);
 154     return newptr;
 155 }
 156
 157 static void *__CFAllocatorCustomValloc(malloc_zone_t *zone, size_t size) {
 158     CFAllocatorRef allocator = (CFAllocatorRef)zone;
 159     if (size >= ULONG_MAX - 2 * vm_page_size) return NULL; // avoid integer overflow plus don't allow all pages to be allocated either
 160     void *newptr = CFAllocatorAllocate(allocator, size + vm_page_size, 0);
 161     newptr = (void *)round_page((uintptr_t)newptr);
 162     return newptr;
 163 }
 164
 165 static void __CFAllocatorCustomFree(malloc_zone_t *zone, void *ptr) {
 166     CFAllocatorRef allocator = (CFAllocatorRef)zone;
 167     CFAllocatorDeallocate(allocator, ptr);
 168 }
 169
 170 static void *__CFAllocatorCustomRealloc(malloc_zone_t *zone, void *ptr, size_t size) {
 171     CFAllocatorRef allocator = (CFAllocatorRef)zone;
 172     return CFAllocatorReallocate(allocator, ptr, size, 0);
 173 }
 174
 175 static void __CFAllocatorCustomDestroy(malloc_zone_t *zone) {
 176     CFAllocatorRef allocator = (CFAllocatorRef)zone;
 177     // !!! we do it, and caller of malloc_destroy_zone() assumes
 178     // COMPLETE responsibility for the result; NO Apple library
 179     // code should be modified as a result of discovering that
 180     // some activity results in inconveniences to developers
 181     // trying to use malloc_destroy_zone() with a CFAllocatorRef;
 182     // that's just too bad for them.
 183     __CFAllocatorDeallocate(allocator);
 184 }
 185
 186 static size_t __CFAllocatorCustomGoodSize(malloc_zone_t *zone, size_t size) {
 187     CFAllocatorRef allocator = (CFAllocatorRef)zone;
 188     return CFAllocatorGetPreferredSizeForSize(allocator, size, 0);
 189 }
 190
 191 static struct malloc_introspection_t __CFAllocatorZoneIntrospect = {
 192     (void *)__CFAllocatorZoneIntrospectNoOp,
 193     (void *)__CFAllocatorCustomGoodSize,
 194     (void *)__CFAllocatorZoneIntrospectTrue,
 195     (void *)__CFAllocatorZoneIntrospectNoOp,
 196     (void *)__CFAllocatorZoneIntrospectNoOp,
 197     (void *)__CFAllocatorZoneIntrospectNoOp,
 198     (void *)__CFAllocatorZoneIntrospectNoOp,
 199     (void *)__CFAllocatorZoneIntrospectNoOp
 200 };
 201
 202 static size_t __CFAllocatorNullSize(malloc_zone_t *zone, const void *ptr) {
 203     return 0;
 204 }
 205
 206 static void * __CFAllocatorNullMalloc(malloc_zone_t *zone, size_t size) {
 207     return NULL;
 208 }
 209
 210 static void * __CFAllocatorNullCalloc(malloc_zone_t *zone, size_t num_items, size_t size) {
 211     return NULL;
 212 }
 213
 214 static void * __CFAllocatorNullValloc(malloc_zone_t *zone, size_t size) {
 215     return NULL;
 216 }
 217
 218 static void __CFAllocatorNullFree(malloc_zone_t *zone, void *ptr) {
 219 }
 220
 221 static void * __CFAllocatorNullRealloc(malloc_zone_t *zone, void *ptr, size_t size) {
 222     return NULL;
 223 }
 224
 225 static void __CFAllocatorNullDestroy(malloc_zone_t *zone) {
 226 }
 227
 228 static size_t __CFAllocatorNullGoodSize(malloc_zone_t *zone, size_t size) {
 229     return size;
 230 }
 231
 232 static struct malloc_introspection_t __CFAllocatorNullZoneIntrospect = {
 233     (void *)__CFAllocatorZoneIntrospectNoOp,
 234     (void *)__CFAllocatorNullGoodSize,
 235     (void *)__CFAllocatorZoneIntrospectTrue,
 236     (void *)__CFAllocatorZoneIntrospectNoOp,
 237     (void *)__CFAllocatorZoneIntrospectNoOp,
 238     (void *)__CFAllocatorZoneIntrospectNoOp,
 239     (void *)__CFAllocatorZoneIntrospectNoOp,
 240     (void *)__CFAllocatorZoneIntrospectNoOp
 241 };
 242
 243 static void *__CFAllocatorSystemAllocate(CFIndex size, CFOptionFlags hint, void *info) {
 244     return malloc_zone_malloc(info, size);
 245 }
 246
 247 static void *__CFAllocatorSystemReallocate(void *ptr, CFIndex newsize, CFOptionFlags hint, void *info) {
 248     return malloc_zone_realloc(info, ptr, newsize);
 249 }
 250
 251 static void __CFAllocatorSystemDeallocate(void *ptr, void *info) {
 252 #if defined(DEBUG)
 253     size_t size = malloc_size(ptr);
 254     if (size) memset(ptr, 0xCC, size);
 255 #endif
 256     malloc_zone_free(info, ptr);
 257 }
 258
 259 #endif
 260
 261 #if DEPLOYMENT_TARGET_WINDOWS || DEPLOYMENT_TARGET_LINUX || DEPLOYMENT_TARGET_FREEBSD
 262 static void *__CFAllocatorSystemAllocate(CFIndex size, CFOptionFlags hint, void *info) {
 263     return malloc(size);
 264 }
 265
 266 static void *__CFAllocatorSystemReallocate(void *ptr, CFIndex newsize, CFOptionFlags hint, void *info) {
 267     return realloc(ptr, newsize);
 268 }
 269
 270 static void __CFAllocatorSystemDeallocate(void *ptr, void *info) {
 271     free(ptr);
 272 }
 273 #endif
 274
 275 static void *__CFAllocatorNullAllocate(CFIndex size, CFOptionFlags hint, void *info) {
 276     return NULL;
 277 }
 278
 279 static void *__CFAllocatorNullReallocate(void *ptr, CFIndex newsize, CFOptionFlags hint, void *info) {
 280     return NULL;
 281 }
 282
 283 #if defined (__cplusplus)
 284 static void * __CFAllocatorCPPMalloc(CFIndex allocSize, CFOptionFlags hint, void *info)
 285 {
 286         return malloc(allocSize);
 287 }
 288 static void * __CFAllocatorCPPReAlloc(void *ptr, CFIndex newsize, CFOptionFlags hint, void *info)
 289 {
 290         return realloc(ptr, newsize);
 291 }
 292 static void __CFAllocatorCPPFree(void *ptr, void *info)
 293 {
 294         free(ptr);
 295 }
 296 #endif // C++
 297
 298
 299 static struct __CFAllocator __kCFAllocatorMalloc = {
 300     INIT_CFRUNTIME_BASE(),
 301 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 302     __CFAllocatorCustomSize,
 303     __CFAllocatorCustomMalloc,
 304     __CFAllocatorCustomCalloc,
 305     __CFAllocatorCustomValloc,
 306     __CFAllocatorCustomFree,
 307     __CFAllocatorCustomRealloc,
 308     __CFAllocatorNullDestroy,
 309     "kCFAllocatorMalloc",
 310     NULL,
 311     NULL,
 312     &__CFAllocatorZoneIntrospect,
 313     6,
 314     NULL,
 315     NULL,
 316 #endif
 317     NULL,       // _allocator
 318     // Using the malloc functions directly is a total cheat, but works (in C)
 319     // because the function signatures match in their common prefix of arguments.
 320     // This saves us one hop through an adaptor function.
 321 #if !defined (__cplusplus)
 322         {0, NULL, NULL, NULL, NULL, (void *)malloc, (void *)realloc, (void *)free, NULL}
 323 #else
 324         {0, NULL, NULL, NULL, NULL, __CFAllocatorCPPMalloc,__CFAllocatorCPPReAlloc, __CFAllocatorCPPFree, NULL}
 325 #endif // __cplusplus
 326 };
 327
 328 static struct __CFAllocator __kCFAllocatorMallocZone = {
 329     INIT_CFRUNTIME_BASE(),
 330 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 331     __CFAllocatorCustomSize,
 332     __CFAllocatorCustomMalloc,
 333     __CFAllocatorCustomCalloc,
 334     __CFAllocatorCustomValloc,
 335     __CFAllocatorCustomFree,
 336     __CFAllocatorCustomRealloc,
 337     __CFAllocatorNullDestroy,
 338     "kCFAllocatorMallocZone",
 339     NULL,
 340     NULL,
 341     &__CFAllocatorZoneIntrospect,
 342     6,
 343     NULL,
 344     NULL,
 345 #endif
 346     NULL,       // _allocator
 347     {0, NULL, NULL, NULL, NULL, __CFAllocatorSystemAllocate, __CFAllocatorSystemReallocate, __CFAllocatorSystemDeallocate, NULL}
 348 };
 349
 350 static struct __CFAllocator __kCFAllocatorSystemDefault = {
 351     INIT_CFRUNTIME_BASE(),
 352 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 353     __CFAllocatorCustomSize,
 354     __CFAllocatorCustomMalloc,
 355     __CFAllocatorCustomCalloc,
 356     __CFAllocatorCustomValloc,
 357     __CFAllocatorCustomFree,
 358     __CFAllocatorCustomRealloc,
 359     __CFAllocatorNullDestroy,
 360     "kCFAllocatorSystemDefault",
 361     NULL,
 362     NULL,
 363     &__CFAllocatorZoneIntrospect,
 364     6,
 365     NULL,
 366     NULL,
 367 #endif
 368     NULL,       // _allocator
 369     {0, NULL, NULL, NULL, NULL, __CFAllocatorSystemAllocate, __CFAllocatorSystemReallocate, __CFAllocatorSystemDeallocate, NULL}
 370 };
 371
 372 static struct __CFAllocator __kCFAllocatorNull = {
 373     INIT_CFRUNTIME_BASE(),
 374 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 375     __CFAllocatorNullSize,
 376     __CFAllocatorNullMalloc,
 377     __CFAllocatorNullCalloc,
 378     __CFAllocatorNullValloc,
 379     __CFAllocatorNullFree,
 380     __CFAllocatorNullRealloc,
 381     __CFAllocatorNullDestroy,
 382     "kCFAllocatorNull",
 383     NULL,
 384     NULL,
 385     &__CFAllocatorNullZoneIntrospect,
 386     6,
 387     NULL,
 388     NULL,
 389 #endif
 390     NULL,       // _allocator
 391     {0, NULL, NULL, NULL, NULL, __CFAllocatorNullAllocate, __CFAllocatorNullReallocate, NULL, NULL}
 392 };
 393
 394 const CFAllocatorRef kCFAllocatorDefault = NULL;
 395 const CFAllocatorRef kCFAllocatorSystemDefault = &__kCFAllocatorSystemDefault;
 396 const CFAllocatorRef kCFAllocatorMalloc = &__kCFAllocatorMalloc;
 397 const CFAllocatorRef kCFAllocatorMallocZone = &__kCFAllocatorMallocZone;
 398 const CFAllocatorRef kCFAllocatorNull = &__kCFAllocatorNull;
 399 const CFAllocatorRef kCFAllocatorUseContext = (CFAllocatorRef)0x03ab;
 400 #undef kCFAllocatorSystemDefaultGCRefZero
 401 #undef kCFAllocatorDefaultGCRefZero
 402 const CFAllocatorRef kCFAllocatorSystemDefaultGCRefZero = (CFAllocatorRef)0x03ad;
 403 const CFAllocatorRef kCFAllocatorDefaultGCRefZero = (CFAllocatorRef)0x03af;
 404
 405 static CFStringRef __CFAllocatorCopyDescription(CFTypeRef cf) {
 406     CFAllocatorRef self = (CFAllocatorRef)cf;
 407     CFAllocatorRef allocator = (kCFAllocatorUseContext == self->_allocator) ? self : self->_allocator;
 408     return CFStringCreateWithFormat(allocator, NULL, CFSTR("<CFAllocator %p [%p]>{info = %p}"), cf, allocator, self->_context.info);
 409 // CF: should use copyDescription function here to describe info field
 410 // remember to release value returned from copydescr function when this happens
 411 }
 412
 413 __private_extern__ CFAllocatorRef __CFAllocatorGetAllocator(CFTypeRef cf) {
 414     CFAllocatorRef allocator = (CFAllocatorRef)cf;
 415     return (kCFAllocatorUseContext == allocator->_allocator) ? allocator : allocator->_allocator;
 416 }
 417
 418 __private_extern__ void __CFAllocatorDeallocate(CFTypeRef cf) {
 419     CFAllocatorRef self = (CFAllocatorRef)cf;
 420     CFAllocatorRef allocator = self->_allocator;
 421     CFAllocatorReleaseCallBack releaseFunc = __CFAllocatorGetReleaseFunction(&self->_context);
 422     if (kCFAllocatorUseContext == allocator) {
 423         /* Rather a chicken and egg problem here, so we do things
 424            in the reverse order from what was done at create time. */
 425         CFAllocatorDeallocateCallBack deallocateFunc = __CFAllocatorGetDeallocateFunction(&self->_context);
 426         void *info = self->_context.info;
 427         if (NULL != deallocateFunc) {
 428             INVOKE_CALLBACK2(deallocateFunc, (void *)self, info);
 429         }
 430         if (NULL != releaseFunc) {
 431             INVOKE_CALLBACK1(releaseFunc, info);
 432         }
 433     } else {
 434         if (NULL != releaseFunc) {
 435             INVOKE_CALLBACK1(releaseFunc, self->_context.info);
 436         }
 437         if (!CF_IS_COLLECTABLE_ALLOCATOR(allocator)) CFAllocatorDeallocate(allocator, (void *)self);
 438     }
 439 }
 440
 441 static CFTypeID __kCFAllocatorTypeID = _kCFRuntimeNotATypeID;
 442
 443 static const CFRuntimeClass __CFAllocatorClass = {
 444     0,
 445     "CFAllocator",
 446     NULL,       // init
 447     NULL,       // copy
 448     NULL,
 449     NULL,       // equal
 450     NULL,       // hash
 451     NULL,       //
 452     __CFAllocatorCopyDescription
 453 };
 454
 455 __private_extern__ void __CFAllocatorInitialize(void) {
 456     __kCFAllocatorTypeID = _CFRuntimeRegisterClass(&__CFAllocatorClass);
 457
 458     _CFRuntimeSetInstanceTypeID(&__kCFAllocatorSystemDefault, __kCFAllocatorTypeID);
 459     __kCFAllocatorSystemDefault._base._cfisa = __CFISAForTypeID(__kCFAllocatorTypeID);
 460 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 461     __kCFAllocatorSystemDefault._context.info = (kCFUseCollectableAllocator ? objc_collectableZone() : malloc_default_zone());
 462 #endif
 463     __kCFAllocatorSystemDefault._allocator = kCFAllocatorSystemDefault;
 464
 465     _CFRuntimeSetInstanceTypeID(&__kCFAllocatorMalloc, __kCFAllocatorTypeID);
 466     __kCFAllocatorMalloc._base._cfisa = __CFISAForTypeID(__kCFAllocatorTypeID);
 467     __kCFAllocatorMalloc._allocator = kCFAllocatorSystemDefault;
 468
 469 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 470         _CFRuntimeSetInstanceTypeID(&__kCFAllocatorMallocZone, __kCFAllocatorTypeID);
 471     __kCFAllocatorMallocZone._base._cfisa = __CFISAForTypeID(__kCFAllocatorTypeID);
 472     __kCFAllocatorMallocZone._allocator = kCFAllocatorSystemDefault;
 473     __kCFAllocatorMallocZone._context.info = malloc_default_zone();
 474 #endif
 475
 476     _CFRuntimeSetInstanceTypeID(&__kCFAllocatorNull, __kCFAllocatorTypeID);
 477     __kCFAllocatorNull._base._cfisa = __CFISAForTypeID(__kCFAllocatorTypeID);
 478     __kCFAllocatorNull._allocator = kCFAllocatorSystemDefault;
 479
 480 }
 481
 482 CFTypeID CFAllocatorGetTypeID(void) {
 483     return __kCFAllocatorTypeID;
 484 }
 485
 486 CFAllocatorRef CFAllocatorGetDefault(void) {
 487     return __CFGetDefaultAllocator();
 488 }
 489
 490 void CFAllocatorSetDefault(CFAllocatorRef allocator) {
 491     if (kCFAllocatorSystemDefaultGCRefZero == allocator || kCFAllocatorDefaultGCRefZero == allocator) {
 492         HALT;
 493     }
 494     CFAllocatorRef current = __CFGetDefaultAllocator();
 495 #if defined(DEBUG)
 496     if (NULL != allocator) {
 497         __CFGenericValidateType(allocator, __kCFAllocatorTypeID);
 498     }
 499 #endif
 500 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 501     if (allocator && allocator->_base._cfisa != __CFISAForTypeID(__kCFAllocatorTypeID)) {       // malloc_zone_t *
 502         return;         // require allocator to this function to be an allocator
 503     }
 504 #endif
 505     if (NULL != allocator && allocator != current) {
 506         if (current) CFRelease(current);
 507         CFRetain(allocator);
 508         // We retain an extra time so that anything set as the default
 509         // allocator never goes away.
 510         CFRetain(allocator);
 511         _CFSetTSD(__CFTSDKeyAllocator, (void *)allocator, NULL);
 512     }
 513 }
 514
 515 static CFAllocatorRef __CFAllocatorCreate(CFAllocatorRef allocator, CFAllocatorContext *context) {
 516     struct __CFAllocator *memory = NULL;
 517     CFAllocatorRetainCallBack retainFunc;
 518     CFAllocatorAllocateCallBack allocateFunc;
 519     void *retainedInfo;
 520 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 521     if (allocator && kCFAllocatorUseContext != allocator && allocator->_base._cfisa != __CFISAForTypeID(__kCFAllocatorTypeID)) {        // malloc_zone_t *
 522         return NULL;    // require allocator to this function to be an allocator
 523     }
 524 #endif
 525     retainFunc = context->retain;
 526     FAULT_CALLBACK((void **)&retainFunc);
 527     allocateFunc = context->allocate;
 528     FAULT_CALLBACK((void **)&allocateFunc);
 529     if (NULL != retainFunc) {
 530         retainedInfo = (void *)INVOKE_CALLBACK1(retainFunc, context->info);
 531     } else {
 532         retainedInfo = context->info;
 533     }
 534     // We don't use _CFRuntimeCreateInstance()
 535     if (kCFAllocatorUseContext == allocator) {
 536         memory = NULL;
 537         if (allocateFunc) {
 538                 memory = (struct __CFAllocator *)INVOKE_CALLBACK3(allocateFunc, sizeof(struct __CFAllocator), 0, retainedInfo);
 539         }
 540         if (NULL == memory) {
 541             return NULL;
 542         }
 543     } else {
 544         allocator = (NULL == allocator) ? __CFGetDefaultAllocator() : allocator;
 545         memory = (struct __CFAllocator *)CFAllocatorAllocate(allocator, sizeof(struct __CFAllocator), __kCFAllocatorGCObjectMemory);
 546         if (NULL == memory) {
 547             return NULL;
 548         }
 549         if (__CFOASafe) __CFSetLastAllocationEventName(memory, "CFAllocator");
 550     }
 551     memset(memory, 0, sizeof(CFRuntimeBase));
 552     memory->_base._cfisa = 0;
 553 #if __LP64__
 554     memory->_base._rc = 1;
 555 #else
 556     memory->_base._cfinfo[CF_RC_BITS] = 1;
 557 #endif
 558     memory->_base._cfinfo[CF_INFO_BITS] = 0;
 559     _CFRuntimeSetInstanceTypeID(memory, __kCFAllocatorTypeID);
 560     memory->_base._cfisa = __CFISAForTypeID(__kCFAllocatorTypeID);
 561 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 562     memory->size = __CFAllocatorCustomSize;
 563     memory->malloc = __CFAllocatorCustomMalloc;
 564     memory->calloc = __CFAllocatorCustomCalloc;
 565     memory->valloc = __CFAllocatorCustomValloc;
 566     memory->free = __CFAllocatorCustomFree;
 567     memory->realloc = __CFAllocatorCustomRealloc;
 568     memory->destroy = __CFAllocatorCustomDestroy;
 569     memory->zone_name = "Custom CFAllocator";
 570     memory->batch_malloc = NULL;
 571     memory->batch_free = NULL;
 572     memory->introspect = &__CFAllocatorZoneIntrospect;
 573     memory->version = 6;
 574     memory->memalign = NULL;
 575     memory->free_definite_size = NULL;
 576 #endif
 577     memory->_allocator = allocator;
 578     memory->_context.version = context->version;
 579     memory->_context.info = retainedInfo;
 580     memory->_context.retain = retainFunc;
 581     memory->_context.release = context->release;
 582     FAULT_CALLBACK((void **)&(memory->_context.release));
 583     memory->_context.copyDescription = context->copyDescription;
 584     FAULT_CALLBACK((void **)&(memory->_context.copyDescription));
 585     memory->_context.allocate = allocateFunc;
 586     memory->_context.reallocate = context->reallocate;
 587     FAULT_CALLBACK((void **)&(memory->_context.reallocate));
 588     memory->_context.deallocate = context->deallocate;
 589     FAULT_CALLBACK((void **)&(memory->_context.deallocate));
 590     memory->_context.preferredSize = context->preferredSize;
 591     FAULT_CALLBACK((void **)&(memory->_context.preferredSize));
 592
 593     return memory;
 594 }
 595
 596 CFAllocatorRef CFAllocatorCreate(CFAllocatorRef allocator, CFAllocatorContext *context) {
 597     return __CFAllocatorCreate(allocator, context);
 598 }
 599
 600 void *CFAllocatorAllocate(CFAllocatorRef allocator, CFIndex size, CFOptionFlags hint) {
 601     CFAllocatorAllocateCallBack allocateFunc;
 602     void *newptr = NULL;
 603
 604     Boolean initialRefcountOne = true;
 605     if (kCFAllocatorSystemDefaultGCRefZero == allocator) {
 606         allocator = kCFAllocatorSystemDefault;
 607         initialRefcountOne = false;
 608     } else if (kCFAllocatorDefaultGCRefZero == allocator) {
 609         // Under GC, we can't use just any old allocator when the GCRefZero allocator was requested
 610         allocator = kCFUseCollectableAllocator ? kCFAllocatorSystemDefault : __CFGetDefaultAllocator();
 611         if (CF_IS_COLLECTABLE_ALLOCATOR(allocator)) initialRefcountOne = false;
 612     } else if (NULL == allocator) {
 613         allocator = __CFGetDefaultAllocator();
 614     }
 615
 616 #if defined(DEBUG) && (DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED)
 617     if (allocator->_base._cfisa == __CFISAForTypeID(__kCFAllocatorTypeID)) {
 618         __CFGenericValidateType(allocator, __kCFAllocatorTypeID);
 619     }
 620 #else
 621     __CFGenericValidateType(allocator, __kCFAllocatorTypeID);
 622 #endif
 623     if (0 == size) return NULL;
 624 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 625     if (allocator->_base._cfisa != __CFISAForTypeID(__kCFAllocatorTypeID)) {    // malloc_zone_t *
 626         return malloc_zone_malloc((malloc_zone_t *)allocator, size);
 627     }
 628 #endif
 629     if (CF_IS_COLLECTABLE_ALLOCATOR(allocator)) {
 630         newptr = auto_zone_allocate_object((auto_zone_t*)allocator->_context.info, size, CF_GET_GC_MEMORY_TYPE(hint), initialRefcountOne, false);
 631     } else {
 632         newptr = NULL;
 633         allocateFunc = __CFAllocatorGetAllocateFunction(&allocator->_context);
 634         if (allocateFunc) {
 635                 newptr = (void *)INVOKE_CALLBACK3(allocateFunc, size, hint, allocator->_context.info);
 636         }
 637     }
 638     return newptr;
 639 }
 640
 641 void *CFAllocatorReallocate(CFAllocatorRef allocator, void *ptr, CFIndex newsize, CFOptionFlags hint) {
 642     CFAllocatorAllocateCallBack allocateFunc;
 643     CFAllocatorReallocateCallBack reallocateFunc;
 644     CFAllocatorDeallocateCallBack deallocateFunc;
 645     void *newptr;
 646
 647     if (kCFAllocatorSystemDefaultGCRefZero == allocator) {
 648         allocator = kCFAllocatorSystemDefault;
 649     } else if (kCFAllocatorDefaultGCRefZero == allocator) {
 650         // Under GC, we can't use just any old allocator when the GCRefZero allocator was requested
 651         allocator = kCFUseCollectableAllocator ? kCFAllocatorSystemDefault : __CFGetDefaultAllocator();
 652     } else if (NULL == allocator) {
 653         allocator = __CFGetDefaultAllocator();
 654     }
 655
 656 #if defined(DEBUG) && (DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED)
 657     if (allocator->_base._cfisa == __CFISAForTypeID(__kCFAllocatorTypeID)) {
 658         __CFGenericValidateType(allocator, __kCFAllocatorTypeID);
 659     }
 660 #else
 661     __CFGenericValidateType(allocator, __kCFAllocatorTypeID);
 662 #endif
 663     if (NULL == ptr && 0 < newsize) {
 664 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 665         if (allocator->_base._cfisa != __CFISAForTypeID(__kCFAllocatorTypeID)) {        // malloc_zone_t *
 666             return malloc_zone_malloc((malloc_zone_t *)allocator, newsize);
 667         }
 668 #endif
 669         newptr = NULL;
 670         allocateFunc = __CFAllocatorGetAllocateFunction(&allocator->_context);
 671         if (allocateFunc) {
 672                 newptr = (void *)INVOKE_CALLBACK3(allocateFunc, newsize, hint, allocator->_context.info);
 673         }
 674         return newptr;
 675     }
 676     if (NULL != ptr && 0 == newsize) {
 677 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 678         if (allocator->_base._cfisa != __CFISAForTypeID(__kCFAllocatorTypeID)) {        // malloc_zone_t *
 679 #if defined(DEBUG)
 680             size_t size = malloc_size(ptr);
 681             if (size) memset(ptr, 0xCC, size);
 682 #endif
 683             malloc_zone_free((malloc_zone_t *)allocator, ptr);
 684             return NULL;
 685         }
 686 #endif
 687         deallocateFunc = __CFAllocatorGetDeallocateFunction(&allocator->_context);
 688         if (NULL != deallocateFunc) {
 689             INVOKE_CALLBACK2(deallocateFunc, ptr, allocator->_context.info);
 690         }
 691         return NULL;
 692     }
 693     if (NULL == ptr && 0 == newsize) return NULL;
 694 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 695     if (allocator->_base._cfisa != __CFISAForTypeID(__kCFAllocatorTypeID)) {    // malloc_zone_t *
 696         return malloc_zone_realloc((malloc_zone_t *)allocator, ptr, newsize);
 697     }
 698 #endif
 699     reallocateFunc = __CFAllocatorGetReallocateFunction(&allocator->_context);
 700     if (NULL == reallocateFunc) return NULL;
 701     newptr = (void *)INVOKE_CALLBACK4(reallocateFunc, ptr, newsize, hint, allocator->_context.info);
 702     return newptr;
 703 }
 704
 705 void CFAllocatorDeallocate(CFAllocatorRef allocator, void *ptr) {
 706     CFAllocatorDeallocateCallBack deallocateFunc;
 707
 708     if (kCFAllocatorSystemDefaultGCRefZero == allocator) {
 709         if (_CFAllocatorIsGCRefZero(allocator)) return;
 710         allocator = kCFAllocatorSystemDefault;
 711     } else if (kCFAllocatorDefaultGCRefZero == allocator) {
 712         // Under GC, we can't use just any old allocator when the GCRefZero allocator was requested
 713         allocator = kCFUseCollectableAllocator ? kCFAllocatorSystemDefault : __CFGetDefaultAllocator();
 714         if (CF_IS_COLLECTABLE_ALLOCATOR(allocator)) return;
 715     } else if (NULL == allocator) {
 716         allocator = __CFGetDefaultAllocator();
 717     }
 718
 719 #if defined(DEBUG) && (DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED)
 720     if (allocator->_base._cfisa == __CFISAForTypeID(__kCFAllocatorTypeID)) {
 721         __CFGenericValidateType(allocator, __kCFAllocatorTypeID);
 722     }
 723 #else
 724     __CFGenericValidateType(allocator, __kCFAllocatorTypeID);
 725 #endif
 726 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 727     if (allocator->_base._cfisa != __CFISAForTypeID(__kCFAllocatorTypeID)) {    // malloc_zone_t *
 728 #if defined(DEBUG)
 729         size_t size = malloc_size(ptr);
 730         if (size) memset(ptr, 0xCC, size);
 731 #endif
 732         return malloc_zone_free((malloc_zone_t *)allocator, ptr);
 733     }
 734 #endif
 735     deallocateFunc = __CFAllocatorGetDeallocateFunction(&allocator->_context);
 736     if (NULL != ptr && NULL != deallocateFunc) {
 737         INVOKE_CALLBACK2(deallocateFunc, ptr, allocator->_context.info);
 738     }
 739 }
 740
 741 CFIndex CFAllocatorGetPreferredSizeForSize(CFAllocatorRef allocator, CFIndex size, CFOptionFlags hint) {
 742     CFAllocatorPreferredSizeCallBack prefFunc;
 743     CFIndex newsize = 0;
 744
 745     if (kCFAllocatorSystemDefaultGCRefZero == allocator) {
 746         allocator = kCFAllocatorSystemDefault;
 747     } else if (kCFAllocatorDefaultGCRefZero == allocator) {
 748         // Under GC, we can't use just any old allocator when the GCRefZero allocator was requested
 749         allocator = kCFUseCollectableAllocator ? kCFAllocatorSystemDefault : __CFGetDefaultAllocator();
 750     } else if (NULL == allocator) {
 751         allocator = __CFGetDefaultAllocator();
 752     }
 753
 754 #if defined(DEBUG) && (DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED)
 755     if (allocator->_base._cfisa == __CFISAForTypeID(__kCFAllocatorTypeID)) {
 756         __CFGenericValidateType(allocator, __kCFAllocatorTypeID);
 757     }
 758 #else
 759     __CFGenericValidateType(allocator, __kCFAllocatorTypeID);
 760 #endif
 761 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 762     if (allocator->_base._cfisa != __CFISAForTypeID(__kCFAllocatorTypeID)) {    // malloc_zone_t *
 763         return malloc_good_size(size);
 764     }
 765 #endif
 766     prefFunc = __CFAllocatorGetPreferredSizeFunction(&allocator->_context);
 767     if (0 < size && NULL != prefFunc) {
 768         newsize = (CFIndex)(INVOKE_CALLBACK3(prefFunc, size, hint, allocator->_context.info));
 769     }
 770     if (newsize < size) newsize = size;
 771     return newsize;
 772 }
 773
 774 void CFAllocatorGetContext(CFAllocatorRef allocator, CFAllocatorContext *context) {
 775     if (kCFAllocatorSystemDefaultGCRefZero == allocator) {
 776         allocator = kCFAllocatorSystemDefault;
 777     } else if (kCFAllocatorDefaultGCRefZero == allocator) {
 778         // Under GC, we can't use just any old allocator when the GCRefZero allocator was requested
 779         allocator = kCFUseCollectableAllocator ? kCFAllocatorSystemDefault : __CFGetDefaultAllocator();
 780     } else if (NULL == allocator) {
 781         allocator = __CFGetDefaultAllocator();
 782     }
 783
 784 #if defined(DEBUG) && (DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED)
 785     if (allocator->_base._cfisa == __CFISAForTypeID(__kCFAllocatorTypeID)) {
 786         __CFGenericValidateType(allocator, __kCFAllocatorTypeID);
 787     }
 788 #else
 789     __CFGenericValidateType(allocator, __kCFAllocatorTypeID);
 790 #endif
 791     CFAssert1(0 == context->version, __kCFLogAssertion, "%s(): context version not initialized to 0", __PRETTY_FUNCTION__);
 792 #if DEPLOYMENT_TARGET_MACOSX || DEPLOYMENT_TARGET_EMBEDDED
 793     if (allocator->_base._cfisa != __CFISAForTypeID(__kCFAllocatorTypeID)) {    // malloc_zone_t *
 794         return;
 795     }
 796 #endif
 797     context->version = 0;
 798     context->info = allocator->_context.info;
 799     context->retain = __CFAllocatorGetRetainFunction(&allocator->_context);
 800     context->release = __CFAllocatorGetReleaseFunction(&allocator->_context);
 801     context->copyDescription = __CFAllocatorGetCopyDescriptionFunction(&allocator->_context);
 802     context->allocate = __CFAllocatorGetAllocateFunction(&allocator->_context);
 803     context->reallocate = __CFAllocatorGetReallocateFunction(&allocator->_context);
 804     context->deallocate = __CFAllocatorGetDeallocateFunction(&allocator->_context);
 805     context->preferredSize = __CFAllocatorGetPreferredSizeFunction(&allocator->_context);
 806 #if SUPPORT_CFM
 807     context->retain = (void *)((uintptr_t)context->retain & ~0x3);
 808     context->release = (void *)((uintptr_t)context->release & ~0x3);
 809     context->copyDescription = (void *)((uintptr_t)context->copyDescription & ~0x3);
 810     context->allocate = (void *)((uintptr_t)context->allocate & ~0x3);
 811     context->reallocate = (void *)((uintptr_t)context->reallocate & ~0x3);
 812     context->deallocate = (void *)((uintptr_t)context->deallocate & ~0x3);
 813     context->preferredSize = (void *)((uintptr_t)context->preferredSize & ~0x3);
 814 #endif
 815 }
 816
 817 __private_extern__ void *_CFAllocatorAllocateGC(CFAllocatorRef allocator, CFIndex size, CFOptionFlags hint)
 818 {
 819     if (CF_IS_COLLECTABLE_ALLOCATOR(allocator))
 820         return auto_zone_allocate_object((auto_zone_t*)kCFAllocatorSystemDefault->_context.info, size, CF_GET_GC_MEMORY_TYPE(hint), false, false);
 821     else
 822         return CFAllocatorAllocate(allocator, size, hint);
 823 }
 824
 825 __private_extern__ void *_CFAllocatorReallocateGC(CFAllocatorRef allocator, void *ptr, CFIndex newsize, CFOptionFlags hint)
 826 {
 827     if (CF_IS_COLLECTABLE_ALLOCATOR(allocator)) {
 828         if (ptr && (newsize == 0)) {
 829             return NULL; // equivalent to _CFAllocatorDeallocateGC.
 830         }
 831         if (ptr == NULL) {
 832             return auto_zone_allocate_object((auto_zone_t*)kCFAllocatorSystemDefault->_context.info, newsize, CF_GET_GC_MEMORY_TYPE(hint), false, false); // eq. to _CFAllocator
 833         }
 834     }
 835     // otherwise, auto_realloc() now preserves layout type and refCount.
 836     return CFAllocatorReallocate(allocator, ptr, newsize, hint);
 837 }
 838
 839 __private_extern__ void _CFAllocatorDeallocateGC(CFAllocatorRef allocator, void *ptr)
 840 {
 841     // when running GC, don't deallocate.
 842     if (!CF_IS_COLLECTABLE_ALLOCATOR(allocator)) CFAllocatorDeallocate(allocator, ptr);
 843 }
 844
 845 // -------- -------- -------- -------- -------- -------- -------- --------
 846
 847
 848 CFRange __CFRangeMake(CFIndex loc, CFIndex len) {
 849     CFRange range;
 850     range.location = loc;
 851     range.length = len;
 852     return range;
 853 }
 854
 855
 856 struct __CFNull {
 857     CFRuntimeBase _base;
 858 };
 859
 860 static struct __CFNull __kCFNull = {
 861     INIT_CFRUNTIME_BASE()
 862 };
 863 const CFNullRef kCFNull = &__kCFNull;
 864
 865 static CFStringRef __CFNullCopyDescription(CFTypeRef cf) {
 866     return CFStringCreateWithFormat(kCFAllocatorSystemDefault, NULL, CFSTR("<CFNull %p [%p]>"), cf, CFGetAllocator(cf));
 867 }
 868
 869 static CFStringRef __CFNullCopyFormattingDescription(CFTypeRef cf, CFDictionaryRef formatOptions) {
 870     return (CFStringRef)CFRetain(CFSTR("null"));
 871 }
 872
 873 static void __CFNullDeallocate(CFTypeRef cf) {
 874     CFAssert(false, __kCFLogAssertion, "Deallocated CFNull!");
 875 }
 876
 877 static CFTypeID __kCFNullTypeID = _kCFRuntimeNotATypeID;
 878
 879 static const CFRuntimeClass __CFNullClass = {
 880     0,
 881     "CFNull",
 882     NULL,      // init
 883     NULL,      // copy
 884     __CFNullDeallocate,
 885     NULL,
 886     NULL,
 887     __CFNullCopyFormattingDescription,
 888     __CFNullCopyDescription
 889 };
 890
 891 __private_extern__ void __CFNullInitialize(void) {
 892     __kCFNullTypeID = _CFRuntimeRegisterClass(&__CFNullClass);
 893     _CFRuntimeSetInstanceTypeID(&__kCFNull, __kCFNullTypeID);
 894     __kCFNull._base._cfisa = __CFISAForTypeID(__kCFNullTypeID);
 895 }
 896
 897 CFTypeID CFNullGetTypeID(void) {
 898     return __kCFNullTypeID;
 899 }
 900
 901 void CFCollection_non_gc_storage_error(void) { }
 902
 903
 904 #if !SUPPORT_CFM
 905 void _CFRuntimeSetCFMPresent(void *addr) {
 906 }
 907 #endif
 908
 909 #if SUPPORT_CFM
 910
 911 static int hasCFM = 0;
 912
 913 void _CFRuntimeSetCFMPresent(void *addr) {
 914     hasCFM = 1;
 915 }
 916
 917 /* See comments below */
 918 __private_extern__ void __CF_FAULT_CALLBACK(void **ptr) {
 919     uintptr_t p = (uintptr_t)*ptr;
 920     if ((0 == p) || (p & 0x1)) return;
 921     if (0 == hasCFM || (0x90000000 <= p && p < 0xA0000000)) {
 922         *ptr = (void *)(p | 0x1);
 923     } else {
 924         uintptr_t known = ~0;
 925             Dl_info info;
 926             known = dladdr((void *)p, &info);
 927         *ptr = (void *)(p | (known ? 0x1 : 0x3));
 928     }
 929 }
 930
 931 /*
 932 Jump to callback function.  r2 is not saved and restored
 933 in the jump-to-CFM case, since we assume that dyld code
 934 never uses that register and that CF is dyld.
 935
 936 There are three states for (ptr & 0x3):
 937         0b00:   check not yet done (or not going to be done, and is a dyld func ptr)
 938         0b01:   check done, dyld function pointer
 939         0b11:   check done, CFM tvector pointer
 940 (but a NULL callback just stays NULL)
 941
 942 There may be up to 5 word-sized arguments. Floating point
 943 arguments can be done, but count as two word arguments.
 944 Return value can be integral or real.
 945 */
 946
 947 /* Keep this assembly at the bottom of the source file! */
 948
 949 __asm__ (
 950 ".text\n"
 951 "        .align 2\n"
 952 ".private_extern ___CF_INVOKE_CALLBACK\n"
 953 "___CF_INVOKE_CALLBACK:\n"
 954         "rlwinm r12,r3,0,0,29\n"
 955         "andi. r0,r3,0x2\n"
 956         "or r3,r4,r4\n"
 957         "or r4,r5,r5\n"
 958         "or r5,r6,r6\n"
 959         "or r6,r7,r7\n"
 960         "or r7,r8,r8\n"
 961         "beq- Lcall\n"
 962         "lwz r2,0x4(r12)\n"
 963         "lwz r12,0x0(r12)\n"
 964 "Lcall:  mtspr ctr,r12\n"
 965         "bctr\n");
 966
 967 #endif
 968
 969
 970 // void __HALT(void);
 971
 972 /* Keep this assembly at the bottom of the source file! */
 973
 974
 975 extern void __HALT() {
 976 #if defined(__ppc__)
 977     __asm__("trap");
 978 #elif defined(__i386__) || defined(__x86_64__)
 979 #if defined(_MSC_VER)
 980     __asm int 3;
 981 #else
 982     __asm__("int3");
 983 #endif
 984 #endif
 985 }
 986
 987