X-Git-Url: https://git.saurik.com/apple/objc4.git/blobdiff_plain/8070259c3936ee823b758fc1ad1645ae016ba500..refs/heads/master:/runtime/objc-private.h diff --git a/runtime/objc-private.h b/runtime/objc-private.h index 055fc72..c801ba0 100644 --- a/runtime/objc-private.h +++ b/runtime/objc-private.h @@ -21,8 +21,8 @@ * @APPLE_LICENSE_HEADER_END@ */ /* - * objc-private.h - * Copyright 1988-1996, NeXT Software, Inc. + * objc-private.h + * Copyright 1988-1996, NeXT Software, Inc. */ #ifndef _OBJC_PRIVATE_H_ @@ -39,87 +39,74 @@ #endif #define OBJC_TYPES_DEFINED 1 +#undef OBJC_OLD_DISPATCH_PROTOTYPES #define OBJC_OLD_DISPATCH_PROTOTYPES 0 #include // for nullptr_t #include #include +// An assert that's disabled for release builds but still ensures the expression compiles. +#ifdef NDEBUG +#define ASSERT(x) (void)sizeof(!(x)) +#else +#define ASSERT(x) assert(x) +#endif + +// `this` is never NULL in C++ unless we encounter UB, but checking for what's impossible +// is the point of these asserts, so disable the corresponding warning, and let's hope +// we will reach the assert despite the UB +#define ASSERT_THIS_NOT_NULL \ +_Pragma("clang diagnostic push") \ +_Pragma("clang diagnostic ignored \"-Wundefined-bool-conversion\"") \ +ASSERT(this) \ +_Pragma("clang diagnostic pop") + + struct objc_class; struct objc_object; +struct category_t; typedef struct objc_class *Class; typedef struct objc_object *id; +typedef struct classref *classref_t; namespace { - class SideTable; + struct SideTable; }; +#include "isa.h" -union isa_t -{ +union isa_t { isa_t() { } isa_t(uintptr_t value) : bits(value) { } - Class cls; uintptr_t bits; -#if SUPPORT_NONPOINTER_ISA - - // extra_rc must be the MSB-most field (so it matches carry/overflow flags) - // indexed must be the LSB (fixme or get rid of it) - // shiftcls must occupy the same bits that a real class pointer would - // bits + RC_ONE is equivalent to extra_rc + 1 - // RC_HALF is the high bit of extra_rc (i.e. half of its range) - - // future expansion: - // uintptr_t fast_rr : 1; // no r/r overrides - // uintptr_t lock : 2; // lock for atomic property, @synch - // uintptr_t extraBytes : 1; // allocated with extra bytes - -# if __arm64__ -# define ISA_MASK 0x00000001fffffff8ULL -# define ISA_MAGIC_MASK 0x000003fe00000001ULL -# define ISA_MAGIC_VALUE 0x000001a400000001ULL - struct { - uintptr_t indexed : 1; - uintptr_t has_assoc : 1; - uintptr_t has_cxx_dtor : 1; - uintptr_t shiftcls : 30; // MACH_VM_MAX_ADDRESS 0x1a0000000 - uintptr_t magic : 9; - uintptr_t weakly_referenced : 1; - uintptr_t deallocating : 1; - uintptr_t has_sidetable_rc : 1; - uintptr_t extra_rc : 19; -# define RC_ONE (1ULL<<45) -# define RC_HALF (1ULL<<18) - }; +private: + // Accessing the class requires custom ptrauth operations, so + // force clients to go through setClass/getClass by making this + // private. + Class cls; -# elif __x86_64__ -# define ISA_MASK 0x00007ffffffffff8ULL -# define ISA_MAGIC_MASK 0x0000000000000001ULL -# define ISA_MAGIC_VALUE 0x0000000000000001ULL +public: +#if defined(ISA_BITFIELD) struct { - uintptr_t indexed : 1; - uintptr_t has_assoc : 1; - uintptr_t has_cxx_dtor : 1; - uintptr_t shiftcls : 44; // MACH_VM_MAX_ADDRESS 0x7fffffe00000 - uintptr_t weakly_referenced : 1; - uintptr_t deallocating : 1; - uintptr_t has_sidetable_rc : 1; - uintptr_t extra_rc : 14; -# define RC_ONE (1ULL<<50) -# define RC_HALF (1ULL<<13) + ISA_BITFIELD; // defined in isa.h }; -# else - // Available bits in isa field are architecture-specific. -# error unknown architecture -# endif - -// SUPPORT_NONPOINTER_ISA + bool isDeallocating() { + return extra_rc == 0 && has_sidetable_rc == 0; + } + void setDeallocating() { + extra_rc = 0; + has_sidetable_rc = 0; + } #endif + void setClass(Class cls, objc_object *obj); + Class getClass(bool authenticated); + Class getDecodedClass(bool authenticated); }; @@ -130,10 +117,15 @@ private: public: // ISA() assumes this is NOT a tagged pointer object - Class ISA(); + Class ISA(bool authenticated = false); + + // rawISA() assumes this is NOT a tagged pointer object or a non pointer ISA + Class rawISA(); // getIsa() allows this to be a tagged pointer object Class getIsa(); + + uintptr_t isaBits() const; // initIsa() should be used to init the isa of new objects only. // If this object already has an isa, use changeIsa() for correctness. @@ -141,17 +133,20 @@ public: // initClassIsa(): class objects // initProtocolIsa(): protocol objects // initIsa(): other objects - void initIsa(Class cls /*indexed=false*/); - void initClassIsa(Class cls /*indexed=maybe*/); - void initProtocolIsa(Class cls /*indexed=maybe*/); + void initIsa(Class cls /*nonpointer=false*/); + void initClassIsa(Class cls /*nonpointer=maybe*/); + void initProtocolIsa(Class cls /*nonpointer=maybe*/); void initInstanceIsa(Class cls, bool hasCxxDtor); // changeIsa() should be used to change the isa of existing objects. // If this is a new object, use initIsa() for performance. Class changeIsa(Class newCls); - bool hasIndexedIsa(); + bool hasNonpointerIsa(); bool isTaggedPointer(); + bool isTaggedPointerOrNil(); + bool isBasicTaggedPointer(); + bool isExtTaggedPointer(); bool isClass(); // object may have associated objects? @@ -184,29 +179,43 @@ public: void rootDealloc(); private: - void initIsa(Class newCls, bool indexed, bool hasCxxDtor); + void initIsa(Class newCls, bool nonpointer, bool hasCxxDtor); // Slow paths for inline control id rootAutorelease2(); - bool overrelease_error(); + uintptr_t overrelease_error(); #if SUPPORT_NONPOINTER_ISA + // Controls what parts of root{Retain,Release} to emit/inline + // - Full means the full (slow) implementation + // - Fast means the fastpaths only + // - FastOrMsgSend means the fastpaths but checking whether we should call + // -retain/-release or Swift, for the usage of objc_{retain,release} + enum class RRVariant { + Full, + Fast, + FastOrMsgSend, + }; + // Unified retain count manipulation for nonpointer isa - id rootRetain(bool tryRetain, bool handleOverflow); - bool rootRelease(bool performDealloc, bool handleUnderflow); + inline id rootRetain(bool tryRetain, RRVariant variant); + inline bool rootRelease(bool performDealloc, RRVariant variant); id rootRetain_overflow(bool tryRetain); - bool rootRelease_underflow(bool performDealloc); + uintptr_t rootRelease_underflow(bool performDealloc); - void clearDeallocating_weak(); + void clearDeallocating_slow(); // Side table retain count overflow for nonpointer isa + struct SidetableBorrow { size_t borrowed, remaining; }; + void sidetable_lock(); void sidetable_unlock(); void sidetable_moveExtraRC_nolock(size_t extra_rc, bool isDeallocating, bool weaklyReferenced); bool sidetable_addExtraRC_nolock(size_t delta_rc); - bool sidetable_subExtraRC_nolock(size_t delta_rc); + SidetableBorrow sidetable_subExtraRC_nolock(size_t delta_rc); size_t sidetable_getExtraRC_nolock(); + void sidetable_clearExtraRC_nolock(); #endif // Side-table-only retain count @@ -216,16 +225,16 @@ private: bool sidetable_isWeaklyReferenced(); void sidetable_setWeaklyReferenced_nolock(); - id sidetable_retain(); - id sidetable_retain_slow(SideTable *table); + id sidetable_retain(bool locked = false); + id sidetable_retain_slow(SideTable& table); - bool sidetable_release(bool performDealloc = true); - bool sidetable_release_slow(SideTable *table, bool performDealloc = true); + uintptr_t sidetable_release(bool locked = false, bool performDealloc = true); + uintptr_t sidetable_release_slow(SideTable& table, bool performDealloc = true); bool sidetable_tryRetain(); uintptr_t sidetable_retainCount(); -#if !NDEBUG +#if DEBUG bool sidetable_present(); #endif }; @@ -250,7 +259,6 @@ typedef struct old_property *objc_property_t; #include "objc-os.h" #include "objc-abi.h" #include "objc-api.h" -#include "objc-auto.h" #include "objc-config.h" #include "objc-internal.h" #include "maptable.h" @@ -266,6 +274,8 @@ typedef struct old_property *objc_property_t; // Private headers +#include "objc-ptrauth.h" + #if __OBJC2__ #include "objc-runtime-new.h" #else @@ -277,80 +287,234 @@ typedef struct old_property *objc_property_t; #include "objc-loadmethod.h" +#define STRINGIFY(x) #x +#define STRINGIFY2(x) STRINGIFY(x) + __BEGIN_DECLS +namespace objc { -#if (defined(OBJC_NO_GC) && SUPPORT_GC) || \ - (!defined(OBJC_NO_GC) && !SUPPORT_GC) -# error OBJC_NO_GC and SUPPORT_GC inconsistent +struct SafeRanges { +private: + struct Range { + uintptr_t start; + uintptr_t end; + + inline bool contains(uintptr_t ptr) const { + uintptr_t m_start, m_end; +#if __arm64__ + // Force the compiler to use ldp + // we really don't want 2 loads and 2 jumps. + __asm__( +# if __LP64__ + "ldp %x[one], %x[two], [%x[src]]" +# else + "ldp %w[one], %w[two], [%x[src]]" +# endif + : [one] "=r" (m_start), [two] "=r" (m_end) + : [src] "r" (this) + ); +#else + m_start = start; + m_end = end; #endif + return m_start <= ptr && ptr < m_end; + } + }; + + struct Range shared_cache; + struct Range *ranges; + uint32_t count; + uint32_t size : 31; + uint32_t sorted : 1; + +public: + inline bool inSharedCache(uintptr_t ptr) const { + return shared_cache.contains(ptr); + } + inline bool contains(uint16_t witness, uintptr_t ptr) const { + return witness < count && ranges[witness].contains(ptr); + } + + inline void setSharedCacheRange(uintptr_t start, uintptr_t end) { + shared_cache = Range{start, end}; + add(start, end); + } + bool find(uintptr_t ptr, uint32_t &pos); + void add(uintptr_t start, uintptr_t end); + void remove(uintptr_t start, uintptr_t end); +}; + +extern struct SafeRanges dataSegmentsRanges; + +static inline bool inSharedCache(uintptr_t ptr) { + return dataSegmentsRanges.inSharedCache(ptr); +} + +} // objc + +struct header_info; + +// Split out the rw data from header info. For now put it in a huge array +// that more than exceeds the space needed. In future we'll just allocate +// this in the shared cache builder. +typedef struct header_info_rw { + + bool getLoaded() const { + return isLoaded; + } + + void setLoaded(bool v) { + isLoaded = v ? 1: 0; + } + + bool getAllClassesRealized() const { + return allClassesRealized; + } -#if SUPPORT_GC -# include - // PRIVATE_EXTERN is needed to help the compiler know "how" extern these are - PRIVATE_EXTERN extern int8_t UseGC; // equivalent to calling objc_collecting_enabled() - PRIVATE_EXTERN extern auto_zone_t *gc_zone; // the GC zone, or NULL if no GC - extern void objc_addRegisteredClass(Class c); - extern void objc_removeRegisteredClass(Class c); + void setAllClassesRealized(bool v) { + allClassesRealized = v ? 1: 0; + } + + header_info *getNext() const { + return (header_info *)(next << 2); + } + + void setNext(header_info *v) { + next = ((uintptr_t)v) >> 2; + } + +private: +#ifdef __LP64__ + uintptr_t isLoaded : 1; + uintptr_t allClassesRealized : 1; + uintptr_t next : 62; #else -# define UseGC NO -# define gc_zone NULL -# define objc_addRegisteredClass(c) do {} while(0) -# define objc_removeRegisteredClass(c) do {} while(0) - /* Uses of the following must be protected with UseGC. */ - extern id gc_unsupported_dont_call(); -# define auto_zone_allocate_object gc_unsupported_dont_call -# define auto_zone_retain gc_unsupported_dont_call -# define auto_zone_release gc_unsupported_dont_call -# define auto_zone_is_valid_pointer gc_unsupported_dont_call -# define auto_zone_write_barrier_memmove gc_unsupported_dont_call -# define AUTO_OBJECT_SCANNED 0 + uintptr_t isLoaded : 1; + uintptr_t allClassesRealized : 1; + uintptr_t next : 30; #endif +} header_info_rw; +struct header_info_rw* getPreoptimizedHeaderRW(const struct header_info *const hdr); -#define _objcHeaderIsReplacement(h) ((h)->info && ((h)->info->flags & OBJC_IMAGE_IS_REPLACEMENT)) - -/* OBJC_IMAGE_IS_REPLACEMENT: - Don't load any classes - Don't load any categories - Do fix up selector refs (@selector points to them) - Do fix up class refs (@class and objc_msgSend points to them) - Do fix up protocols (@protocol points to them) - Do fix up superclass pointers in classes ([super ...] points to them) - Future: do load new classes? - Future: do load new categories? - Future: do insert new methods on existing classes? - Future: do insert new methods on existing categories? -*/ +typedef struct header_info { +private: + // Note, this is no longer a pointer, but instead an offset to a pointer + // from this location. + intptr_t mhdr_offset; -#define _objcInfoSupportsGC(info) (((info)->flags & OBJC_IMAGE_SUPPORTS_GC) ? 1 : 0) -#define _objcInfoRequiresGC(info) (((info)->flags & OBJC_IMAGE_REQUIRES_GC) ? 1 : 0) -#define _objcHeaderSupportsGC(h) ((h)->info && _objcInfoSupportsGC((h)->info)) -#define _objcHeaderRequiresGC(h) ((h)->info && _objcInfoRequiresGC((h)->info)) + // Note, this is no longer a pointer, but instead an offset to a pointer + // from this location. + intptr_t info_offset; -/* OBJC_IMAGE_SUPPORTS_GC: - was compiled with -fobjc-gc flag, regardless of whether write-barriers were issued - if executable image compiled this way, then all subsequent libraries etc. must also be this way -*/ + // Offset from this location to the non-lazy class list + intptr_t nlclslist_offset; + uintptr_t nlclslist_count; -#define _objcHeaderOptimizedByDyld(h) ((h)->info && ((h)->info->flags & OBJC_IMAGE_OPTIMIZED_BY_DYLD)) + // Offset from this location to the non-lazy category list + intptr_t nlcatlist_offset; + uintptr_t nlcatlist_count; -/* OBJC_IMAGE_OPTIMIZED_BY_DYLD: - Assorted metadata precooked in the dyld shared cache. - Never set for images outside the shared cache file itself. -*/ - + // Offset from this location to the category list + intptr_t catlist_offset; + uintptr_t catlist_count; -typedef struct _header_info { - struct _header_info *next; - const headerType *mhdr; - const objc_image_info *info; - const char *fname; // same as Dl_info.dli_fname - bool loaded; - bool inSharedCache; - bool allClassesRealized; + // Offset from this location to the category list 2 + intptr_t catlist2_offset; + uintptr_t catlist2_count; // Do not add fields without editing ObjCModernAbstraction.hpp +public: + + header_info_rw *getHeaderInfoRW() { + header_info_rw *preopt = + isPreoptimized() ? getPreoptimizedHeaderRW(this) : nil; + if (preopt) return preopt; + else return &rw_data[0]; + } + + const headerType *mhdr() const { + return (const headerType *)(((intptr_t)&mhdr_offset) + mhdr_offset); + } + + void setmhdr(const headerType *mhdr) { + mhdr_offset = (intptr_t)mhdr - (intptr_t)&mhdr_offset; + } + + const objc_image_info *info() const { + return (const objc_image_info *)(((intptr_t)&info_offset) + info_offset); + } + + void setinfo(const objc_image_info *info) { + info_offset = (intptr_t)info - (intptr_t)&info_offset; + } + + const classref_t *nlclslist(size_t *outCount) const; + + void set_nlclslist(const void *list) { + nlclslist_offset = (intptr_t)list - (intptr_t)&nlclslist_offset; + } + + category_t * const *nlcatlist(size_t *outCount) const; + + void set_nlcatlist(const void *list) { + nlcatlist_offset = (intptr_t)list - (intptr_t)&nlcatlist_offset; + } + + category_t * const *catlist(size_t *outCount) const; + + void set_catlist(const void *list) { + catlist_offset = (intptr_t)list - (intptr_t)&catlist_offset; + } + + category_t * const *catlist2(size_t *outCount) const; + + void set_catlist2(const void *list) { + catlist2_offset = (intptr_t)list - (intptr_t)&catlist2_offset; + } + + bool isLoaded() { + return getHeaderInfoRW()->getLoaded(); + } + + void setLoaded(bool v) { + getHeaderInfoRW()->setLoaded(v); + } + + bool areAllClassesRealized() { + return getHeaderInfoRW()->getAllClassesRealized(); + } + + void setAllClassesRealized(bool v) { + getHeaderInfoRW()->setAllClassesRealized(v); + } + + header_info *getNext() { + return getHeaderInfoRW()->getNext(); + } + + void setNext(header_info *v) { + getHeaderInfoRW()->setNext(v); + } + + bool isBundle() { + return mhdr()->filetype == MH_BUNDLE; + } + + const char *fname() const { + return dyld_image_path_containing_address(mhdr()); + } + + bool isPreoptimized() const; + + bool hasPreoptimizedSelectors() const; + + bool hasPreoptimizedClasses() const; + + bool hasPreoptimizedProtocols() const; + + bool hasPreoptimizedSectionLookups() const; #if !__OBJC2__ struct old_protocol **proto_refs; @@ -370,88 +534,92 @@ typedef struct _header_info { TCHAR *moduleName; # endif #endif + +private: + // Images in the shared cache will have an empty array here while those + // allocated at run time will allocate a single entry. + header_info_rw rw_data[]; } header_info; extern header_info *FirstHeader; extern header_info *LastHeader; -extern int HeaderCount; - -extern uint32_t AppSDKVersion; // X.Y.Z is 0xXXXXYYZZ extern void appendHeader(header_info *hi); extern void removeHeader(header_info *hi); extern objc_image_info *_getObjcImageInfo(const headerType *head, size_t *size); -extern BOOL _hasObjcContents(const header_info *hi); +extern bool _hasObjcContents(const header_info *hi); + + +// Mach-O segment and section names are 16 bytes and may be un-terminated. + +static inline bool segnameEquals(const char *lhs, const char *rhs) { + return 0 == strncmp(lhs, rhs, 16); +} + +static inline bool segnameStartsWith(const char *segname, const char *prefix) { + return 0 == strncmp(segname, prefix, strlen(prefix)); +} + +static inline bool sectnameEquals(const char *lhs, const char *rhs) { + return segnameEquals(lhs, rhs); +} + +static inline bool sectnameStartsWith(const char *sectname, const char *prefix){ + return segnameStartsWith(sectname, prefix); +} + + +#if __OBJC2__ +extern bool didCallDyldNotifyRegister; +#endif /* selectors */ -extern void sel_init(BOOL gc, size_t selrefCount); -extern SEL sel_registerNameNoLock(const char *str, BOOL copy); -extern void sel_lock(void); -extern void sel_unlock(void); -extern BOOL sel_preoptimizationValid(const header_info *hi); - -extern SEL SEL_load; -extern SEL SEL_initialize; -extern SEL SEL_resolveClassMethod; -extern SEL SEL_resolveInstanceMethod; +extern void sel_init(size_t selrefCount); +extern SEL sel_registerNameNoLock(const char *str, bool copy); + extern SEL SEL_cxx_construct; extern SEL SEL_cxx_destruct; -extern SEL SEL_retain; -extern SEL SEL_release; -extern SEL SEL_autorelease; -extern SEL SEL_retainCount; -extern SEL SEL_alloc; -extern SEL SEL_allocWithZone; -extern SEL SEL_dealloc; -extern SEL SEL_copy; -extern SEL SEL_new; -extern SEL SEL_finalize; -extern SEL SEL_forwardInvocation; -extern SEL SEL_tryRetain; -extern SEL SEL_isDeallocating; -extern SEL SEL_retainWeakReference; -extern SEL SEL_allowsWeakReference; /* preoptimization */ extern void preopt_init(void); extern void disableSharedCacheOptimizations(void); extern bool isPreoptimized(void); +extern bool noMissingWeakSuperclasses(void); extern header_info *preoptimizedHinfoForHeader(const headerType *mhdr); -#if SUPPORT_PREOPT && __cplusplus -#include -using objc_selopt_t = const objc_opt::objc_selopt_t; -#else -struct objc_selopt_t; -#endif +extern Protocol *getPreoptimizedProtocol(const char *name); +extern Protocol *getSharedCachePreoptimizedProtocol(const char *name); -extern objc_selopt_t *preoptimizedSelectors(void); +extern unsigned getPreoptimizedClassUnreasonableCount(); extern Class getPreoptimizedClass(const char *name); extern Class* copyPreoptimizedClasses(const char *name, int *outCount); - -/* optional malloc zone for runtime data */ -extern malloc_zone_t *_objc_internal_zone(void); -extern void *_malloc_internal(size_t size); -extern void *_calloc_internal(size_t count, size_t size); -extern void *_realloc_internal(void *ptr, size_t size); -extern char *_strdup_internal(const char *str); -extern char *_strdupcat_internal(const char *s1, const char *s2); -extern uint8_t *_ustrdup_internal(const uint8_t *str); -extern void *_memdup_internal(const void *mem, size_t size); -extern void _free_internal(void *ptr); -extern size_t _malloc_size_internal(void *ptr); - extern Class _calloc_class(size_t size); /* method lookup */ -extern IMP lookUpImpOrNil(Class, SEL, id obj, bool initialize, bool cache, bool resolver); -extern IMP lookUpImpOrForward(Class, SEL, id obj, bool initialize, bool cache, bool resolver); +enum { + LOOKUP_INITIALIZE = 1, + LOOKUP_RESOLVER = 2, + LOOKUP_NIL = 4, + LOOKUP_NOCACHE = 8, +}; +extern IMP lookUpImpOrForward(id obj, SEL, Class cls, int behavior); +extern IMP lookUpImpOrForwardTryCache(id obj, SEL, Class cls, int behavior = 0); +extern IMP lookUpImpOrNilTryCache(id obj, SEL, Class cls, int behavior = 0); extern IMP lookupMethodInClassAndLoadCache(Class cls, SEL sel); -extern BOOL class_respondsToSelector_inst(Class cls, SEL sel, id inst); + +struct IMPAndSEL { + IMP imp; + SEL sel; +}; + +extern IMPAndSEL _method_getImplementationAndName(Method m); + +extern BOOL class_respondsToSelector_inst(id inst, SEL sel, Class cls); +extern Class class_initialize(Class cls, id inst); extern bool objcMsgLogEnabled; extern bool logMessageSend(bool isClassMethod, @@ -460,27 +628,22 @@ extern bool logMessageSend(bool isClassMethod, SEL selector); /* message dispatcher */ -extern IMP _class_lookupMethodAndLoadCache3(id, SEL, Class); #if !OBJC_OLD_DISPATCH_PROTOTYPES extern void _objc_msgForward_impcache(void); -extern void _objc_ignored_method(void); -extern void _objc_msgSend_uncached_impcache(void); #else extern id _objc_msgForward_impcache(id, SEL, ...); -extern id _objc_ignored_method(id, SEL, ...); -extern id _objc_msgSend_uncached_impcache(id, SEL, ...); #endif /* errors */ -extern void __objc_error(id, const char *, ...) __attribute__((format (printf, 2, 3), noreturn)); -extern void _objc_inform(const char *fmt, ...) __attribute__((format (printf, 1, 2))); -extern void _objc_inform_on_crash(const char *fmt, ...) __attribute__((format (printf, 1, 2))); -extern void _objc_inform_now_and_on_crash(const char *fmt, ...) __attribute__((format (printf, 1, 2))); -extern void _objc_inform_deprecated(const char *oldname, const char *newname) __attribute__((noinline)); +extern id(*badAllocHandler)(Class); +extern id _objc_callBadAllocHandler(Class cls) __attribute__((cold, noinline)); +extern void __objc_error(id, const char *, ...) __attribute__((cold, format (printf, 2, 3), noreturn)); +extern void _objc_inform(const char *fmt, ...) __attribute__((cold, format(printf, 1, 2))); +extern void _objc_inform_on_crash(const char *fmt, ...) __attribute__((cold, format (printf, 1, 2))); +extern void _objc_inform_now_and_on_crash(const char *fmt, ...) __attribute__((cold, format (printf, 1, 2))); +extern void _objc_inform_deprecated(const char *oldname, const char *newname) __attribute__((cold, noinline)); extern void inform_duplicate(const char *name, Class oldCls, Class cls); -extern bool crashlog_header_name(header_info *hi); -extern bool crashlog_header_name_string(const char *name); /* magic */ extern Class _objc_getFreedObjectClass (void); @@ -499,163 +662,22 @@ extern objc_property_attribute_t *copyPropertyAttributeList(const char *attrs, u extern char *copyPropertyAttributeValue(const char *attrs, const char *name); /* locking */ -extern void lock_init(void); -extern rwlock_t selLock; -extern mutex_t cacheUpdateLock; -extern recursive_mutex_t loadMethodLock; -#if __OBJC2__ -extern rwlock_t runtimeLock; -#else -extern mutex_t classLock; -extern mutex_t methodListLock; -#endif - -/* Lock debugging */ -#if defined(NDEBUG) || TARGET_OS_WIN32 - -#define mutex_lock(m) _mutex_lock_nodebug(m) -#define mutex_try_lock(m) _mutex_try_lock_nodebug(m) -#define mutex_unlock(m) _mutex_unlock_nodebug(m) -#define mutex_assert_locked(m) do { } while (0) -#define mutex_assert_unlocked(m) do { } while (0) - -#define recursive_mutex_lock(m) _recursive_mutex_lock_nodebug(m) -#define recursive_mutex_try_lock(m) _recursive_mutex_try_lock_nodebug(m) -#define recursive_mutex_unlock(m) _recursive_mutex_unlock_nodebug(m) -#define recursive_mutex_assert_locked(m) do { } while (0) -#define recursive_mutex_assert_unlocked(m) do { } while (0) - -#define monitor_enter(m) _monitor_enter_nodebug(m) -#define monitor_exit(m) _monitor_exit_nodebug(m) -#define monitor_wait(m) _monitor_wait_nodebug(m) -#define monitor_assert_locked(m) do { } while (0) -#define monitor_assert_unlocked(m) do { } while (0) - -#define rwlock_read(m) _rwlock_read_nodebug(m) -#define rwlock_write(m) _rwlock_write_nodebug(m) -#define rwlock_try_read(m) _rwlock_try_read_nodebug(m) -#define rwlock_try_write(m) _rwlock_try_write_nodebug(m) -#define rwlock_unlock_read(m) _rwlock_unlock_read_nodebug(m) -#define rwlock_unlock_write(m) _rwlock_unlock_write_nodebug(m) -#define rwlock_assert_reading(m) do { } while (0) -#define rwlock_assert_writing(m) do { } while (0) -#define rwlock_assert_locked(m) do { } while (0) -#define rwlock_assert_unlocked(m) do { } while (0) -#else - -extern int _mutex_lock_debug(mutex_t *lock, const char *name); -extern int _mutex_try_lock_debug(mutex_t *lock, const char *name); -extern int _mutex_unlock_debug(mutex_t *lock, const char *name); -extern void _mutex_assert_locked_debug(mutex_t *lock, const char *name); -extern void _mutex_assert_unlocked_debug(mutex_t *lock, const char *name); - -extern int _recursive_mutex_lock_debug(recursive_mutex_t *lock, const char *name); -extern int _recursive_mutex_try_lock_debug(recursive_mutex_t *lock, const char *name); -extern int _recursive_mutex_unlock_debug(recursive_mutex_t *lock, const char *name); -extern void _recursive_mutex_assert_locked_debug(recursive_mutex_t *lock, const char *name); -extern void _recursive_mutex_assert_unlocked_debug(recursive_mutex_t *lock, const char *name); - -extern int _monitor_enter_debug(monitor_t *lock, const char *name); -extern int _monitor_exit_debug(monitor_t *lock, const char *name); -extern int _monitor_wait_debug(monitor_t *lock, const char *name); -extern void _monitor_assert_locked_debug(monitor_t *lock, const char *name); -extern void _monitor_assert_unlocked_debug(monitor_t *lock, const char *name); - -extern void _rwlock_read_debug(rwlock_t *l, const char *name); -extern void _rwlock_write_debug(rwlock_t *l, const char *name); -extern int _rwlock_try_read_debug(rwlock_t *l, const char *name); -extern int _rwlock_try_write_debug(rwlock_t *l, const char *name); -extern void _rwlock_unlock_read_debug(rwlock_t *l, const char *name); -extern void _rwlock_unlock_write_debug(rwlock_t *l, const char *name); -extern void _rwlock_assert_reading_debug(rwlock_t *l, const char *name); -extern void _rwlock_assert_writing_debug(rwlock_t *l, const char *name); -extern void _rwlock_assert_locked_debug(rwlock_t *l, const char *name); -extern void _rwlock_assert_unlocked_debug(rwlock_t *l, const char *name); - -#define mutex_lock(m) _mutex_lock_debug (m, #m) -#define mutex_try_lock(m) _mutex_try_lock_debug (m, #m) -#define mutex_unlock(m) _mutex_unlock_debug (m, #m) -#define mutex_assert_locked(m) _mutex_assert_locked_debug (m, #m) -#define mutex_assert_unlocked(m) _mutex_assert_unlocked_debug (m, #m) - -#define recursive_mutex_lock(m) _recursive_mutex_lock_debug (m, #m) -#define recursive_mutex_try_lock(m) _recursive_mutex_try_lock_debug (m, #m) -#define recursive_mutex_unlock(m) _recursive_mutex_unlock_debug (m, #m) -#define recursive_mutex_assert_locked(m) _recursive_mutex_assert_locked_debug (m, #m) -#define recursive_mutex_assert_unlocked(m) _recursive_mutex_assert_unlocked_debug (m, #m) - -#define monitor_enter(m) _monitor_enter_debug(m, #m) -#define monitor_exit(m) _monitor_exit_debug(m, #m) -#define monitor_wait(m) _monitor_wait_debug(m, #m) -#define monitor_assert_locked(m) _monitor_assert_locked_debug(m, #m) -#define monitor_assert_unlocked(m) _monitor_assert_unlocked_debug(m, #m) - -#define rwlock_read(m) _rwlock_read_debug(m, #m) -#define rwlock_write(m) _rwlock_write_debug(m, #m) -#define rwlock_try_read(m) _rwlock_try_read_debug(m, #m) -#define rwlock_try_write(m) _rwlock_try_write_debug(m, #m) -#define rwlock_unlock_read(m) _rwlock_unlock_read_debug(m, #m) -#define rwlock_unlock_write(m) _rwlock_unlock_write_debug(m, #m) -#define rwlock_assert_reading(m) _rwlock_assert_reading_debug(m, #m) -#define rwlock_assert_writing(m) _rwlock_assert_writing_debug(m, #m) -#define rwlock_assert_locked(m) _rwlock_assert_locked_debug(m, #m) -#define rwlock_assert_unlocked(m) _rwlock_assert_unlocked_debug(m, #m) - -#endif - -#define rwlock_unlock(m, s) \ - do { \ - if ((s) == RDONLY) rwlock_unlock_read(m); \ - else if ((s) == RDWR) rwlock_unlock_write(m); \ - } while (0) - - -/* ignored selector support */ - -/* Non-GC: no ignored selectors - GC (i386 Mac): some selectors ignored, remapped to kIgnore - GC (others): some selectors ignored, but not remapped -*/ - -static inline int ignoreSelector(SEL sel) -{ -#if !SUPPORT_GC - return NO; -#elif SUPPORT_IGNORED_SELECTOR_CONSTANT - return UseGC && sel == (SEL)kIgnore; -#else - return UseGC && - (sel == @selector(retain) || - sel == @selector(release) || - sel == @selector(autorelease) || - sel == @selector(retainCount) || - sel == @selector(dealloc)); -#endif -} +class monitor_locker_t : nocopy_t { + monitor_t& lock; + public: + monitor_locker_t(monitor_t& newLock) : lock(newLock) { lock.enter(); } + ~monitor_locker_t() { lock.leave(); } +}; -static inline int ignoreSelectorNamed(const char *sel) -{ -#if !SUPPORT_GC - return NO; -#else - // release retain retainCount dealloc autorelease - return (UseGC && - ( (sel[0] == 'r' && sel[1] == 'e' && - (strcmp(&sel[2], "lease") == 0 || - strcmp(&sel[2], "tain") == 0 || - strcmp(&sel[2], "tainCount") == 0 )) - || - (strcmp(sel, "dealloc") == 0) - || - (sel[0] == 'a' && sel[1] == 'u' && - strcmp(&sel[2], "torelease") == 0))); -#endif -} +class recursive_mutex_locker_t : nocopy_t { + recursive_mutex_t& lock; + public: + recursive_mutex_locker_t(recursive_mutex_t& newLock) + : lock(newLock) { lock.lock(); } + ~recursive_mutex_locker_t() { lock.unlock(); } +}; -/* GC startup */ -extern void gc_init(BOOL wantsGC); -extern void gc_init2(void); /* Exceptions */ struct alt_handler_list; @@ -670,36 +692,6 @@ extern void _destroyAltHandlerList(struct alt_handler_list *list); extern void gdb_objc_class_changed(Class cls, unsigned long changes, const char *classname) __attribute__((noinline)); -#if SUPPORT_GC - -/* Write barrier implementations */ -extern id objc_getAssociatedObject_non_gc(id object, const void *key); -extern void objc_setAssociatedObject_non_gc(id object, const void *key, id value, objc_AssociationPolicy policy); - -extern id objc_getAssociatedObject_gc(id object, const void *key); -extern void objc_setAssociatedObject_gc(id object, const void *key, id value, objc_AssociationPolicy policy); - -/* xrefs */ -extern objc_xref_t _object_addExternalReference_non_gc(id obj, objc_xref_t type); -extern id _object_readExternalReference_non_gc(objc_xref_t ref); -extern void _object_removeExternalReference_non_gc(objc_xref_t ref); - -extern objc_xref_t _object_addExternalReference_gc(id obj, objc_xref_t type); -extern id _object_readExternalReference_gc(objc_xref_t ref); -extern void _object_removeExternalReference_gc(objc_xref_t ref); - -/* GC weak reference fixup. */ -extern void gc_fixup_weakreferences(id newObject, id oldObject); - -/* GC datasegment registration. */ -extern void gc_register_datasegment(uintptr_t base, size_t size); -extern void gc_unregister_datasegment(uintptr_t base, size_t size); - -/* objc_dumpHeap implementation */ -extern BOOL _objc_dumpHeap(auto_zone_t *zone, const char *filename); - -#endif - // Settings from environment variables #define OPTION(var, env, help) extern bool var; @@ -707,18 +699,9 @@ extern BOOL _objc_dumpHeap(auto_zone_t *zone, const char *filename); #undef OPTION extern void environ_init(void); +extern void runtime_init(void); -extern void logReplacedMethod(const char *className, SEL s, BOOL isMeta, const char *catName, IMP oldImp, IMP newImp); - -static __inline uint32_t _objc_strhash(const char *s) { - uint32_t hash = 0; - for (;;) { - int a = *s++; - if (0 == a) break; - hash += (hash << 8) + a; - } - return hash; -} +extern void logReplacedMethod(const char *className, SEL s, bool isMeta, const char *catName, IMP oldImp, IMP newImp); // objc per-thread storage @@ -727,13 +710,16 @@ typedef struct { struct SyncCache *syncCache; // for @synchronize struct alt_handler_list *handlerList; // for exception alt handlers char *printableNames[4]; // temporary demangled names for logging + const char **classNameLookups; // for objc_getClass() hooks + unsigned classNameLookupsAllocated; + unsigned classNameLookupsUsed; // If you add new fields here, don't forget to update // _objc_pthread_destroyspecific() } _objc_pthread_data; -extern _objc_pthread_data *_objc_fetch_pthread_data(BOOL create); +extern _objc_pthread_data *_objc_fetch_pthread_data(bool create); extern void tls_init(void); // encoding.h @@ -753,6 +739,7 @@ extern void arr_init(void); extern id objc_autoreleaseReturnValue(id obj); // block trampolines +extern void _imp_implementationWithBlock_init(void); extern IMP _imp_implementationWithBlockNoCopy(id block); // layout.h @@ -760,57 +747,60 @@ typedef struct { uint8_t *bits; size_t bitCount; size_t bitsAllocated; - BOOL weak; + bool weak; } layout_bitmap; -extern layout_bitmap layout_bitmap_create(const unsigned char *layout_string, size_t layoutStringInstanceSize, size_t instanceSize, BOOL weak); -extern layout_bitmap layout_bitmap_create_empty(size_t instanceSize, BOOL weak); +extern layout_bitmap layout_bitmap_create(const unsigned char *layout_string, size_t layoutStringInstanceSize, size_t instanceSize, bool weak); +extern layout_bitmap layout_bitmap_create_empty(size_t instanceSize, bool weak); extern void layout_bitmap_free(layout_bitmap bits); extern const unsigned char *layout_string_create(layout_bitmap bits); extern void layout_bitmap_set_ivar(layout_bitmap bits, const char *type, size_t offset); extern void layout_bitmap_grow(layout_bitmap *bits, size_t newCount); extern void layout_bitmap_slide(layout_bitmap *bits, size_t oldPos, size_t newPos); extern void layout_bitmap_slide_anywhere(layout_bitmap *bits, size_t oldPos, size_t newPos); -extern BOOL layout_bitmap_splat(layout_bitmap dst, layout_bitmap src, +extern bool layout_bitmap_splat(layout_bitmap dst, layout_bitmap src, size_t oldSrcInstanceSize); -extern BOOL layout_bitmap_or(layout_bitmap dst, layout_bitmap src, const char *msg); -extern BOOL layout_bitmap_clear(layout_bitmap dst, layout_bitmap src, const char *msg); +extern bool layout_bitmap_or(layout_bitmap dst, layout_bitmap src, const char *msg); +extern bool layout_bitmap_clear(layout_bitmap dst, layout_bitmap src, const char *msg); extern void layout_bitmap_print(layout_bitmap bits); // fixme runtime -extern Class look_up_class(const char *aClassName, BOOL includeUnconnected, BOOL includeClassHandler); -extern const char *map_images(enum dyld_image_states state, uint32_t infoCount, const struct dyld_image_info infoList[]); -extern const char *map_images_nolock(enum dyld_image_states state, uint32_t infoCount, const struct dyld_image_info infoList[]); -extern const char * load_images(enum dyld_image_states state, uint32_t infoCount, const struct dyld_image_info infoList[]); -extern BOOL load_images_nolock(enum dyld_image_states state, uint32_t infoCount, const struct dyld_image_info infoList[]); -extern void unmap_image(const struct mach_header *mh, intptr_t vmaddr_slide); +extern bool MultithreadedForkChild; +extern id objc_noop_imp(id self, SEL _cmd); +extern Class look_up_class(const char *aClassName, bool includeUnconnected, bool includeClassHandler); +extern "C" void map_images(unsigned count, const char * const paths[], + const struct mach_header * const mhdrs[]); +extern void map_images_nolock(unsigned count, const char * const paths[], + const struct mach_header * const mhdrs[]); +extern void load_images(const char *path, const struct mach_header *mh); +extern void unmap_image(const char *path, const struct mach_header *mh); extern void unmap_image_nolock(const struct mach_header *mh); -extern void _read_images(header_info **hList, uint32_t hCount); -extern void prepare_load_methods(header_info *hi); +extern void _read_images(header_info **hList, uint32_t hCount, int totalClasses, int unoptimizedTotalClass); extern void _unload_image(header_info *hi); -extern const char ** _objc_copyClassNamesForImage(header_info *hi, unsigned int *outCount); - extern const header_info *_headerForClass(Class cls); extern Class _class_remap(Class cls); -extern Class _class_getNonMetaClass(Class cls, id obj); -extern Ivar _class_getVariable(Class cls, const char *name, Class *memberOf); -extern BOOL _class_usesAutomaticRetainRelease(Class cls); -extern uint32_t _class_getInstanceStart(Class cls); +extern Ivar _class_getVariable(Class cls, const char *name); extern unsigned _class_createInstancesFromZone(Class cls, size_t extraBytes, void *zone, id *results, unsigned num_requested); -extern id _objc_constructOrFree(id bytes, Class cls); extern const char *_category_getName(Category cat); extern const char *_category_getClassName(Category cat); extern Class _category_getClass(Category cat); extern IMP _category_getLoadMethod(Category cat); -extern id object_cxxConstructFromClass(id obj, Class cls); +enum { + OBJECT_CONSTRUCT_NONE = 0, + OBJECT_CONSTRUCT_FREE_ONFAILURE = 1, + OBJECT_CONSTRUCT_CALL_BADALLOC = 2, +}; +extern id object_cxxConstructFromClass(id obj, Class cls, int flags); extern void object_cxxDestruct(id obj); -extern void _class_resolveMethod(Class cls, SEL sel, id inst); +extern void fixupCopiedIvars(id newObject, id oldObject); +extern Class _class_getClassForIvar(Class cls, Ivar ivar); + #define OBJC_WARN_DEPRECATED \ do { \ @@ -836,28 +826,174 @@ __END_DECLS #define countof(arr) (sizeof(arr) / sizeof((arr)[0])) +static __inline uint32_t _objc_strhash(const char *s) { + uint32_t hash = 0; + for (;;) { + int a = *s++; + if (0 == a) break; + hash += (hash << 8) + a; + } + return hash; +} + +#if __cplusplus + +template +static inline T log2u(T x) { + return (x<2) ? 0 : log2u(x>>1)+1; +} + +template +static inline T exp2u(T x) { + return (1 << x); +} + +template +static T exp2m1u(T x) { + return (1 << x) - 1; +} + +#endif + +// Misalignment-safe integer types +__attribute__((aligned(1))) typedef uintptr_t unaligned_uintptr_t; +__attribute__((aligned(1))) typedef intptr_t unaligned_intptr_t; +__attribute__((aligned(1))) typedef uint64_t unaligned_uint64_t; +__attribute__((aligned(1))) typedef int64_t unaligned_int64_t; +__attribute__((aligned(1))) typedef uint32_t unaligned_uint32_t; +__attribute__((aligned(1))) typedef int32_t unaligned_int32_t; +__attribute__((aligned(1))) typedef uint16_t unaligned_uint16_t; +__attribute__((aligned(1))) typedef int16_t unaligned_int16_t; + + // Global operator new and delete. We must not use any app overrides. // This ALSO REQUIRES each of these be in libobjc's unexported symbol list. -#if __cplusplus +#if __cplusplus && !defined(TEST_OVERRIDES_NEW) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Winline-new-delete" #include -inline void* operator new(std::size_t size) throw (std::bad_alloc) { return _malloc_internal(size); } -inline void* operator new[](std::size_t size) throw (std::bad_alloc) { return _malloc_internal(size); } -inline void* operator new(std::size_t size, const std::nothrow_t&) throw() { return _malloc_internal(size); } -inline void* operator new[](std::size_t size, const std::nothrow_t&) throw() { return _malloc_internal(size); } -inline void operator delete(void* p) throw() { _free_internal(p); } -inline void operator delete[](void* p) throw() { _free_internal(p); } -inline void operator delete(void* p, const std::nothrow_t&) throw() { _free_internal(p); } -inline void operator delete[](void* p, const std::nothrow_t&) throw() { _free_internal(p); } +inline void* operator new(std::size_t size) { return malloc(size); } +inline void* operator new[](std::size_t size) { return malloc(size); } +inline void* operator new(std::size_t size, const std::nothrow_t&) noexcept(true) { return malloc(size); } +inline void* operator new[](std::size_t size, const std::nothrow_t&) noexcept(true) { return malloc(size); } +inline void operator delete(void* p) noexcept(true) { free(p); } +inline void operator delete[](void* p) noexcept(true) { free(p); } +inline void operator delete(void* p, const std::nothrow_t&) noexcept(true) { free(p); } +inline void operator delete[](void* p, const std::nothrow_t&) noexcept(true) { free(p); } #pragma clang diagnostic pop #endif +class TimeLogger { + uint64_t mStart; + bool mRecord; + public: + TimeLogger(bool record = true) + : mStart(nanoseconds()) + , mRecord(record) + { } + + void log(const char *msg) { + if (mRecord) { + uint64_t end = nanoseconds(); + _objc_inform("%.2f ms: %s", (end - mStart) / 1000000.0, msg); + mStart = nanoseconds(); + } + } +}; + +enum { CacheLineSize = 64 }; + +// StripedMap is a map of void* -> T, sized appropriately +// for cache-friendly lock striping. +// For example, this may be used as StripedMap +// or as StripedMap where SomeStruct stores a spin lock. +template +class StripedMap { +#if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR + enum { StripeCount = 8 }; +#else + enum { StripeCount = 64 }; +#endif + + struct PaddedT { + T value alignas(CacheLineSize); + }; + + PaddedT array[StripeCount]; + + static unsigned int indexForPointer(const void *p) { + uintptr_t addr = reinterpret_cast(p); + return ((addr >> 4) ^ (addr >> 9)) % StripeCount; + } + + public: + T& operator[] (const void *p) { + return array[indexForPointer(p)].value; + } + const T& operator[] (const void *p) const { + return const_cast>(this)[p]; + } + + // Shortcuts for StripedMaps of locks. + void lockAll() { + for (unsigned int i = 0; i < StripeCount; i++) { + array[i].value.lock(); + } + } + + void unlockAll() { + for (unsigned int i = 0; i < StripeCount; i++) { + array[i].value.unlock(); + } + } + + void forceResetAll() { + for (unsigned int i = 0; i < StripeCount; i++) { + array[i].value.forceReset(); + } + } + + void defineLockOrder() { + for (unsigned int i = 1; i < StripeCount; i++) { + lockdebug_lock_precedes_lock(&array[i-1].value, &array[i].value); + } + } + + void precedeLock(const void *newlock) { + // assumes defineLockOrder is also called + lockdebug_lock_precedes_lock(&array[StripeCount-1].value, newlock); + } + + void succeedLock(const void *oldlock) { + // assumes defineLockOrder is also called + lockdebug_lock_precedes_lock(oldlock, &array[0].value); + } + + const void *getLock(int i) { + if (i < StripeCount) return &array[i].value; + else return nil; + } + +#if DEBUG + StripedMap() { + // Verify alignment expectations. + uintptr_t base = (uintptr_t)&array[0].value; + uintptr_t delta = (uintptr_t)&array[1].value - base; + ASSERT(delta % CacheLineSize == 0); + ASSERT(base % CacheLineSize == 0); + } +#else + constexpr StripedMap() {} +#endif +}; + + // DisguisedPtr acts like pointer type T*, except the // stored value is disguised to hide it from tools like `leaks`. // nil is disguised as itself so zero-filled memory works as expected, -// which means 0x80..00 is also diguised as itself but we don't care +// which means 0x80..00 is also disguised as itself but we don't care. +// Note that weak_entry_t knows about this encoding. template class DisguisedPtr { uintptr_t value; @@ -903,6 +1039,117 @@ class DisguisedPtr { // because we don't currently use them anywhere }; +// fixme type id is weird and not identical to objc_object* +static inline bool operator == (DisguisedPtr lhs, id rhs) { + return lhs == (objc_object *)rhs; +} +static inline bool operator != (DisguisedPtr lhs, id rhs) { + return lhs != (objc_object *)rhs; +} + + +// Storage for a thread-safe chained hook function. +// get() returns the value for calling. +// set() installs a new function and returns the old one for chaining. +// More precisely, set() writes the old value to a variable supplied by +// the caller. get() and set() use appropriate barriers so that the +// old value is safely written to the variable before the new value is +// called to use it. +// +// T1: store to old variable; store-release to hook variable +// T2: load-acquire from hook variable; call it; called hook loads old variable + +template +class ChainedHookFunction { + std::atomic hook{nil}; + +public: + constexpr ChainedHookFunction(Fn f) : hook{f} { }; + + Fn get() { + return hook.load(std::memory_order_acquire); + } + + void set(Fn newValue, Fn *oldVariable) + { + Fn oldValue = hook.load(std::memory_order_relaxed); + do { + *oldVariable = oldValue; + } while (!hook.compare_exchange_weak(oldValue, newValue, + std::memory_order_release, + std::memory_order_relaxed)); + } +}; + + +// A small vector for use as a global variable. Only supports appending and +// iteration. Stores up to N elements inline, and multiple elements in a heap +// allocation. There is no attempt to amortize reallocation cost; this is +// intended to be used in situation where a small number of elements is +// common, more might happen, and significantly more is very rare. +// +// This does not clean up its allocation, and thus cannot be used as a local +// variable or member of something with limited lifetime. + +template +class GlobalSmallVector { + static_assert(std::is_pod::value, "SmallVector requires POD types"); + +protected: + unsigned count{0}; + union { + T inlineElements[InlineCount]; + T *elements{nullptr}; + }; + +public: + void append(const T &val) { + if (count < InlineCount) { + // We have space. Store the new value inline. + inlineElements[count] = val; + } else if (count == InlineCount) { + // Inline storage is full. Switch to a heap allocation. + T *newElements = (T *)malloc((count + 1) * sizeof(T)); + memcpy(newElements, inlineElements, count * sizeof(T)); + newElements[count] = val; + elements = newElements; + } else { + // Resize the heap allocation and append. + elements = (T *)realloc(elements, (count + 1) * sizeof(T)); + elements[count] = val; + } + count++; + } + + const T *begin() const { + return count <= InlineCount ? inlineElements : elements; + } + + const T *end() const { + return begin() + count; + } +}; + +// A small vector that cleans up its internal memory allocation when destroyed. +template +class SmallVector: public GlobalSmallVector { +public: + ~SmallVector() { + if (this->count > InlineCount) + free(this->elements); + } + + template + void initFrom(const GlobalSmallVector &other) { + ASSERT(this->count == 0); + this->count = (unsigned)(other.end() - other.begin()); + if (this->count > InlineCount) { + this->elements = (T *)memdup(other.begin(), this->count * sizeof(T)); + } else { + memcpy(this->inlineElements, other.begin(), this->count * sizeof(T)); + } + } +}; // Pointer hash function. // This is not a terrific hash, but it is fast @@ -954,6 +1201,10 @@ static uint32_t ptr_hash(uint32_t key) */ + +// Lock declarations +#include "objc-locks.h" + // Inlined parts of objc_object's implementation #include "objc-object.h"