]>
Commit | Line | Data |
---|---|---|
412ebb8e A |
1 | /* -*- mode: C++; c-basic-offset: 4; tab-width: 4 -*- |
2 | * | |
3 | * Copyright (c) 2010 Apple Inc. All rights reserved. | |
4 | * | |
5 | * @APPLE_LICENSE_HEADER_START@ | |
6 | * | |
7 | * This file contains Original Code and/or Modifications of Original Code | |
8 | * as defined in and that are subject to the Apple Public Source License | |
9 | * Version 2.0 (the 'License'). You may not use this file except in | |
10 | * compliance with the License. Please obtain a copy of the License at | |
11 | * http://www.opensource.apple.com/apsl/ and read it before using this | |
12 | * file. | |
13 | * | |
14 | * The Original Code and all software distributed under the License are | |
15 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER | |
16 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, | |
17 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, | |
18 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. | |
19 | * Please see the License for the specific language governing rights and | |
20 | * limitations under the License. | |
21 | * | |
22 | * @APPLE_LICENSE_HEADER_END@ | |
23 | */ | |
24 | ||
25 | ||
26 | #include <stdlib.h> | |
27 | #include <stdint.h> | |
28 | #include <string.h> | |
29 | #include <stddef.h> | |
30 | #include <stdio.h> | |
31 | #include <pthread.h> | |
32 | #include <Block.h> | |
33 | #include <malloc/malloc.h> | |
34 | #include <mach-o/loader.h> | |
35 | #include <libkern/OSAtomic.h> | |
36 | ||
cf998323 | 37 | #include <mach-o/dyld_priv.h> |
412ebb8e A |
38 | |
39 | ||
40 | #if __LP64__ | |
41 | typedef struct mach_header_64 macho_header; | |
42 | #define LC_SEGMENT_COMMAND LC_SEGMENT_64 | |
43 | typedef struct segment_command_64 macho_segment_command; | |
44 | typedef struct section_64 macho_section; | |
45 | #else | |
46 | typedef struct mach_header macho_header; | |
47 | #define LC_SEGMENT_COMMAND LC_SEGMENT | |
48 | typedef struct segment_command macho_segment_command; | |
49 | typedef struct section macho_section; | |
50 | #endif | |
51 | ||
2fd3f4e8 A |
52 | |
53 | typedef void (*TermFunc)(void*); | |
54 | ||
55 | ||
56 | ||
df9d6cf7 | 57 | #if __has_feature(tls) || __arm64__ || __arm__ |
412ebb8e | 58 | |
cf998323 A |
59 | // |
60 | // Info about thread-local variable storage. | |
61 | // | |
62 | typedef struct { | |
63 | size_t info_size; // sizeof(dyld_tlv_info) | |
64 | void * tlv_addr; // Base address of TLV storage | |
65 | size_t tlv_size; // Byte size of TLV storage | |
66 | } dyld_tlv_info; | |
412ebb8e A |
67 | |
68 | ||
69 | struct TLVDescriptor | |
70 | { | |
71 | void* (*thunk)(struct TLVDescriptor*); | |
72 | unsigned long key; | |
73 | unsigned long offset; | |
74 | }; | |
75 | typedef struct TLVDescriptor TLVDescriptor; | |
76 | ||
77 | ||
78 | // implemented in assembly | |
79 | extern void* tlv_get_addr(TLVDescriptor*); | |
80 | ||
81 | struct TLVImageInfo | |
82 | { | |
83 | pthread_key_t key; | |
84 | const struct mach_header* mh; | |
85 | }; | |
86 | typedef struct TLVImageInfo TLVImageInfo; | |
87 | ||
88 | static TLVImageInfo* tlv_live_images = NULL; | |
89 | static unsigned int tlv_live_image_alloc_count = 0; | |
90 | static unsigned int tlv_live_image_used_count = 0; | |
91 | static pthread_mutex_t tlv_live_image_lock = PTHREAD_MUTEX_INITIALIZER; | |
92 | ||
93 | static void tlv_set_key_for_image(const struct mach_header* mh, pthread_key_t key) | |
94 | { | |
95 | pthread_mutex_lock(&tlv_live_image_lock); | |
96 | if ( tlv_live_image_used_count == tlv_live_image_alloc_count ) { | |
97 | unsigned int newCount = (tlv_live_images == NULL) ? 8 : 2*tlv_live_image_alloc_count; | |
98 | struct TLVImageInfo* newBuffer = malloc(sizeof(TLVImageInfo)*newCount); | |
99 | if ( tlv_live_images != NULL ) { | |
100 | memcpy(newBuffer, tlv_live_images, sizeof(TLVImageInfo)*tlv_live_image_used_count); | |
101 | free(tlv_live_images); | |
102 | } | |
103 | tlv_live_images = newBuffer; | |
104 | tlv_live_image_alloc_count = newCount; | |
105 | } | |
106 | tlv_live_images[tlv_live_image_used_count].key = key; | |
107 | tlv_live_images[tlv_live_image_used_count].mh = mh; | |
108 | ++tlv_live_image_used_count; | |
109 | pthread_mutex_unlock(&tlv_live_image_lock); | |
110 | } | |
111 | ||
112 | static const struct mach_header* tlv_get_image_for_key(pthread_key_t key) | |
113 | { | |
114 | const struct mach_header* result = NULL; | |
115 | pthread_mutex_lock(&tlv_live_image_lock); | |
116 | for(unsigned int i=0; i < tlv_live_image_used_count; ++i) { | |
117 | if ( tlv_live_images[i].key == key ) { | |
118 | result = tlv_live_images[i].mh; | |
119 | break; | |
120 | } | |
121 | } | |
122 | pthread_mutex_unlock(&tlv_live_image_lock); | |
123 | return result; | |
124 | } | |
125 | ||
126 | ||
412ebb8e A |
127 | // called lazily when TLV is first accessed |
128 | __attribute__((visibility("hidden"))) | |
129 | void* tlv_allocate_and_initialize_for_key(pthread_key_t key) | |
130 | { | |
131 | const struct mach_header* mh = tlv_get_image_for_key(key); | |
2fd3f4e8 A |
132 | if ( mh == NULL ) |
133 | return NULL; // if data structures are screwed up, don't crash | |
134 | ||
412ebb8e A |
135 | // first pass, find size and template |
136 | uint8_t* start = NULL; | |
2fd3f4e8 | 137 | unsigned long size = 0; |
412ebb8e A |
138 | intptr_t slide = 0; |
139 | bool slideComputed = false; | |
140 | bool hasInitializers = false; | |
141 | const uint32_t cmd_count = mh->ncmds; | |
142 | const struct load_command* const cmds = (struct load_command*)(((uint8_t*)mh) + sizeof(macho_header)); | |
143 | const struct load_command* cmd = cmds; | |
144 | for (uint32_t i = 0; i < cmd_count; ++i) { | |
145 | if ( cmd->cmd == LC_SEGMENT_COMMAND) { | |
146 | const macho_segment_command* seg = (macho_segment_command*)cmd; | |
147 | if ( !slideComputed && (seg->filesize != 0) ) { | |
148 | slide = (uintptr_t)mh - seg->vmaddr; | |
149 | slideComputed = true; | |
150 | } | |
151 | const macho_section* const sectionsStart = (macho_section*)((char*)seg + sizeof(macho_segment_command)); | |
152 | const macho_section* const sectionsEnd = §ionsStart[seg->nsects]; | |
153 | for (const macho_section* sect=sectionsStart; sect < sectionsEnd; ++sect) { | |
154 | switch ( sect->flags & SECTION_TYPE ) { | |
155 | case S_THREAD_LOCAL_INIT_FUNCTION_POINTERS: | |
156 | hasInitializers = true; | |
157 | break; | |
158 | case S_THREAD_LOCAL_ZEROFILL: | |
159 | case S_THREAD_LOCAL_REGULAR: | |
160 | if ( start == NULL ) { | |
161 | // first of N contiguous TLV template sections, record as if this was only section | |
162 | start = (uint8_t*)(sect->addr + slide); | |
163 | size = sect->size; | |
164 | } | |
165 | else { | |
166 | // non-first of N contiguous TLV template sections, accumlate values | |
167 | const uint8_t* newEnd = (uint8_t*)(sect->addr + slide + sect->size); | |
168 | size = newEnd - start; | |
169 | } | |
170 | break; | |
171 | } | |
172 | } | |
173 | } | |
174 | cmd = (const struct load_command*)(((char*)cmd)+cmd->cmdsize); | |
175 | } | |
10b92d3b A |
176 | // no thread local storage in image: should never happen |
177 | if ( size == 0 ) | |
178 | return NULL; | |
412ebb8e A |
179 | |
180 | // allocate buffer and fill with template | |
181 | void* buffer = malloc(size); | |
182 | memcpy(buffer, start, size); | |
183 | ||
184 | // set this thread's value for key to be the new buffer. | |
185 | pthread_setspecific(key, buffer); | |
412ebb8e A |
186 | |
187 | // second pass, run initializers | |
188 | if ( hasInitializers ) { | |
189 | cmd = cmds; | |
190 | for (uint32_t i = 0; i < cmd_count; ++i) { | |
191 | if ( cmd->cmd == LC_SEGMENT_COMMAND) { | |
192 | const macho_segment_command* seg = (macho_segment_command*)cmd; | |
193 | const macho_section* const sectionsStart = (macho_section*)((char*)seg + sizeof(macho_segment_command)); | |
194 | const macho_section* const sectionsEnd = §ionsStart[seg->nsects]; | |
195 | for (const macho_section* sect=sectionsStart; sect < sectionsEnd; ++sect) { | |
196 | if ( (sect->flags & SECTION_TYPE) == S_THREAD_LOCAL_INIT_FUNCTION_POINTERS ) { | |
197 | typedef void (*InitFunc)(void); | |
198 | InitFunc* funcs = (InitFunc*)(sect->addr + slide); | |
19894a12 | 199 | const size_t count = sect->size / sizeof(uintptr_t); |
9f83892a A |
200 | for (size_t j=count; j > 0; --j) { |
201 | InitFunc func = funcs[j-1]; | |
412ebb8e A |
202 | func(); |
203 | } | |
204 | } | |
205 | } | |
206 | } | |
207 | cmd = (const struct load_command*)(((char*)cmd)+cmd->cmdsize); | |
208 | } | |
209 | } | |
210 | return buffer; | |
211 | } | |
212 | ||
213 | ||
214 | // pthread destructor for TLV storage | |
215 | static void | |
216 | tlv_free(void *storage) | |
217 | { | |
412ebb8e A |
218 | free(storage); |
219 | } | |
220 | ||
221 | ||
222 | // called when image is loaded | |
223 | static void tlv_initialize_descriptors(const struct mach_header* mh) | |
224 | { | |
225 | pthread_key_t key = 0; | |
226 | intptr_t slide = 0; | |
227 | bool slideComputed = false; | |
228 | const uint32_t cmd_count = mh->ncmds; | |
229 | const struct load_command* const cmds = (struct load_command*)(((uint8_t*)mh) + sizeof(macho_header)); | |
230 | const struct load_command* cmd = cmds; | |
231 | for (uint32_t i = 0; i < cmd_count; ++i) { | |
232 | if ( cmd->cmd == LC_SEGMENT_COMMAND) { | |
233 | const macho_segment_command* seg = (macho_segment_command*)cmd; | |
234 | if ( !slideComputed && (seg->filesize != 0) ) { | |
235 | slide = (uintptr_t)mh - seg->vmaddr; | |
236 | slideComputed = true; | |
237 | } | |
238 | const macho_section* const sectionsStart = (macho_section*)((char*)seg + sizeof(macho_segment_command)); | |
239 | const macho_section* const sectionsEnd = §ionsStart[seg->nsects]; | |
240 | for (const macho_section* sect=sectionsStart; sect < sectionsEnd; ++sect) { | |
241 | if ( (sect->flags & SECTION_TYPE) == S_THREAD_LOCAL_VARIABLES ) { | |
242 | if ( sect->size != 0 ) { | |
243 | // allocate pthread key when we first discover this image has TLVs | |
244 | if ( key == 0 ) { | |
245 | int result = pthread_key_create(&key, &tlv_free); | |
246 | if ( result != 0 ) | |
247 | abort(); | |
248 | tlv_set_key_for_image(mh, key); | |
249 | } | |
250 | // initialize each descriptor | |
251 | TLVDescriptor* start = (TLVDescriptor*)(sect->addr + slide); | |
252 | TLVDescriptor* end = (TLVDescriptor*)(sect->addr + sect->size + slide); | |
253 | for (TLVDescriptor* d=start; d < end; ++d) { | |
254 | d->thunk = tlv_get_addr; | |
255 | d->key = key; | |
256 | //d->offset = d->offset; // offset unchanged | |
257 | } | |
258 | } | |
259 | } | |
260 | } | |
261 | } | |
262 | cmd = (const struct load_command*)(((char*)cmd)+cmd->cmdsize); | |
263 | } | |
264 | } | |
265 | ||
9f83892a | 266 | |
10b92d3b | 267 | static void tlv_load_notification(const struct mach_header* mh, intptr_t slide) |
412ebb8e | 268 | { |
9f83892a A |
269 | // This is called on all images, even those without TLVs. So we want this to be fast. |
270 | // The linker sets MH_HAS_TLV_DESCRIPTORS so we don't have to search images just to find the don't have TLVs. | |
271 | if ( mh->flags & MH_HAS_TLV_DESCRIPTORS ) | |
272 | tlv_initialize_descriptors(mh); | |
412ebb8e A |
273 | } |
274 | ||
275 | ||
412ebb8e A |
276 | // |
277 | // thread_local terminators | |
278 | // | |
279 | // C++ 0x allows thread_local C++ objects which have constructors run | |
280 | // on the thread before any use of the object and the object's destructor | |
281 | // is run on the thread when the thread terminates. | |
282 | // | |
283 | // To support this libdyld gets a pthread key early in process start up and | |
284 | // uses tlv_finalize and the key's destructor function. This key must be | |
285 | // allocated before any thread local variables are instantiated because when | |
286 | // a thread is terminated, the pthread package runs the destructor function | |
287 | // on each key's storage values in key allocation order. Since we want | |
288 | // C++ objects to be destructred before they are deallocated, we need the | |
289 | // destructor key to come before the deallocation key. | |
290 | // | |
291 | ||
412ebb8e A |
292 | struct TLVTerminatorListEntry |
293 | { | |
294 | TermFunc termFunc; | |
295 | void* objAddr; | |
296 | }; | |
297 | ||
298 | struct TLVTerminatorList | |
299 | { | |
300 | uint32_t allocCount; | |
301 | uint32_t useCount; | |
302 | struct TLVTerminatorListEntry entries[1]; // variable length | |
303 | }; | |
304 | ||
305 | ||
306 | static pthread_key_t tlv_terminators_key = 0; | |
307 | ||
308 | void _tlv_atexit(TermFunc func, void* objAddr) | |
309 | { | |
310 | // NOTE: this does not need locks because it only operates on current thread data | |
311 | struct TLVTerminatorList* list = (struct TLVTerminatorList*)pthread_getspecific(tlv_terminators_key); | |
312 | if ( list == NULL ) { | |
313 | // handle first allocation | |
314 | list = (struct TLVTerminatorList*)malloc(offsetof(struct TLVTerminatorList, entries[1])); | |
315 | list->allocCount = 1; | |
316 | list->useCount = 1; | |
317 | list->entries[0].termFunc = func; | |
318 | list->entries[0].objAddr = objAddr; | |
319 | pthread_setspecific(tlv_terminators_key, list); | |
320 | } | |
321 | else { | |
10b92d3b | 322 | if ( list->useCount == list->allocCount ) { |
412ebb8e A |
323 | // handle resizing allocation |
324 | uint32_t newAllocCount = list->allocCount * 2; | |
19894a12 | 325 | size_t newAllocSize = offsetof(struct TLVTerminatorList, entries[newAllocCount]); |
412ebb8e A |
326 | struct TLVTerminatorList* newlist = (struct TLVTerminatorList*)malloc(newAllocSize); |
327 | newlist->allocCount = newAllocCount; | |
328 | newlist->useCount = list->useCount; | |
329 | for(uint32_t i=0; i < list->useCount; ++i) | |
330 | newlist->entries[i] = list->entries[i]; | |
331 | pthread_setspecific(tlv_terminators_key, newlist); | |
332 | free(list); | |
333 | list = newlist; | |
334 | } | |
335 | // handle appending new entry | |
336 | list->entries[list->useCount].termFunc = func; | |
337 | list->entries[list->useCount].objAddr = objAddr; | |
338 | list->useCount += 1; | |
339 | } | |
340 | } | |
341 | ||
cf998323 | 342 | static void tlv_finalize_list(struct TLVTerminatorList* list) { |
2fd3f4e8 A |
343 | // destroy in reverse order of construction |
344 | for(uint32_t i=list->useCount; i > 0 ; --i) { | |
345 | struct TLVTerminatorListEntry* entry = &list->entries[i-1]; | |
412ebb8e A |
346 | if ( entry->termFunc != NULL ) { |
347 | (*entry->termFunc)(entry->objAddr); | |
348 | } | |
cf998323 A |
349 | |
350 | // If a new tlv was added via tlv_atexit, then we need to immediately | |
351 | // destroy it | |
352 | struct TLVTerminatorList* newlist = (struct TLVTerminatorList*)pthread_getspecific(tlv_terminators_key); | |
353 | if ( newlist != NULL ) { | |
354 | // Set the list to NULL so that if yet another tlv is registered, we put it in a new list | |
355 | pthread_setspecific(tlv_terminators_key, NULL); | |
356 | tlv_finalize_list(newlist); | |
357 | } | |
412ebb8e | 358 | } |
cf998323 A |
359 | free(list); |
360 | } | |
361 | ||
362 | // called by pthreads when the current thread is going away and | |
363 | // _tlv_atexit() has been called on the thread. | |
364 | static void tlv_finalize(void* storage) | |
365 | { | |
366 | // Note, on entry to this function, _tlv_exit set the list to NULL. libpthread | |
367 | // also set it to NULL before calling us. If new thread locals are added to our | |
368 | // tlv_terminators_key, then they will be on a new list, but the list we have here | |
369 | // is one we own and need to destroy it | |
370 | tlv_finalize_list((struct TLVTerminatorList*)storage); | |
412ebb8e A |
371 | } |
372 | ||
2fd3f4e8 A |
373 | // <rdar://problem/13741816> |
374 | // called by exit() before it calls cxa_finalize() so that thread_local | |
375 | // objects are destroyed before global objects. | |
cf998323 A |
376 | // Note this is only called on macOS, and by libc. |
377 | // iOS only destroys tlv's when each thread is destroyed and libpthread calls | |
378 | // tlv_finalize as that is the pointer we provided when we created the key | |
2fd3f4e8 A |
379 | void _tlv_exit() |
380 | { | |
cf998323 A |
381 | void* termFuncs = pthread_getspecific(tlv_terminators_key); |
382 | if ( termFuncs != NULL ) { | |
383 | // Clear the value so that calls to tlv_atexit during tlv_finalize | |
384 | // will go on to a new list to destroy. | |
385 | pthread_setspecific(tlv_terminators_key, NULL); | |
386 | tlv_finalize(termFuncs); | |
387 | } | |
2fd3f4e8 A |
388 | } |
389 | ||
412ebb8e A |
390 | |
391 | __attribute__((visibility("hidden"))) | |
392 | void tlv_initializer() | |
393 | { | |
394 | // create pthread key to handle thread_local destructors | |
395 | // NOTE: this key must be allocated before any keys for TLV | |
396 | // so that _pthread_tsd_cleanup will run destructors before deallocation | |
397 | (void)pthread_key_create(&tlv_terminators_key, &tlv_finalize); | |
398 | ||
399 | // register with dyld for notification when images are loaded | |
9f83892a A |
400 | _dyld_register_func_for_add_image(tlv_load_notification); |
401 | ||
412ebb8e A |
402 | } |
403 | ||
404 | ||
405 | // linked images with TLV have references to this symbol, but it is never used at runtime | |
406 | void _tlv_bootstrap() | |
407 | { | |
408 | abort(); | |
409 | } | |
410 | ||
411 | ||
2fd3f4e8 | 412 | |
412ebb8e | 413 | #else |
2fd3f4e8 | 414 | |
2fd3f4e8 A |
415 | void _tlv_exit() |
416 | { | |
417 | } | |
418 | ||
419 | void _tlv_atexit(TermFunc func, void* objAddr) | |
420 | { | |
421 | } | |
422 | ||
412ebb8e A |
423 | __attribute__((visibility("hidden"))) |
424 | void tlv_initializer() | |
425 | { | |
426 | } | |
427 | ||
428 | ||
2fd3f4e8 A |
429 | |
430 | #endif // __has_feature(tls) | |
412ebb8e A |
431 | |
432 |