]> git.saurik.com Git - apple/libdispatch.git/blob - src/voucher.c
projects / apple / libdispatch.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
libdispatch-501.40.12.tar.gz
[apple/libdispatch.git] / src / voucher.c
1 /*
2 * Copyright (c) 2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_APACHE_LICENSE_HEADER_START@
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * @APPLE_APACHE_LICENSE_HEADER_END@
19 */
20
21 #include "internal.h"
22
23 #if !defined(VOUCHER_EXPORT_PERSONA_SPI)
24 #if TARGET_OS_IPHONE
25 #define VOUCHER_EXPORT_PERSONA_SPI 1
26 #else
27 #define VOUCHER_EXPORT_PERSONA_SPI 0
28 #endif
29 #endif
30
31 #ifndef PERSONA_ID_NONE
32 #define PERSONA_ID_NONE ((uid_t)-1)
33 #endif
34
35 #if VOUCHER_USE_MACH_VOUCHER
36
37 #include <mach/mach_voucher.h>
38 #include <sys/kauth.h>
39
40 // <rdar://16363550>
41 #ifndef VM_MEMORY_GENEALOGY
42 #define VM_MEMORY_GENEALOGY 78
43 #endif
44
45 #ifndef VOUCHER_ATM_COLLECT_THRESHOLD
46 #define VOUCHER_ATM_COLLECT_THRESHOLD 1
47 #endif
48 #define VATM_COLLECT_THRESHOLD_VALUE(t) (((t) - 1) * 2)
49 static uint64_t volatile _voucher_atm_generation;
50
51 typedef struct _voucher_atm_s *_voucher_atm_t;
52
53 static void _voucher_activity_atfork_child(void);
54 static _voucher_activity_t _voucher_activity_copy_from_mach_voucher(
55 mach_voucher_t kv, voucher_activity_id_t va_id);
56 static inline _voucher_activity_t _voucher_activity_retain(
57 _voucher_activity_t act);
58 static inline void _voucher_activity_release(_voucher_activity_t act);
59 static void _voucher_activity_remove(_voucher_activity_t act);
60 static inline _voucher_atm_t _voucher_atm_retain(_voucher_atm_t vatm);
61 static inline void _voucher_atm_release(_voucher_atm_t vatm);
62
63 #pragma mark -
64 #pragma mark voucher_t
65
66 #if USE_OBJC
67 OS_OBJECT_OBJC_CLASS_DECL(voucher);
68 #define VOUCHER_CLASS OS_OBJECT_OBJC_CLASS(voucher)
69 #else
70 const _os_object_class_s _voucher_class = {
71 ._os_obj_xref_dispose = (void(*)(_os_object_t))_voucher_xref_dispose,
72 ._os_obj_dispose = (void(*)(_os_object_t))_voucher_dispose,
73 };
74 #define VOUCHER_CLASS &_voucher_class
75 #endif // USE_OBJC
76
77 static const voucher_activity_trace_id_t _voucher_activity_trace_id_release =
78 (voucher_activity_trace_id_t)voucher_activity_tracepoint_type_release <<
79 _voucher_activity_trace_id_type_shift;
80 static const unsigned int _voucher_max_activities = 16;
81
82 DISPATCH_ALWAYS_INLINE
83 static inline void
84 _voucher_recipes_init(mach_voucher_attr_recipe_data_t *recipes,
85 mach_voucher_attr_content_size_t bits_size)
86 {
87 static const mach_voucher_attr_recipe_data_t base_recipe = {
88 .key = MACH_VOUCHER_ATTR_KEY_ALL,
89 .command = MACH_VOUCHER_ATTR_COPY,
90 };
91 _voucher_recipes_base(recipes) = base_recipe;
92 static const mach_voucher_attr_recipe_data_t atm_recipe = {
93 .key = MACH_VOUCHER_ATTR_KEY_ATM,
94 .command = MACH_VOUCHER_ATTR_COPY,
95 };
96 _voucher_recipes_atm(recipes) = atm_recipe;
97 static const mach_voucher_attr_recipe_data_t bits_recipe = {
98 .key = MACH_VOUCHER_ATTR_KEY_USER_DATA,
99 .command = MACH_VOUCHER_ATTR_USER_DATA_STORE,
100 };
101 _voucher_recipes_bits(recipes) = bits_recipe;
102 if (!bits_size) return;
103 _voucher_recipes_bits(recipes).content_size = bits_size;
104 *_voucher_recipes_magic(recipes) = _voucher_magic_v1;
105 }
106
107 static inline voucher_t
108 _voucher_alloc(unsigned int activities, pthread_priority_t priority,
109 mach_voucher_attr_recipe_size_t extra)
110 {
111 if (activities > _voucher_max_activities) {
112 activities = _voucher_max_activities;
113 }
114 voucher_t voucher;
115 size_t voucher_size, recipes_size;
116 mach_voucher_attr_content_size_t bits_size;
117 recipes_size = (priority||activities||extra) ? _voucher_recipes_size() : 0;
118 bits_size = recipes_size ? _voucher_bits_size(activities) : 0;
119 voucher_size = sizeof(voucher_s) + recipes_size + bits_size + extra;
120 voucher = (voucher_t)_os_object_alloc_realized(VOUCHER_CLASS, voucher_size);
121 #if VOUCHER_ENABLE_RECIPE_OBJECTS
122 voucher->v_recipe_extra_size = extra;
123 voucher->v_recipe_extra_offset = voucher_size - extra;
124 #else
125 dispatch_assert(!extra);
126 #endif
127 voucher->v_has_priority = priority ? 1 : 0;
128 voucher->v_activities = activities;
129 if (!recipes_size) return voucher;
130 _voucher_recipes_init(voucher->v_recipes, bits_size);
131 *_voucher_priority(voucher) = (_voucher_priority_t)priority;
132 _dispatch_voucher_debug("alloc", voucher);
133 return voucher;
134 }
135
136 #if VOUCHER_ENABLE_RECIPE_OBJECTS
137 voucher_t
138 voucher_create(voucher_recipe_t recipe)
139 {
140 // TODO: capture current activities or current kvoucher ?
141 mach_voucher_attr_recipe_size_t extra = recipe ? recipe->vr_size : 0;
142 voucher_t voucher = _voucher_alloc(0, 0, extra);
143 if (extra) {
144 memcpy(_voucher_extra_recipes(voucher), recipe->vr_data, extra);
145 }
146 return voucher;
147 }
148 #endif
149
150 voucher_t
151 voucher_adopt(voucher_t voucher)
152 {
153 return _voucher_adopt(voucher);
154 }
155
156 voucher_t
157 voucher_copy(void)
158 {
159 return _voucher_copy();
160 }
161
162 voucher_t
163 voucher_copy_without_importance(void)
164 {
165 return _voucher_copy_without_importance();
166 }
167
168 voucher_t
169 voucher_retain(voucher_t voucher)
170 {
171 return _voucher_retain(voucher);
172 }
173
174 void
175 voucher_release(voucher_t voucher)
176 {
177 return _voucher_release(voucher);
178 }
179
180 void
181 _voucher_thread_cleanup(void *voucher)
182 {
183 _voucher_swap(voucher, NULL);
184 }
185
186 DISPATCH_CACHELINE_ALIGN
187 static TAILQ_HEAD(, voucher_s) _vouchers[VL_HASH_SIZE];
188 #define _vouchers_head(kv) (&_vouchers[VL_HASH((kv))])
189 static os_lock_handoff_s _vouchers_lock = OS_LOCK_HANDOFF_INIT;
190 #define _vouchers_lock_lock() os_lock_lock(&_vouchers_lock)
191 #define _vouchers_lock_unlock() os_lock_unlock(&_vouchers_lock)
192
193 static voucher_t
194 _voucher_find_and_retain(mach_voucher_t kv)
195 {
196 voucher_t v;
197 if (!kv) return NULL;
198 _vouchers_lock_lock();
199 TAILQ_FOREACH(v, _vouchers_head(kv), v_list) {
200 if (v->v_ipc_kvoucher == kv) {
201 int xref_cnt = dispatch_atomic_inc2o(v, os_obj_xref_cnt, relaxed);
202 _dispatch_voucher_debug("retain -> %d", v, xref_cnt + 1);
203 if (slowpath(xref_cnt < 0)) {
204 _dispatch_voucher_debug("overrelease", v);
205 DISPATCH_CRASH("Voucher overrelease");
206 }
207 if (xref_cnt == 0) {
208 // resurrection: raced with _voucher_remove
209 (void)dispatch_atomic_inc2o(v, os_obj_ref_cnt, relaxed);
210 }
211 break;
212 }
213 }
214 _vouchers_lock_unlock();
215 return v;
216 }
217
218 static void
219 _voucher_insert(voucher_t v)
220 {
221 mach_voucher_t kv = v->v_ipc_kvoucher;
222 if (!kv) return;
223 _vouchers_lock_lock();
224 if (slowpath(_TAILQ_IS_ENQUEUED(v, v_list))) {
225 _dispatch_voucher_debug("corruption", v);
226 DISPATCH_CRASH("Voucher corruption");
227 }
228 TAILQ_INSERT_TAIL(_vouchers_head(kv), v, v_list);
229 _vouchers_lock_unlock();
230 }
231
232 static void
233 _voucher_remove(voucher_t v)
234 {
235 mach_voucher_t kv = v->v_ipc_kvoucher;
236 if (!_TAILQ_IS_ENQUEUED(v, v_list)) return;
237 _vouchers_lock_lock();
238 if (slowpath(!kv)) {
239 _dispatch_voucher_debug("corruption", v);
240 DISPATCH_CRASH("Voucher corruption");
241 }
242 // check for resurrection race with _voucher_find_and_retain
243 if (dispatch_atomic_load2o(v, os_obj_xref_cnt, seq_cst) < 0 &&
244 _TAILQ_IS_ENQUEUED(v, v_list)) {
245 TAILQ_REMOVE(_vouchers_head(kv), v, v_list);
246 _TAILQ_MARK_NOT_ENQUEUED(v, v_list);
247 v->v_list.tqe_next = (void*)~0ull;
248 }
249 _vouchers_lock_unlock();
250 }
251
252 void
253 _voucher_dealloc_mach_voucher(mach_voucher_t kv)
254 {
255 _dispatch_kvoucher_debug("dealloc", kv);
256 _dispatch_voucher_debug_machport(kv);
257 kern_return_t kr = mach_voucher_deallocate(kv);
258 DISPATCH_VERIFY_MIG(kr);
259 (void)dispatch_assume_zero(kr);
260 }
261
262 static inline kern_return_t
263 _voucher_create_mach_voucher(const mach_voucher_attr_recipe_data_t *recipes,
264 size_t recipes_size, mach_voucher_t *kvp)
265 {
266 kern_return_t kr;
267 mach_port_t mhp = _dispatch_get_mach_host_port();
268 mach_voucher_t kv = MACH_VOUCHER_NULL;
269 mach_voucher_attr_raw_recipe_array_t kvr;
270 mach_voucher_attr_recipe_size_t kvr_size;
271 kvr = (mach_voucher_attr_raw_recipe_array_t)recipes;
272 kvr_size = (mach_voucher_attr_recipe_size_t)recipes_size;
273 kr = host_create_mach_voucher(mhp, kvr, kvr_size, &kv);
274 DISPATCH_VERIFY_MIG(kr);
275 if (!kr) {
276 _dispatch_kvoucher_debug("create", kv);
277 _dispatch_voucher_debug_machport(kv);
278 }
279 *kvp = kv;
280 return kr;
281 }
282
283 #if __has_include(<bank/bank_types.h>) && !defined(VOUCHER_USE_ATTR_BANK)
284 #include <bank/bank_types.h>
285 #define VOUCHER_USE_ATTR_BANK 1
286 mach_voucher_t _voucher_default_task_mach_voucher;
287 #endif
288
289 #if !defined(VOUCHER_USE_PERSONA)
290 #if VOUCHER_USE_ATTR_BANK && defined(BANK_PERSONA_TOKEN) && \
291 !TARGET_IPHONE_SIMULATOR
292 #define VOUCHER_USE_PERSONA 1
293 #else
294 #define VOUCHER_USE_PERSONA 0
295 #endif
296 #endif
297
298 void
299 _voucher_task_mach_voucher_init(void* ctxt DISPATCH_UNUSED)
300 {
301 #if VOUCHER_USE_ATTR_BANK
302 kern_return_t kr;
303 mach_voucher_t kv;
304 static const mach_voucher_attr_recipe_data_t task_create_recipe = {
305 .key = MACH_VOUCHER_ATTR_KEY_BANK,
306 .command = MACH_VOUCHER_ATTR_BANK_CREATE,
307 };
308 kr = _voucher_create_mach_voucher(&task_create_recipe,
309 sizeof(task_create_recipe), &kv);
310 if (dispatch_assume_zero(kr)) {
311 DISPATCH_CLIENT_CRASH("Could not create task mach voucher");
312 }
313 _voucher_default_task_mach_voucher = kv;
314 _voucher_task_mach_voucher = kv;
315 #endif
316 }
317
318 void
319 voucher_replace_default_voucher(void)
320 {
321 #if VOUCHER_USE_ATTR_BANK
322 (void)_voucher_get_task_mach_voucher(); // initalize task mach voucher
323 mach_voucher_t kv, tkv = MACH_VOUCHER_NULL;
324 voucher_t v = _voucher_get();
325 if (v && v->v_kvoucher) {
326 kern_return_t kr;
327 kv = v->v_ipc_kvoucher ? v->v_ipc_kvoucher : v->v_kvoucher;
328 const mach_voucher_attr_recipe_data_t task_copy_recipe = {
329 .key = MACH_VOUCHER_ATTR_KEY_BANK,
330 .command = MACH_VOUCHER_ATTR_COPY,
331 .previous_voucher = kv,
332 };
333 kr = _voucher_create_mach_voucher(&task_copy_recipe,
334 sizeof(task_copy_recipe), &tkv);
335 if (dispatch_assume_zero(kr)) {
336 tkv = MACH_VOUCHER_NULL;
337 }
338 }
339 if (!tkv) tkv = _voucher_default_task_mach_voucher;
340 kv = dispatch_atomic_xchg(&_voucher_task_mach_voucher, tkv, relaxed);
341 if (kv && kv != _voucher_default_task_mach_voucher) {
342 _voucher_dealloc_mach_voucher(kv);
343 }
344 _dispatch_voucher_debug("kvoucher[0x%08x] replace default voucher", v, tkv);
345 #endif
346 }
347
348 static inline _voucher_atm_t
349 _voucher_get_atm(voucher_t voucher)
350 {
351 _voucher_atm_t vatm;
352 vatm = voucher && voucher->v_atm ? voucher->v_atm : _voucher_task_atm;
353 return vatm;
354 }
355
356 static inline mach_voucher_t
357 _voucher_get_atm_mach_voucher(voucher_t voucher)
358 {
359 _voucher_atm_t vatm = _voucher_get_atm(voucher);
360 mach_voucher_t kv = vatm ? vatm->vatm_kvoucher : MACH_VOUCHER_NULL;
361 return kv;
362 }
363
364 mach_voucher_t
365 _voucher_get_mach_voucher(voucher_t voucher)
366 {
367 if (!voucher) return MACH_VOUCHER_NULL;
368 if (voucher->v_ipc_kvoucher) return voucher->v_ipc_kvoucher;
369 mach_voucher_t kvb = voucher->v_kvoucher;
370 if (!kvb) kvb = _voucher_get_task_mach_voucher();
371 if (!voucher->v_has_priority && !voucher->v_activities &&
372 !_voucher_extra_size(voucher)) {
373 return kvb;
374 }
375 kern_return_t kr;
376 mach_voucher_t kv, kvo;
377 _voucher_base_recipe(voucher).previous_voucher = kvb;
378 _voucher_atm_recipe(voucher).previous_voucher =
379 _voucher_get_atm_mach_voucher(voucher);
380 size_t recipes_size = _voucher_recipes_size() +
381 _voucher_extra_size(voucher) +
382 _voucher_bits_recipe(voucher).content_size;
383 kr = _voucher_create_mach_voucher(voucher->v_recipes, recipes_size, &kv);
384 if (dispatch_assume_zero(kr) || !kv){
385 return MACH_VOUCHER_NULL;
386 }
387 if (!dispatch_atomic_cmpxchgv2o(voucher, v_ipc_kvoucher, MACH_VOUCHER_NULL,
388 kv, &kvo, relaxed)) {
389 _voucher_dealloc_mach_voucher(kv);
390 kv = kvo;
391 } else {
392 if (kv == voucher->v_kvoucher) {
393 // if v_kvoucher == v_ipc_kvoucher we keep only one reference
394 _voucher_dealloc_mach_voucher(kv);
395 }
396 _voucher_insert(voucher);
397 _dispatch_voucher_debug("kvoucher[0x%08x] create", voucher, kv);
398 }
399 return kv;
400 }
401
402 mach_voucher_t
403 _voucher_create_mach_voucher_with_priority(voucher_t voucher,
404 pthread_priority_t priority)
405 {
406 if (priority == _voucher_get_priority(voucher)) {
407 return MACH_VOUCHER_NULL; // caller will use _voucher_get_mach_voucher
408 }
409 kern_return_t kr;
410 mach_voucher_t kv, kvb = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL;
411 if (!kvb) kvb = _voucher_get_task_mach_voucher();
412 mach_voucher_attr_recipe_data_t *recipes;
413 size_t recipes_size = _voucher_recipes_size();
414 if (voucher && (voucher->v_has_priority || voucher->v_activities ||
415 _voucher_extra_size(voucher))) {
416 recipes_size += _voucher_bits_recipe(voucher).content_size +
417 _voucher_extra_size(voucher);
418 recipes = alloca(recipes_size);
419 memcpy(recipes, voucher->v_recipes, recipes_size);
420 _voucher_recipes_atm(recipes).previous_voucher =
421 _voucher_get_atm_mach_voucher(voucher);
422 } else {
423 mach_voucher_attr_content_size_t bits_size = _voucher_bits_size(0);
424 recipes_size += bits_size;
425 recipes = alloca(recipes_size);
426 _voucher_recipes_init(recipes, bits_size);
427 }
428 _voucher_recipes_base(recipes).previous_voucher = kvb;
429 *_voucher_recipes_priority(recipes) = (_voucher_priority_t)priority;
430 kr = _voucher_create_mach_voucher(recipes, recipes_size, &kv);
431 if (dispatch_assume_zero(kr) || !kv){
432 return MACH_VOUCHER_NULL;
433 }
434 _dispatch_kvoucher_debug("create with priority from voucher[%p]", kv,
435 voucher);
436 return kv;
437 }
438
439 static voucher_t
440 _voucher_create_with_mach_voucher(mach_voucher_t kv)
441 {
442 if (!kv) return NULL;
443 kern_return_t kr;
444 mach_voucher_attr_recipe_t vr;
445 size_t vr_size;
446 mach_voucher_attr_recipe_size_t kvr_size = 0;
447 voucher_t v = _voucher_find_and_retain(kv);
448 if (v) {
449 _dispatch_voucher_debug("kvoucher[0x%08x] found", v, kv);
450 _voucher_dealloc_mach_voucher(kv);
451 return v;
452 }
453 vr_size = sizeof(*vr) + _voucher_bits_size(_voucher_max_activities);
454 vr = alloca(vr_size);
455 if (kv) {
456 kvr_size = (mach_voucher_attr_recipe_size_t)vr_size;
457 kr = mach_voucher_extract_attr_recipe(kv,
458 MACH_VOUCHER_ATTR_KEY_USER_DATA, (void*)vr, &kvr_size);
459 DISPATCH_VERIFY_MIG(kr);
460 if (dispatch_assume_zero(kr)) kvr_size = 0;
461 }
462 mach_voucher_attr_content_size_t content_size = vr->content_size;
463 uint8_t *content = vr->content;
464 bool valid = false, has_priority = false;
465 unsigned int activities = 0;
466 if (kvr_size >= sizeof(*vr) + sizeof(_voucher_magic_t)) {
467 valid = (*(_voucher_magic_t*)content == _voucher_magic_v1);
468 content += sizeof(_voucher_magic_t);
469 content_size -= sizeof(_voucher_magic_t);
470 }
471 if (valid) {
472 has_priority = (content_size >= sizeof(_voucher_priority_t));
473 activities = has_priority ? (content_size - sizeof(_voucher_priority_t))
474 / sizeof(voucher_activity_id_t) : 0;
475 }
476 pthread_priority_t priority = 0;
477 if (has_priority) {
478 priority = (pthread_priority_t)*(_voucher_priority_t*)content;
479 content += sizeof(_voucher_priority_t);
480 content_size -= sizeof(_voucher_priority_t);
481 }
482 voucher_activity_id_t va_id = 0, va_base_id = 0;
483 _voucher_activity_t act = NULL;
484 _voucher_atm_t vatm = NULL;
485 if (activities) {
486 va_id = *(voucher_activity_id_t*)content;
487 act = _voucher_activity_copy_from_mach_voucher(kv, va_id);
488 if (!act && _voucher_activity_default) {
489 activities++;
490 // default to _voucher_activity_default base activity
491 va_base_id = _voucher_activity_default->va_id;
492 } else if (act && act->va_id != va_id) {
493 activities++;
494 va_base_id = act->va_id;
495 }
496 if (act) {
497 vatm = _voucher_atm_retain(act->va_atm);
498 }
499 }
500 v = _voucher_alloc(activities, priority, 0);
501 v->v_atm = vatm;
502 v->v_activity = act;
503 voucher_activity_id_t *activity_ids = _voucher_activity_ids(v);
504 if (activities && va_base_id) {
505 *activity_ids++ = va_base_id;
506 activities--;
507 }
508 if (activities) {
509 memcpy(activity_ids, content, content_size);
510 }
511 v->v_ipc_kvoucher = v->v_kvoucher = kv;
512 _voucher_insert(v);
513 _dispatch_voucher_debug("kvoucher[0x%08x] create", v, kv);
514 return v;
515 }
516
517 voucher_t
518 _voucher_create_with_priority_and_mach_voucher(voucher_t ov,
519 pthread_priority_t priority, mach_voucher_t kv)
520 {
521 if (priority == _voucher_get_priority(ov)) {
522 if (kv) _voucher_dealloc_mach_voucher(kv);
523 return ov ? _voucher_retain(ov) : NULL;
524 }
525 voucher_t v = _voucher_find_and_retain(kv);
526 if (v) {
527 _dispatch_voucher_debug("kvoucher[0x%08x] find", v, kv);
528 _voucher_dealloc_mach_voucher(kv);
529 return v;
530 }
531 unsigned int activities = ov ? ov->v_activities : 0;
532 mach_voucher_attr_recipe_size_t extra = ov ? _voucher_extra_size(ov) : 0;
533 v = _voucher_alloc(activities, priority, extra);
534 if (extra) {
535 memcpy(_voucher_extra_recipes(v), _voucher_extra_recipes(ov), extra);
536 }
537 if (activities) {
538 if (ov->v_activity) {
539 v->v_activity = _voucher_activity_retain(ov->v_activity);
540 v->v_atm = _voucher_atm_retain(ov->v_atm);
541 }
542 memcpy(_voucher_activity_ids(v), _voucher_activity_ids(ov),
543 activities * sizeof(voucher_activity_id_t));
544 }
545 if (kv) {
546 v->v_ipc_kvoucher = v->v_kvoucher = kv;
547 _voucher_insert(v);
548 _dispatch_voucher_debug("kvoucher[0x%08x] create with priority from "
549 "voucher[%p]", v, kv, ov);
550 _dispatch_voucher_debug_machport(kv);
551 } else if (ov && ov->v_kvoucher) {
552 voucher_t kvb = ov->v_kvbase ? ov->v_kvbase : ov;
553 v->v_kvbase = _voucher_retain(kvb);
554 v->v_kvoucher = kvb->v_kvoucher;
555 }
556 return v;
557 }
558
559 voucher_t
560 _voucher_create_without_importance(voucher_t ov)
561 {
562 // Nothing to do unless the old voucher has a kernel voucher. If it
563 // doesn't, it can't have any importance, now or in the future.
564 if (!ov) return NULL;
565 // TODO: 17487167: track presence of importance attribute
566 if (!ov->v_kvoucher) return _voucher_retain(ov);
567 kern_return_t kr;
568 mach_voucher_t kv, okv;
569 // Copy kernel voucher, removing importance.
570 okv = ov->v_ipc_kvoucher ? ov->v_ipc_kvoucher : ov->v_kvoucher;
571 const mach_voucher_attr_recipe_data_t importance_remove_recipe[] = {
572 [0] = {
573 .key = MACH_VOUCHER_ATTR_KEY_ALL,
574 .command = MACH_VOUCHER_ATTR_COPY,
575 .previous_voucher = okv,
576 },
577 [1] = {
578 .key = MACH_VOUCHER_ATTR_KEY_IMPORTANCE,
579 .command = MACH_VOUCHER_ATTR_REMOVE,
580 },
581 };
582 kr = _voucher_create_mach_voucher(importance_remove_recipe,
583 sizeof(importance_remove_recipe), &kv);
584 if (dispatch_assume_zero(kr) || !kv){
585 if (ov->v_ipc_kvoucher) return NULL;
586 kv = MACH_VOUCHER_NULL;
587 }
588 if (kv == okv) {
589 _voucher_dealloc_mach_voucher(kv);
590 return _voucher_retain(ov);
591 }
592 voucher_t v = _voucher_find_and_retain(kv);
593 if (v && ov->v_ipc_kvoucher) {
594 _dispatch_voucher_debug("kvoucher[0x%08x] find without importance "
595 "from voucher[%p]", v, kv, ov);
596 _voucher_dealloc_mach_voucher(kv);
597 return v;
598 }
599 voucher_t kvbase = v;
600 // Copy userspace contents
601 unsigned int activities = ov->v_activities;
602 pthread_priority_t priority = _voucher_get_priority(ov);
603 mach_voucher_attr_recipe_size_t extra = _voucher_extra_size(ov);
604 v = _voucher_alloc(activities, priority, extra);
605 if (extra) {
606 memcpy(_voucher_extra_recipes(v), _voucher_extra_recipes(ov), extra);
607 }
608 if (activities) {
609 if (ov->v_activity) {
610 v->v_activity = _voucher_activity_retain(ov->v_activity);
611 v->v_atm = _voucher_atm_retain(ov->v_atm);
612 }
613 memcpy(_voucher_activity_ids(v), _voucher_activity_ids(ov),
614 activities * sizeof(voucher_activity_id_t));
615 }
616 v->v_kvoucher = kv;
617 if (ov->v_ipc_kvoucher) {
618 v->v_ipc_kvoucher = kv;
619 _voucher_insert(v);
620 } else if (kvbase) {
621 v->v_kvbase = kvbase;
622 _voucher_dealloc_mach_voucher(kv); // borrow base reference
623 }
624 if (!kvbase) {
625 _dispatch_voucher_debug("kvoucher[0x%08x] create without importance "
626 "from voucher[%p]", v, kv, ov);
627 }
628 return v;
629 }
630
631 voucher_t
632 _voucher_create_accounting_voucher(voucher_t ov)
633 {
634 // Nothing to do unless the old voucher has a kernel voucher. If it does
635 // doesn't, it can't have any accounting attributes.
636 if (!ov || !ov->v_kvoucher) return NULL;
637 kern_return_t kr = KERN_SUCCESS;
638 mach_voucher_t okv, kv = MACH_VOUCHER_NULL;
639 okv = ov->v_ipc_kvoucher ? ov->v_ipc_kvoucher : ov->v_kvoucher;
640 #if VOUCHER_USE_ATTR_BANK
641 const mach_voucher_attr_recipe_data_t accounting_copy_recipe = {
642 .key = MACH_VOUCHER_ATTR_KEY_BANK,
643 .command = MACH_VOUCHER_ATTR_COPY,
644 .previous_voucher = okv,
645 };
646 kr = _voucher_create_mach_voucher(&accounting_copy_recipe,
647 sizeof(accounting_copy_recipe), &kv);
648 #endif
649 if (dispatch_assume_zero(kr) || !kv){
650 return NULL;
651 }
652 voucher_t v = _voucher_find_and_retain(kv);
653 if (v) {
654 _dispatch_voucher_debug("kvoucher[0x%08x] find accounting voucher "
655 "from voucher[%p]", v, kv, ov);
656 _voucher_dealloc_mach_voucher(kv);
657 return v;
658 }
659 v = _voucher_alloc(0, 0, 0);
660 v->v_ipc_kvoucher = v->v_kvoucher = kv;
661 if (kv == okv) {
662 v->v_kvbase = _voucher_retain(ov);
663 _voucher_dealloc_mach_voucher(kv); // borrow base reference
664 }
665 _voucher_insert(v);
666 _dispatch_voucher_debug("kvoucher[0x%08x] create accounting voucher "
667 "from voucher[%p]", v, kv, ov);
668 return v;
669 }
670
671 voucher_t
672 voucher_create_with_mach_msg(mach_msg_header_t *msg)
673 {
674 voucher_t v = _voucher_create_with_mach_voucher(_voucher_mach_msg_get(msg));
675 _voucher_activity_trace_msg(v, msg, receive);
676 return v;
677 }
678
679 #ifndef MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL
680 #define MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL 2
681 #endif
682
683 void
684 voucher_decrement_importance_count4CF(voucher_t v)
685 {
686 if (!v || !v->v_kvoucher) return;
687 // TODO: 17487167: track presence of importance attribute
688 kern_return_t kr;
689 mach_voucher_t kv = v->v_ipc_kvoucher ? v->v_ipc_kvoucher : v->v_kvoucher;
690 uint32_t dec = 1;
691 mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&dec;
692 mach_voucher_attr_content_size_t kvc_in_size = sizeof(dec);
693 mach_voucher_attr_content_t kvc_out = NULL;
694 mach_voucher_attr_content_size_t kvc_out_size = 0;
695 #if DISPATCH_DEBUG
696 uint32_t count = UINT32_MAX;
697 kvc_out = (mach_voucher_attr_content_t)&count;
698 kvc_out_size = sizeof(count);
699 #endif
700 kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_IMPORTANCE,
701 MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL, kvc_in, kvc_in_size,
702 kvc_out, &kvc_out_size);
703 DISPATCH_VERIFY_MIG(kr);
704 #if DISPATCH_DEBUG
705 _dispatch_voucher_debug("kvoucher[0x%08x] decrement importance count to %u:"
706 " %s - 0x%x", v, kv, count, mach_error_string(kr), kr);
707 #endif
708 if (kr != KERN_INVALID_ARGUMENT &&
709 dispatch_assume_zero(kr) == KERN_FAILURE) {
710 // TODO: 17487167: skip KERN_INVALID_ARGUMENT check
711 DISPATCH_CLIENT_CRASH("Voucher importance count underflow");
712 }
713 }
714
715 #if VOUCHER_ENABLE_GET_MACH_VOUCHER
716 mach_voucher_t
717 voucher_get_mach_voucher(voucher_t voucher)
718 {
719 return _voucher_get_mach_voucher(voucher);
720 }
721 #endif
722
723 void
724 _voucher_xref_dispose(voucher_t voucher)
725 {
726 _dispatch_voucher_debug("xref_dispose", voucher);
727 _voucher_remove(voucher);
728 return _os_object_release_internal_inline((_os_object_t)voucher);
729 }
730
731 void
732 _voucher_dispose(voucher_t voucher)
733 {
734 _dispatch_voucher_debug("dispose", voucher);
735 if (slowpath(_TAILQ_IS_ENQUEUED(voucher, v_list))) {
736 _dispatch_voucher_debug("corruption", voucher);
737 DISPATCH_CRASH("Voucher corruption");
738 }
739 voucher->v_list.tqe_next = DISPATCH_OBJECT_LISTLESS;
740 if (voucher->v_ipc_kvoucher) {
741 if (voucher->v_ipc_kvoucher != voucher->v_kvoucher) {
742 _voucher_dealloc_mach_voucher(voucher->v_ipc_kvoucher);
743 }
744 voucher->v_ipc_kvoucher = MACH_VOUCHER_NULL;
745 }
746 if (voucher->v_kvoucher) {
747 if (!voucher->v_kvbase) {
748 _voucher_dealloc_mach_voucher(voucher->v_kvoucher);
749 }
750 voucher->v_kvoucher = MACH_VOUCHER_NULL;
751 }
752 if (voucher->v_kvbase) {
753 _voucher_release(voucher->v_kvbase);
754 voucher->v_kvbase = NULL;
755 }
756 if (voucher->v_activity) {
757 _voucher_activity_release(voucher->v_activity);
758 voucher->v_activity = NULL;
759 }
760 if (voucher->v_atm) {
761 _voucher_atm_release(voucher->v_atm);
762 voucher->v_atm = NULL;
763 }
764 voucher->v_has_priority = 0;
765 voucher->v_activities = 0;
766 #if VOUCHER_ENABLE_RECIPE_OBJECTS
767 voucher->v_recipe_extra_size = 0;
768 voucher->v_recipe_extra_offset = 0;
769 #endif
770 return _os_object_dealloc((_os_object_t)voucher);
771 }
772
773 void
774 _voucher_atfork_child(void)
775 {
776 _voucher_activity_atfork_child();
777 _dispatch_thread_setspecific(dispatch_voucher_key, NULL);
778 _voucher_task_mach_voucher_pred = 0;
779 _voucher_task_mach_voucher = MACH_VOUCHER_NULL;
780
781 // TODO: voucher/activity inheritance on fork ?
782 }
783
784 #if VOUCHER_EXPORT_PERSONA_SPI
785 #if VOUCHER_USE_PERSONA
786 static kern_return_t
787 _voucher_get_current_persona_token(struct persona_token *token)
788 {
789 kern_return_t kr = KERN_FAILURE;
790 voucher_t v = _voucher_get();
791
792 if (v && v->v_kvoucher) {
793 mach_voucher_t kv = v->v_ipc_kvoucher ?: v->v_kvoucher;
794 mach_voucher_attr_content_t kvc_in = NULL;
795 mach_voucher_attr_content_size_t kvc_in_size = 0;
796 mach_voucher_attr_content_t kvc_out =
797 (mach_voucher_attr_content_t)token;
798 mach_voucher_attr_content_size_t kvc_out_size = sizeof(*token);
799
800 kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_BANK,
801 BANK_PERSONA_TOKEN, kvc_in, kvc_in_size,
802 kvc_out, &kvc_out_size);
803 if (kr != KERN_NOT_SUPPORTED
804 // Voucher doesn't have a PERSONA_TOKEN
805 && kr != KERN_INVALID_VALUE
806 // Kernel doesn't understand BANK_PERSONA_TOKEN
807 && kr != KERN_INVALID_ARGUMENT) {
808 (void)dispatch_assume_zero(kr);
809 }
810 }
811 return kr;
812 }
813 #endif
814
815 uid_t
816 voucher_get_current_persona(void)
817 {
818 uid_t persona_id = PERSONA_ID_NONE;
819
820 #if VOUCHER_USE_PERSONA
821 struct persona_token token;
822 int err;
823
824 if (_voucher_get_current_persona_token(&token) == KERN_SUCCESS) {
825 return token.originator.persona_id;
826 }
827
828 // falling back to the process persona if there is no adopted voucher
829 if (kpersona_get(&persona_id) < 0) {
830 err = errno;
831 if (err != ESRCH) {
832 (void)dispatch_assume_zero(err);
833 }
834 }
835 #endif
836 return persona_id;
837 }
838
839 int
840 voucher_get_current_persona_originator_info(struct proc_persona_info *persona_info)
841 {
842 #if VOUCHER_USE_PERSONA
843 struct persona_token token;
844 if (_voucher_get_current_persona_token(&token) == KERN_SUCCESS) {
845 *persona_info = token.originator;
846 return 0;
847 }
848 #else
849 (void)persona_info;
850 #endif
851 return -1;
852 }
853
854 int
855 voucher_get_current_persona_proximate_info(struct proc_persona_info *persona_info)
856 {
857 #if VOUCHER_USE_PERSONA
858 struct persona_token token;
859 if (_voucher_get_current_persona_token(&token) == KERN_SUCCESS) {
860 *persona_info = token.proximate;
861 return 0;
862 }
863 #else
864 (void)persona_info;
865 #endif
866 return -1;
867 }
868 #endif
869
870 #pragma mark -
871 #pragma mark _voucher_init
872
873 boolean_t
874 voucher_mach_msg_set(mach_msg_header_t *msg)
875 {
876 voucher_t v = _voucher_get();
877 bool clear_voucher = _voucher_mach_msg_set(msg, v);
878 if (clear_voucher) _voucher_activity_trace_msg(v, msg, send);
879 return clear_voucher;
880 }
881
882 void
883 voucher_mach_msg_clear(mach_msg_header_t *msg)
884 {
885 (void)_voucher_mach_msg_clear(msg, false);
886 }
887
888 voucher_mach_msg_state_t
889 voucher_mach_msg_adopt(mach_msg_header_t *msg)
890 {
891 mach_voucher_t kv = _voucher_mach_msg_get(msg);
892 if (!kv) return VOUCHER_MACH_MSG_STATE_UNCHANGED;
893 voucher_t v = _voucher_create_with_mach_voucher(kv);
894 _voucher_activity_trace_msg(v, msg, receive);
895 return (voucher_mach_msg_state_t)_voucher_adopt(v);
896 }
897
898 void
899 voucher_mach_msg_revert(voucher_mach_msg_state_t state)
900 {
901 if (state == VOUCHER_MACH_MSG_STATE_UNCHANGED) return;
902 _voucher_replace((voucher_t)state);
903 }
904
905 #if DISPATCH_USE_LIBKERNEL_VOUCHER_INIT
906 #include <_libkernel_init.h>
907
908 static const struct _libkernel_voucher_functions _voucher_libkernel_functions =
909 {
910 .version = 1,
911 .voucher_mach_msg_set = voucher_mach_msg_set,
912 .voucher_mach_msg_clear = voucher_mach_msg_clear,
913 .voucher_mach_msg_adopt = voucher_mach_msg_adopt,
914 .voucher_mach_msg_revert = voucher_mach_msg_revert,
915 };
916
917 static void
918 _voucher_libkernel_init(void)
919 {
920 kern_return_t kr = __libkernel_voucher_init(&_voucher_libkernel_functions);
921 dispatch_assert(!kr);
922 }
923 #else
924 #define _voucher_libkernel_init()
925 #endif
926
927 void
928 _voucher_init(void)
929 {
930 _voucher_libkernel_init();
931 char *e;
932 unsigned int i;
933 for (i = 0; i < VL_HASH_SIZE; i++) {
934 TAILQ_INIT(&_vouchers[i]);
935 }
936 voucher_activity_mode_t mode;
937 mode = DISPATCH_DEBUG ? voucher_activity_mode_debug
938 : voucher_activity_mode_release;
939 e = getenv("OS_ACTIVITY_MODE");
940 if (e) {
941 if (strcmp(e, "release") == 0) {
942 mode = voucher_activity_mode_release;
943 } else if (strcmp(e, "debug") == 0) {
944 mode = voucher_activity_mode_debug;
945 } else if (strcmp(e, "stream") == 0) {
946 mode = voucher_activity_mode_stream;
947 } else if (strcmp(e, "disable") == 0) {
948 mode = voucher_activity_mode_disable;
949 }
950 }
951 _voucher_activity_mode = mode;
952 if (_voucher_activity_disabled()) return;
953
954 // default task activity
955 bool default_task_activity = DISPATCH_DEBUG;
956 e = getenv("LIBDISPATCH_DEFAULT_TASK_ACTIVITY");
957 if (e) default_task_activity = atoi(e);
958 if (default_task_activity) {
959 (void)voucher_activity_start(_voucher_activity_trace_id_release, 0);
960 }
961 }
962
963 #pragma mark -
964 #pragma mark _voucher_activity_lock_s
965
966 DISPATCH_ALWAYS_INLINE
967 static inline void
968 _voucher_activity_lock_init(_voucher_activity_lock_s *lock) {
969 static const os_lock_handoff_s _os_lock_handoff_init = OS_LOCK_HANDOFF_INIT;
970 *lock = _os_lock_handoff_init;
971 }
972
973 DISPATCH_ALWAYS_INLINE
974 static inline void
975 _voucher_activity_lock_lock(_voucher_activity_lock_s *lock) {
976 return os_lock_lock(lock);
977 }
978
979 DISPATCH_ALWAYS_INLINE
980 static inline void
981 _voucher_activity_lock_unlock(_voucher_activity_lock_s *lock) {
982 return os_lock_unlock(lock);
983 }
984
985 #pragma mark -
986 #pragma mark _voucher_activity_heap
987
988 #if __has_extension(c_static_assert)
989 _Static_assert(sizeof(struct _voucher_activity_tracepoint_s) == 64,
990 "Tracepoint too large");
991 _Static_assert(sizeof(struct _voucher_activity_buffer_header_s) <=
992 sizeof(struct _voucher_activity_tracepoint_s),
993 "Buffer header too large");
994 #if __LP64__
995 _Static_assert(offsetof(struct _voucher_activity_s, va_buffers_lock) % 64 == 0,
996 "Bad activity padding");
997 _Static_assert(sizeof(struct _voucher_atm_s) <= 128,
998 "ATM too large");
999 #else
1000 _Static_assert(sizeof(struct _voucher_atm_s) <= 64,
1001 "ATM too large");
1002 #endif
1003 _Static_assert(sizeof(_voucher_activity_buffer_t) ==
1004 sizeof(struct {char x[_voucher_activity_buffer_size];}),
1005 "Buffer too large");
1006 _Static_assert(sizeof(struct _voucher_activity_metadata_s) <=
1007 sizeof(struct _voucher_activity_metadata_opaque_s),
1008 "Metadata too large");
1009 _Static_assert(sizeof(_voucher_activity_bitmap_t) % 64 == 0,
1010 "Bad metadata bitmap size");
1011 #endif
1012
1013 #define va_buffers_lock(va) (&(va)->va_buffers_lock)
1014 #define vatm_activities(vatm) (&(vatm)->vatm_activities)
1015 #define vam_atms_lock() (&_voucher_activity_heap->vam_atms_lock)
1016 #define vam_activities_lock() (&_voucher_activity_heap->vam_activities_lock)
1017 #define vam_atms(hash) (&_voucher_activity_heap->vam_atms[hash])
1018 #define vam_activities(hash) (&_voucher_activity_heap->vam_activities[hash])
1019 #define vam_buffer_bitmap() (_voucher_activity_heap->vam_buffer_bitmap)
1020 #define vam_pressure_locked_bitmap() \
1021 (_voucher_activity_heap->vam_pressure_locked_bitmap)
1022 #define vam_buffer(i) ((void*)((char*)_voucher_activity_heap + \
1023 (i) * _voucher_activity_buffer_size))
1024
1025 static _voucher_activity_t _voucher_activity_create_with_atm(
1026 _voucher_atm_t vatm, voucher_activity_id_t va_id,
1027 voucher_activity_trace_id_t trace_id, uint64_t location,
1028 _voucher_activity_buffer_header_t buffer);
1029 static _voucher_atm_t _voucher_atm_create(mach_voucher_t kv, atm_aid_t atm_id);
1030 static void _voucher_activity_firehose_wait(_voucher_activity_t act,
1031 _voucher_activity_buffer_header_t buffer);
1032
1033 DISPATCH_ALWAYS_INLINE
1034 static inline uint32_t
1035 _voucher_default_activity_buffer_limit()
1036 {
1037 #if 0 // FIXME: tune buffer chain sizes
1038 switch (_voucher_activity_mode) {
1039 case voucher_activity_mode_debug:
1040 case voucher_activity_mode_stream:
1041 // High-profile modes: Default activity can use 1/32nd of the heap
1042 // (twice as much as non-default activities)
1043 return MAX(_voucher_activity_buffers_per_heap / 32, 3) - 1;
1044 }
1045 #endif
1046 // Low-profile modes: Default activity can use a total of 4 buffers.
1047 return 3;
1048 }
1049
1050 DISPATCH_ALWAYS_INLINE
1051 static inline uint32_t
1052 _voucher_activity_buffer_limit()
1053 {
1054 #if 0 // FIXME: tune buffer chain sizes
1055 switch (_voucher_activity_mode) {
1056 case voucher_activity_mode_debug:
1057 case voucher_activity_mode_stream:
1058 // High-profile modes: 64 activities, each of which can use 1/64th
1059 // of the entire heap.
1060 return MAX(_voucher_activity_buffers_per_heap / 64, 2) - 1;
1061 }
1062 #endif
1063 // Low-profile modes: Each activity can use a total of 2 buffers.
1064 return 1;
1065 }
1066
1067 // The two functions above return the number of *additional* buffers activities
1068 // may allocate, hence the gymnastics with - 1.
1069
1070 DISPATCH_ALWAYS_INLINE
1071 static inline uint32_t
1072 _voucher_heap_buffer_limit()
1073 {
1074 switch (_voucher_activity_mode) {
1075 case voucher_activity_mode_debug:
1076 case voucher_activity_mode_stream:
1077 // High-profile modes: Use it all.
1078 return _voucher_activity_buffers_per_heap;
1079 }
1080 #if TARGET_OS_EMBEDDED
1081 // Low-profile modes: 3 activities, each of which can use 2 buffers;
1082 // plus the default activity, which can use 3; plus 3 buffers of overhead.
1083 return 12;
1084 #else
1085 // Low-profile modes: 13 activities, each of which can use 4 buffers;
1086 // plus the default activity, which can use 8; plus 3 buffers of overhead.
1087 return 64;
1088 #endif
1089 }
1090
1091 #define NO_BITS_WERE_UNSET (UINT_MAX)
1092
1093 DISPATCH_ALWAYS_INLINE
1094 static inline size_t
1095 _voucher_activity_bitmap_set_first_unset_bit_upto(
1096 _voucher_activity_bitmap_t volatile bitmap,
1097 unsigned int max_index)
1098 {
1099 dispatch_assert(max_index != 0);
1100 unsigned int index = NO_BITS_WERE_UNSET, max_map, max_bit, i;
1101 max_map = max_index / _voucher_activity_bits_per_bitmap_base_t;
1102 max_map = MIN(max_map, _voucher_activity_bitmaps_per_heap - 1);
1103 max_bit = max_index % _voucher_activity_bits_per_bitmap_base_t;
1104 for (i = 0; i < max_map; i++) {
1105 index = dispatch_atomic_set_first_bit(&bitmap[i], UINT_MAX);
1106 if (fastpath(index < NO_BITS_WERE_UNSET)) {
1107 return index + i * _voucher_activity_bits_per_bitmap_base_t;
1108 }
1109 }
1110 index = dispatch_atomic_set_first_bit(&bitmap[i], max_bit);
1111 if (fastpath(index < NO_BITS_WERE_UNSET)) {
1112 return index + i * _voucher_activity_bits_per_bitmap_base_t;
1113 }
1114 return index;
1115 }
1116
1117 DISPATCH_ALWAYS_INLINE DISPATCH_UNUSED
1118 static inline size_t
1119 _voucher_activity_bitmap_set_first_unset_bit(
1120 _voucher_activity_bitmap_t volatile bitmap)
1121 {
1122 return _voucher_activity_bitmap_set_first_unset_bit_upto(bitmap, UINT_MAX);
1123 }
1124
1125 DISPATCH_ALWAYS_INLINE
1126 static inline void
1127 _voucher_activity_bitmap_clear_bit(
1128 _voucher_activity_bitmap_t volatile bitmap, size_t index)
1129 {
1130 size_t i = index / _voucher_activity_bits_per_bitmap_base_t;
1131 _voucher_activity_bitmap_base_t mask = ((typeof(mask))1) <<
1132 (index % _voucher_activity_bits_per_bitmap_base_t);
1133 if (slowpath((bitmap[i] & mask) == 0)) {
1134 DISPATCH_CRASH("Corruption: failed to clear bit exclusively");
1135 }
1136 (void)dispatch_atomic_and(&bitmap[i], ~mask, release);
1137 }
1138
1139 _voucher_activity_metadata_t _voucher_activity_heap;
1140 static dispatch_once_t _voucher_activity_heap_pred;
1141
1142 static void
1143 _voucher_activity_heap_init(void *ctxt DISPATCH_UNUSED)
1144 {
1145 if (_voucher_activity_disabled()) return;
1146 kern_return_t kr;
1147 mach_vm_size_t vm_size = _voucher_activity_buffer_size *
1148 _voucher_activity_buffers_per_heap;
1149 mach_vm_address_t vm_addr = vm_page_size;
1150 while (slowpath(kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
1151 0, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_MEMORY_GENEALOGY),
1152 MEMORY_OBJECT_NULL, 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL,
1153 VM_INHERIT_NONE))) {
1154 if (kr != KERN_NO_SPACE) {
1155 (void)dispatch_assume_zero(kr);
1156 _voucher_activity_mode = voucher_activity_mode_disable;
1157 return;
1158 }
1159 _dispatch_temporary_resource_shortage();
1160 vm_addr = vm_page_size;
1161 }
1162 _voucher_activity_metadata_t heap;
1163 task_trace_memory_info_data_t trace_memory_info = {
1164 .user_memory_address = vm_addr,
1165 .buffer_size = vm_size,
1166 };
1167 kr = task_set_info(mach_task_self(), TASK_TRACE_MEMORY_INFO,
1168 (task_info_t)&trace_memory_info, TASK_TRACE_MEMORY_INFO_COUNT);
1169 DISPATCH_VERIFY_MIG(kr);
1170 if (kr) {
1171 if (kr != KERN_NOT_SUPPORTED) (void)dispatch_assume_zero(kr);
1172 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
1173 (void)dispatch_assume_zero(kr);
1174 _voucher_activity_mode = voucher_activity_mode_disable;
1175 return;
1176 }
1177 heap = (void*)vm_addr;
1178 heap->vasm_baseaddr = (void*)vm_addr;
1179 heap->vam_buffer_bitmap[0] = 0x7; // first three buffers are reserved
1180 uint32_t i;
1181 for (i = 0; i < _voucher_activity_hash_size; i++) {
1182 TAILQ_INIT(&heap->vam_activities[i]);
1183 TAILQ_INIT(&heap->vam_atms[i]);
1184 }
1185 _voucher_activity_lock_init(&heap->vam_atms_lock);
1186 _voucher_activity_lock_init(&heap->vam_activities_lock);
1187 _voucher_activity_heap = heap;
1188
1189 _voucher_atm_t vatm = _voucher_atm_create(0, 0);
1190 dispatch_assert(vatm->vatm_kvoucher);
1191 _voucher_atm_retain(vatm);
1192
1193 _voucher_activity_buffer_header_t buffer = vam_buffer(2); // reserved index
1194 // consumes vatm reference:
1195 _voucher_activity_t va = _voucher_activity_create_with_atm(vatm, 0, 0, 0,
1196 buffer);
1197 dispatch_assert(va);
1198 va->va_buffer_limit = _voucher_default_activity_buffer_limit();
1199 _voucher_activity_default = va;
1200 _voucher_task_atm = vatm;
1201 }
1202
1203 static void
1204 _voucher_activity_atfork_child(void)
1205 {
1206 _voucher_activity_heap_pred = 0;
1207 _voucher_activity_heap = NULL; // activity heap is VM_INHERIT_NONE
1208 _voucher_activity_default = NULL;
1209 }
1210
1211 void*
1212 voucher_activity_get_metadata_buffer(size_t *length)
1213 {
1214 dispatch_once_f(&_voucher_activity_heap_pred, NULL,
1215 _voucher_activity_heap_init);
1216 if (_voucher_activity_disabled()) {
1217 *length = 0;
1218 return NULL;
1219 }
1220 *length = sizeof(_voucher_activity_heap->vam_client_metadata);
1221 return _voucher_activity_heap->vam_client_metadata;
1222 }
1223
1224 static _voucher_activity_buffer_hook_t _voucher_activity_buffer_hook;
1225
1226 void
1227 voucher_activity_buffer_hook_install_4libtrace(
1228 _voucher_activity_buffer_hook_t hook)
1229 {
1230 if (dispatch_atomic_cmpxchg(&_voucher_activity_buffer_hook, NULL,
1231 (void*)hook, release)) return;
1232 DISPATCH_CLIENT_CRASH("_voucher_activity_buffer_hook_install_4libtrace " \
1233 "called more than once");
1234 }
1235
1236 #if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG
1237 #define VOUCHER_ACTIVITY_BUFFER_DEBUG(reason, buffer) \
1238 _dispatch_debug("activity buffer %s (%p)", #reason, buffer)
1239 #else
1240 #define VOUCHER_ACTIVITY_BUFFER_DEBUG(reason, buffer)
1241 #endif
1242
1243 #define VOUCHER_ACTIVITY_BUFFER_HOOK_CALLOUT(reason, buffer) \
1244 if (buffer) { VOUCHER_ACTIVITY_BUFFER_DEBUG(reason, buffer); \
1245 if (slowpath(_voucher_activity_buffer_hook)) { \
1246 _voucher_activity_buffer_hook( \
1247 _voucher_activity_buffer_hook_reason_##reason, (buffer)); \
1248 } }
1249
1250 DISPATCH_ALWAYS_INLINE
1251 static inline _voucher_activity_buffer_header_t
1252 _voucher_activity_heap_buffer_alloc(void)
1253 {
1254 _voucher_activity_buffer_header_t buffer = NULL;
1255 size_t index;
1256 index = _voucher_activity_bitmap_set_first_unset_bit_upto(
1257 vam_buffer_bitmap(), _voucher_heap_buffer_limit() - 1);
1258 if (index < NO_BITS_WERE_UNSET) {
1259 buffer = vam_buffer(index);
1260 }
1261 #if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG
1262 _dispatch_debug("activity heap alloc %zd (%p)", index, buffer);
1263 #endif
1264 return buffer;
1265 }
1266
1267 DISPATCH_ALWAYS_INLINE
1268 static inline void
1269 _voucher_activity_heap_buffer_free(_voucher_activity_buffer_header_t buffer)
1270 {
1271 buffer->vabh_flags = _voucher_activity_trace_flag_buffer_empty;
1272 size_t index = (size_t)((char*)buffer - (char*)_voucher_activity_heap) /
1273 _voucher_activity_buffer_size;
1274 #if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG
1275 _dispatch_debug("activity heap free %zd (%p)", index, buffer);
1276 #endif
1277 _voucher_activity_bitmap_clear_bit(vam_buffer_bitmap(), index);
1278 }
1279
1280 #define _voucher_activity_heap_can_madvise() \
1281 (PAGE_SIZE == _voucher_activity_buffer_size) // <rdar://17445544>
1282
1283 DISPATCH_ALWAYS_INLINE
1284 static inline void
1285 _voucher_activity_heap_madvise(size_t bitmap_num, unsigned int start,
1286 unsigned int len)
1287 {
1288 size_t base = bitmap_num * _voucher_activity_bits_per_bitmap_base_t;
1289 #if DISPATCH_DEBUG
1290 #if DISPATCH_VOUCHER_ACTIVITY_DEBUG
1291 _dispatch_debug("activity heap madvise %zd (%p) -> %zd (%p)", base + start,
1292 vam_buffer(base + start), base + start + len,
1293 vam_buffer(base + start + len));
1294 #endif
1295 dispatch_assert(!(len * _voucher_activity_buffer_size % vm_page_size));
1296 const uint64_t pattern = 0xFACEFACEFACEFACE;
1297 _voucher_activity_buffer_header_t buffer = vam_buffer(base + start);
1298 for (unsigned int i = 0; i < len; i++, buffer++) {
1299 memset_pattern8((char*)buffer + sizeof(buffer->vabh_flags), &pattern,
1300 _voucher_activity_buffer_size - sizeof(buffer->vabh_flags));
1301 }
1302 #endif
1303 (void)dispatch_assume_zero(madvise(vam_buffer(base + start),
1304 len * _voucher_activity_buffer_size, MADV_FREE));
1305 }
1306
1307 DISPATCH_ALWAYS_INLINE
1308 static inline void
1309 _voucher_activity_heap_madvise_contiguous(size_t bitmap_num,
1310 _voucher_activity_bitmap_base_t bits)
1311 {
1312 // TODO: x86 has fast ctz; arm has fast clz; haswell has fast ctz
1313 dispatch_assert(_voucher_activity_heap_can_madvise());
1314 if (bits == 0) {
1315 return;
1316 } else if (~bits == 0) {
1317 _voucher_activity_heap_madvise(bitmap_num, 0,
1318 _voucher_activity_bits_per_bitmap_base_t);
1319 } else while (bits != 0) {
1320 unsigned int start = (typeof(start))__builtin_ctzl(bits), len;
1321 typeof(bits) inverse = ~bits >> start;
1322 if (inverse) {
1323 len = (typeof(len))__builtin_ctzl(inverse);
1324 } else {
1325 len = _voucher_activity_bits_per_bitmap_base_t - start;
1326 }
1327 typeof(bits) mask = ((((typeof(bits))1) << len) - 1) << start;
1328 bits &= ~mask;
1329 _voucher_activity_heap_madvise(bitmap_num, start, len);
1330 }
1331 }
1332
1333 void
1334 _voucher_activity_heap_pressure_warn(void)
1335 {
1336 if (!_voucher_activity_heap_can_madvise() || !_voucher_activity_heap) {
1337 return;
1338 }
1339 volatile _voucher_activity_bitmap_base_t *bitmap, *pressure_locked_bitmap;
1340 bitmap = vam_buffer_bitmap();
1341 pressure_locked_bitmap = vam_pressure_locked_bitmap();
1342
1343 // number of bitmaps needed to map the current buffer limit =
1344 // ceil(buffer limit / bits per bitmap)
1345 size_t nbuffers = _voucher_heap_buffer_limit();
1346 size_t nbitmaps_quot = nbuffers / _voucher_activity_bits_per_bitmap_base_t;
1347 size_t nbitmaps_rem = nbuffers % _voucher_activity_bits_per_bitmap_base_t;
1348 size_t nbitmaps = nbitmaps_quot + ((nbitmaps_rem == 0) ? 0 : 1);
1349
1350 for (size_t i = 0; i < nbitmaps; i++) {
1351 _voucher_activity_bitmap_base_t got_bits;
1352 got_bits = dispatch_atomic_or_orig(&bitmap[i], ~((typeof(bitmap[i]))0),
1353 relaxed);
1354 got_bits = ~got_bits; // Now 1 means 'acquired this one, madvise it'
1355 _voucher_activity_heap_madvise_contiguous(i, got_bits);
1356 pressure_locked_bitmap[i] |= got_bits;
1357 }
1358 }
1359
1360 void
1361 _voucher_activity_heap_pressure_normal(void)
1362 {
1363 if (!_voucher_activity_heap_can_madvise() || !_voucher_activity_heap) {
1364 return;
1365 }
1366 volatile _voucher_activity_bitmap_base_t *bitmap, *pressure_locked_bitmap;
1367 bitmap = vam_buffer_bitmap();
1368 pressure_locked_bitmap = vam_pressure_locked_bitmap();
1369 for (size_t i = 0; i < _voucher_activity_bitmaps_per_heap; i++) {
1370 _voucher_activity_bitmap_base_t free_bits = pressure_locked_bitmap[i];
1371 pressure_locked_bitmap[i] = 0;
1372 if (free_bits != 0) {
1373 (void)dispatch_atomic_and(&bitmap[i], ~free_bits, release);
1374 }
1375 }
1376 }
1377
1378 DISPATCH_ALWAYS_INLINE
1379 static inline void
1380 _voucher_activity_buffer_init(_voucher_activity_t act,
1381 _voucher_activity_buffer_header_t buffer, bool initial)
1382 {
1383 _voucher_activity_tracepoint_t vat = (_voucher_activity_tracepoint_t)buffer;
1384 _voucher_activity_tracepoint_init_with_id(vat, act->va_trace_id,
1385 act->va_location, !initial);
1386 buffer->vabh_flags = _voucher_activity_trace_flag_buffer_header |
1387 _voucher_activity_trace_flag_activity |
1388 (initial ? _voucher_activity_trace_flag_start : 0);
1389 buffer->vabh_activity_id = act->va_id;
1390 buffer->vabh_pos.vabp_atomic_pos = 0;
1391 buffer->vabh_pos.vabp_pos.vabp_next_tracepoint_idx = 1;
1392 }
1393
1394 static _voucher_activity_buffer_header_t
1395 _voucher_activity_buffer_alloc_slow(_voucher_activity_t act,
1396 _voucher_activity_buffer_header_t current)
1397 {
1398 _voucher_activity_buffer_header_t buffer;
1399 _voucher_activity_lock_lock(va_buffers_lock(act)); // TODO: revisit locking
1400 buffer = act->va_current_buffer;
1401 if (buffer != current) {
1402 _voucher_activity_lock_unlock(va_buffers_lock(act));
1403 return buffer;
1404 }
1405 buffer = TAILQ_FIRST(&act->va_buffers);
1406 if (buffer != TAILQ_LAST(&act->va_buffers,
1407 _voucher_activity_buffer_list_s)) {
1408 TAILQ_REMOVE(&act->va_buffers, buffer, vabh_list);
1409 TAILQ_INSERT_TAIL(&act->va_buffers, buffer, vabh_list);
1410 }
1411 _voucher_activity_lock_unlock(va_buffers_lock(act));
1412 if (_voucher_activity_buffer_is_full(buffer)) {
1413 _voucher_activity_firehose_wait(act, buffer);
1414 }
1415 if (dispatch_atomic_cmpxchgv2o(act, va_current_buffer, current, buffer,
1416 &current, release)) {
1417 if (_voucher_activity_buffer_mark_full(current)) {
1418 _voucher_activity_firehose_push(act, current);
1419 }
1420 _dispatch_voucher_activity_debug("buffer reuse %p", act, buffer);
1421 } else {
1422 buffer = current;
1423 }
1424 return buffer;
1425 }
1426
1427 static _voucher_activity_buffer_header_t
1428 _voucher_activity_buffer_alloc(_voucher_activity_t act,
1429 _voucher_activity_buffer_header_t current)
1430 {
1431 _voucher_activity_buffer_header_t buffer = NULL;
1432 if (act->va_buffer_count < act->va_buffer_limit) {
1433 buffer = _voucher_activity_heap_buffer_alloc();
1434 if (buffer && dispatch_atomic_inc2o(act, va_buffer_count, relaxed) >
1435 act->va_buffer_limit) {
1436 dispatch_atomic_dec2o(act, va_buffer_count, relaxed);
1437 _voucher_activity_heap_buffer_free(buffer);
1438 buffer = NULL;
1439 }
1440 }
1441 if (!buffer) return _voucher_activity_buffer_alloc_slow(act, current);
1442 _voucher_activity_buffer_init(act, buffer, false);
1443 if (dispatch_atomic_cmpxchgv2o(act, va_current_buffer, current, buffer,
1444 &current, release)) {
1445 _voucher_activity_lock_lock(va_buffers_lock(act));
1446 TAILQ_INSERT_TAIL(&act->va_buffers, buffer, vabh_list);
1447 _voucher_activity_lock_unlock(va_buffers_lock(act));
1448 if (_voucher_activity_buffer_mark_full(current)) {
1449 _voucher_activity_firehose_push(act, current);
1450 }
1451 _dispatch_voucher_activity_debug("buffer alloc %p", act, buffer);
1452 } else {
1453 dispatch_atomic_dec2o(act, va_buffer_count, relaxed);
1454 _voucher_activity_heap_buffer_free(buffer);
1455 buffer = current;
1456 }
1457 return buffer;
1458 }
1459
1460 #pragma mark -
1461 #pragma mark _voucher_activity_t
1462
1463 #define _voucher_activity_ordered_insert(_act, head, field) do { \
1464 typeof(_act) _vai; \
1465 TAILQ_FOREACH(_vai, (head), field) { \
1466 if (_act->va_id < _vai->va_id) break; \
1467 } \
1468 if (_vai) { \
1469 TAILQ_INSERT_BEFORE(_vai, _act, field); \
1470 } else { \
1471 TAILQ_INSERT_TAIL((head), _act, field); \
1472 } } while (0);
1473
1474 static void _voucher_activity_dispose(_voucher_activity_t act);
1475 static _voucher_atm_t _voucher_atm_copy(atm_aid_t atm_id);
1476 static inline void _voucher_atm_release(_voucher_atm_t vatm);
1477 static atm_aid_t _voucher_mach_voucher_get_atm_id(mach_voucher_t kv);
1478
1479 DISPATCH_ALWAYS_INLINE
1480 static inline bool
1481 _voucher_activity_try_retain(_voucher_activity_t act)
1482 {
1483 // not using _os_object_refcnt* because we don't need barriers:
1484 // activities are immutable and are in a hash table with a lock
1485 int use_cnt = dispatch_atomic_inc2o(act, va_refcnt, relaxed);
1486 _dispatch_voucher_activity_debug("retain -> %d", act, use_cnt + 1);
1487 if (slowpath(use_cnt < 0)) {
1488 _dispatch_voucher_activity_debug("overrelease", act);
1489 DISPATCH_CRASH("Activity overrelease");
1490 }
1491 return use_cnt > 0;
1492 }
1493
1494 DISPATCH_ALWAYS_INLINE
1495 static inline _voucher_activity_t
1496 _voucher_activity_retain(_voucher_activity_t act)
1497 {
1498 if (slowpath(!_voucher_activity_try_retain(act))) {
1499 _dispatch_voucher_activity_debug("resurrection", act);
1500 DISPATCH_CRASH("Activity resurrection");
1501 }
1502 return act;
1503 }
1504
1505 DISPATCH_ALWAYS_INLINE
1506 static inline void
1507 _voucher_activity_release(_voucher_activity_t act)
1508 {
1509 // not using _os_object_refcnt* because we don't need barriers:
1510 // activities are immutable and are in a hash table with a lock
1511 int use_cnt = dispatch_atomic_dec2o(act, va_refcnt, relaxed);
1512 _dispatch_voucher_activity_debug("release -> %d", act, use_cnt + 1);
1513 if (fastpath(use_cnt >= 0)) {
1514 return;
1515 }
1516 if (slowpath(use_cnt < -1)) {
1517 _dispatch_voucher_activity_debug("overrelease", act);
1518 DISPATCH_CRASH("Activity overrelease");
1519 }
1520 _voucher_activity_remove(act);
1521 _voucher_activity_dispose(act);
1522 }
1523
1524 static _voucher_activity_t
1525 _voucher_activity_find_and_retain(voucher_activity_id_t va_id, uint32_t hash)
1526 {
1527 // not using _os_object_refcnt* because we don't need barriers:
1528 // activities are immutable and are in a hash table with a lock
1529 //
1530 // assumes vam_activities_lock held
1531 _voucher_activity_t act;
1532 TAILQ_FOREACH(act, vam_activities(hash), va_list) {
1533 if (act->va_id == va_id) {
1534 if (fastpath(_voucher_activity_try_retain(act))) {
1535 return act;
1536 }
1537
1538 // <rdar://problem/20468375> disallow resurrection
1539 dispatch_atomic_dec2o(act, va_refcnt, relaxed);
1540 _dispatch_voucher_activity_debug("undo resurrection", act);
1541 }
1542 }
1543 return NULL;
1544 }
1545
1546 static _voucher_activity_t
1547 _voucher_activity_copy_from_id(voucher_activity_id_t va_id)
1548 {
1549 uint32_t hash = VACTID_HASH(va_id);
1550 _voucher_activity_lock_lock(vam_activities_lock());
1551 _voucher_activity_t act = _voucher_activity_find_and_retain(va_id, hash);
1552 if (act) {
1553 _dispatch_voucher_activity_debug("copy from id 0x%llx", act, va_id);
1554 }
1555 _voucher_activity_lock_unlock(vam_activities_lock());
1556 return act;
1557 }
1558
1559 static _voucher_activity_t
1560 _voucher_activity_try_insert(_voucher_activity_t act_new)
1561 {
1562 voucher_activity_id_t va_id = act_new->va_id;
1563 uint32_t hash = VACTID_HASH(va_id);
1564 _voucher_activity_lock_lock(vam_activities_lock());
1565 _voucher_activity_t act = _voucher_activity_find_and_retain(va_id, hash);
1566 if (act) {
1567 _dispatch_voucher_activity_debug("try insert: failed (%p)", act,act_new);
1568 } else {
1569 if (slowpath(_TAILQ_IS_ENQUEUED(act_new, va_list))) {
1570 _dispatch_voucher_activity_debug("corruption", act_new);
1571 DISPATCH_CRASH("Activity corruption");
1572 }
1573 TAILQ_INSERT_TAIL(vam_activities(hash), act_new, va_list);
1574 _dispatch_voucher_activity_debug("try insert: succeeded", act_new);
1575 }
1576 _voucher_activity_lock_unlock(vam_activities_lock());
1577 return act;
1578 }
1579
1580 static void
1581 _voucher_activity_remove(_voucher_activity_t act)
1582 {
1583 voucher_activity_id_t va_id = act->va_id;
1584 uint32_t hash = VACTID_HASH(va_id);
1585
1586 _voucher_activity_lock_lock(vam_activities_lock());
1587 if (slowpath(!va_id || !_TAILQ_IS_ENQUEUED(act, va_list))) {
1588 _dispatch_voucher_activity_debug("corruption", act);
1589 DISPATCH_CRASH("Activity corruption");
1590 }
1591 TAILQ_REMOVE(vam_activities(hash), act, va_list);
1592 _TAILQ_MARK_NOT_ENQUEUED(act, va_list);
1593 act->va_list.tqe_next = (void*)~0ull;
1594 _dispatch_voucher_activity_debug("remove", act);
1595 _voucher_activity_lock_unlock(vam_activities_lock());
1596 }
1597
1598 static _voucher_activity_t
1599 _voucher_activity_create_with_atm(_voucher_atm_t vatm,
1600 voucher_activity_id_t va_id, voucher_activity_trace_id_t trace_id,
1601 uint64_t location, _voucher_activity_buffer_header_t buffer)
1602 {
1603 if (!buffer) buffer = _voucher_activity_heap_buffer_alloc();
1604 if (!buffer) {
1605 _dispatch_voucher_atm_debug("no buffer", vatm);
1606 _voucher_atm_release(vatm); // consume vatm reference
1607 return NULL;
1608 }
1609 _voucher_activity_t act = _dispatch_calloc(1ul,
1610 sizeof(struct _voucher_activity_s));
1611 act->va_id = va_id;
1612 act->va_trace_id = trace_id ? trace_id : _voucher_activity_trace_id_release;
1613 act->va_location = location;
1614 act->va_buffer_limit = _voucher_activity_buffer_limit();
1615 TAILQ_INIT(&act->va_buffers);
1616 act->va_current_buffer = buffer;
1617 act->va_atm = vatm; // transfer vatm reference
1618 _voucher_activity_lock_init(va_buffers_lock(act));
1619 if (dispatch_assume_zero(pthread_mutex_init(&act->va_mutex, NULL)) ||
1620 dispatch_assume_zero(pthread_cond_init(&act->va_cond, NULL))) {
1621 DISPATCH_CLIENT_CRASH("Could not initialize activity");
1622 }
1623 _TAILQ_MARK_NOT_ENQUEUED(act, va_list);
1624 _TAILQ_MARK_NOT_ENQUEUED(act, va_atm_list);
1625 _TAILQ_MARK_NOT_ENQUEUED(act, va_atm_used_list);
1626
1627 _voucher_activity_buffer_init(act, buffer, true);
1628 TAILQ_INSERT_TAIL(&act->va_buffers, buffer, vabh_list);
1629 _voucher_activity_t actx = _voucher_activity_try_insert(act);
1630 if (actx) {
1631 _voucher_activity_dispose(act);
1632 act = actx;
1633 }
1634 _dispatch_voucher_activity_debug("create", act);
1635 return act;
1636 }
1637
1638 static void
1639 _voucher_activity_dispose(_voucher_activity_t act)
1640 {
1641 _dispatch_voucher_activity_debug("dispose", act);
1642 _voucher_atm_release(act->va_atm);
1643 if (slowpath(_TAILQ_IS_ENQUEUED(act, va_list))) {
1644 _dispatch_voucher_activity_debug("corruption", act);
1645 DISPATCH_CRASH("Activity corruption");
1646 }
1647 act->va_list.tqe_next = DISPATCH_OBJECT_LISTLESS;
1648 dispatch_assert(!_TAILQ_IS_ENQUEUED(act, va_atm_list));
1649 dispatch_assert(!_TAILQ_IS_ENQUEUED(act, va_atm_used_list));
1650 _voucher_activity_buffer_header_t buffer, tmp;
1651 TAILQ_FOREACH_SAFE(buffer, &act->va_buffers, vabh_list, tmp) {
1652 if (buffer->vabh_pos.vabp_pos.vabp_next_tracepoint_idx > 1) {
1653 dispatch_assert(_voucher_activity_buffer_mark_full(buffer));
1654 _voucher_activity_firehose_push(act, buffer);
1655 }
1656 TAILQ_REMOVE(&act->va_buffers, buffer, vabh_list);
1657 _dispatch_voucher_activity_debug("buffer free %p", act, buffer);
1658 _voucher_activity_heap_buffer_free(buffer);
1659 }
1660 (void)dispatch_assume_zero(pthread_mutex_destroy(&act->va_mutex));
1661 (void)dispatch_assume_zero(pthread_cond_destroy(&act->va_cond));
1662 free(act);
1663 }
1664
1665 DISPATCH_NOINLINE
1666 void
1667 _voucher_activity_firehose_push(_voucher_activity_t act,
1668 _voucher_activity_buffer_header_t buffer)
1669 {
1670 if (dispatch_assume_zero(pthread_mutex_lock(&act->va_mutex))) {
1671 DISPATCH_CLIENT_CRASH("Activity corruption: mutex_lock");
1672 }
1673 _dispatch_voucher_activity_debug("firehose push %p", act, buffer);
1674 // TODO: call firehose_push
1675 VOUCHER_ACTIVITY_BUFFER_HOOK_CALLOUT(full, buffer);
1676 _voucher_activity_buffer_init(act, buffer, false);
1677 if (dispatch_assume_zero(pthread_cond_broadcast(&act->va_cond))) {
1678 DISPATCH_CLIENT_CRASH("Activity corruption: cond_broadcast");
1679 }
1680 if (dispatch_assume_zero(pthread_mutex_unlock(&act->va_mutex))) {
1681 DISPATCH_CLIENT_CRASH("Activity corruption: mutex_unlock");
1682 }
1683 }
1684
1685 DISPATCH_NOINLINE
1686 static void
1687 _voucher_activity_firehose_wait(_voucher_activity_t act,
1688 _voucher_activity_buffer_header_t buffer)
1689 {
1690 if (dispatch_assume_zero(pthread_mutex_lock(&act->va_mutex))) {
1691 DISPATCH_CLIENT_CRASH("Activity corruption: mutex_lock");
1692 }
1693 while (_voucher_activity_buffer_is_full(buffer)) {
1694 _dispatch_voucher_activity_debug("firehose wait %p", act, buffer);
1695 if (dispatch_assume_zero(pthread_cond_wait(&act->va_cond,
1696 &act->va_mutex))){
1697 DISPATCH_CLIENT_CRASH("Activity corruption: cond_wait");
1698 }
1699 }
1700 if (dispatch_assume_zero(pthread_mutex_unlock(&act->va_mutex))) {
1701 DISPATCH_CLIENT_CRASH("Activity corruption: mutex_unlock");
1702 }
1703 }
1704
1705 static _voucher_activity_t
1706 _voucher_activity_copy_from_mach_voucher(mach_voucher_t kv,
1707 voucher_activity_id_t va_id)
1708 {
1709 dispatch_once_f(&_voucher_activity_heap_pred, NULL,
1710 _voucher_activity_heap_init);
1711 if (_voucher_activity_disabled()) return NULL;
1712 _voucher_activity_t act = NULL;
1713 if (dispatch_assume(va_id)) {
1714 if ((act = _voucher_activity_copy_from_id(va_id))) return act;
1715 }
1716 atm_aid_t atm_id = _voucher_mach_voucher_get_atm_id(kv);
1717 if (!dispatch_assume(atm_id)) return NULL;
1718 _voucher_activity_buffer_header_t buffer;
1719 buffer = _voucher_activity_heap_buffer_alloc();
1720 if (!buffer) return NULL;
1721 _dispatch_kvoucher_debug("atm copy/create from <%lld>", kv, atm_id);
1722 _voucher_atm_t vatm = _voucher_atm_copy(atm_id);
1723 if (!vatm) vatm = _voucher_atm_create(kv, atm_id);
1724 if (!vatm) {
1725 _voucher_activity_heap_buffer_free(buffer);
1726 return NULL;
1727 }
1728 // consumes vatm reference:
1729 act = _voucher_activity_create_with_atm(vatm, va_id, 0, 0, buffer);
1730 _dispatch_voucher_activity_debug("copy from kvoucher[0x%08x]", act, kv);
1731 return act;
1732 }
1733
1734 #pragma mark -
1735 #pragma mark _voucher_atm_t
1736
1737 static void _voucher_atm_remove(_voucher_atm_t vatm);
1738 static void _voucher_atm_dispose(_voucher_atm_t vatm, bool unregister);
1739
1740 DISPATCH_ALWAYS_INLINE
1741 static inline bool
1742 _voucher_atm_try_retain(_voucher_atm_t vatm)
1743 {
1744 // not using _os_object_refcnt* because we don't need barriers:
1745 // vouchers atm are immutable and are in a hash table with a lock
1746 //
1747 // assumes vam_atms_lock held
1748 int refcnt = dispatch_atomic_inc2o(vatm, vatm_refcnt, relaxed);
1749 _dispatch_voucher_atm_debug("retain -> %d", vatm, refcnt + 1);
1750 if (slowpath(refcnt < 0)) {
1751 _dispatch_voucher_atm_debug("overrelease", vatm);
1752 DISPATCH_CRASH("ATM overrelease");
1753 }
1754 return refcnt > 0;
1755 }
1756
1757 DISPATCH_ALWAYS_INLINE
1758 static inline _voucher_atm_t
1759 _voucher_atm_retain(_voucher_atm_t vatm)
1760 {
1761 if (slowpath(!_voucher_atm_try_retain(vatm))) {
1762 _dispatch_voucher_atm_debug("resurrection", vatm);
1763 DISPATCH_CRASH("ATM resurrection");
1764 }
1765 return vatm;
1766 }
1767
1768 DISPATCH_ALWAYS_INLINE
1769 static inline void
1770 _voucher_atm_release(_voucher_atm_t vatm)
1771 {
1772 // not using _os_object_refcnt* because we don't need barriers:
1773 // vouchers atm are immutable are into a hash table with a lock
1774 int refcnt = dispatch_atomic_dec2o(vatm, vatm_refcnt, relaxed);
1775 _dispatch_voucher_atm_debug("release -> %d", vatm, refcnt + 1);
1776 if (fastpath(refcnt >= 0)) {
1777 return;
1778 }
1779 if (slowpath(refcnt < -1)) {
1780 _dispatch_voucher_atm_debug("overrelease", vatm);
1781 DISPATCH_CRASH("ATM overrelease");
1782 }
1783 _voucher_atm_remove(vatm);
1784 _voucher_atm_dispose(vatm, true);
1785 }
1786
1787 static _voucher_atm_t
1788 _voucher_atm_find_and_retain(atm_aid_t atm_id, uint32_t hash)
1789 {
1790 // not using _os_object_refcnt* because we don't need barriers:
1791 // vouchers atm are immutable are into a hash table with a lock
1792 //
1793 // assumes vam_atms_lock held
1794 _voucher_atm_t vatm;
1795 TAILQ_FOREACH(vatm, vam_atms(hash), vatm_list){
1796 if (vatm->vatm_id == atm_id) {
1797 if (fastpath(_voucher_atm_try_retain(vatm))) {
1798 return vatm;
1799 }
1800
1801 // <rdar://problem/20468375> disallow resurrection
1802 dispatch_atomic_dec2o(vatm, vatm_refcnt, relaxed);
1803 _dispatch_voucher_atm_debug("undo resurrection", vatm);
1804 }
1805 }
1806 return NULL;
1807 }
1808
1809 static _voucher_atm_t
1810 _voucher_atm_copy(atm_aid_t atm_id)
1811 {
1812 uint32_t hash = VATMID_HASH(atm_id);
1813 _voucher_activity_lock_lock(vam_atms_lock());
1814 _voucher_atm_t vatm = _voucher_atm_find_and_retain(atm_id, hash);
1815 if (vatm) {
1816 _dispatch_voucher_atm_debug("copy", vatm);
1817 }
1818 _voucher_activity_lock_unlock(vam_atms_lock());
1819 return vatm;
1820 }
1821
1822 static _voucher_atm_t
1823 _voucher_atm_try_insert(_voucher_atm_t vatm_new)
1824 {
1825 atm_aid_t atm_id = vatm_new->vatm_id;
1826 uint32_t hash = VATMID_HASH(atm_id);
1827 _voucher_activity_lock_lock(vam_atms_lock());
1828 _voucher_atm_t vatm = _voucher_atm_find_and_retain(atm_id, hash);
1829 if (vatm) {
1830 _dispatch_voucher_atm_debug("try insert: failed (%p)", vatm, vatm_new);
1831 } else {
1832 if (slowpath(_TAILQ_IS_ENQUEUED(vatm_new, vatm_list))) {
1833 _dispatch_voucher_atm_debug("corruption", vatm_new);
1834 DISPATCH_CRASH("ATM corruption");
1835 }
1836 TAILQ_INSERT_TAIL(vam_atms(hash), vatm_new, vatm_list);
1837 _dispatch_voucher_atm_debug("try insert: succeeded", vatm_new);
1838 }
1839 _voucher_activity_lock_unlock(vam_atms_lock());
1840 return vatm;
1841 }
1842
1843 static void
1844 _voucher_atm_remove(_voucher_atm_t vatm)
1845 {
1846 atm_aid_t atm_id = vatm->vatm_id;
1847 uint32_t hash = VATMID_HASH(atm_id);
1848
1849 _voucher_activity_lock_lock(vam_atms_lock());
1850 if (slowpath(!atm_id || !_TAILQ_IS_ENQUEUED(vatm, vatm_list))) {
1851 _dispatch_voucher_atm_debug("corruption", vatm);
1852 DISPATCH_CRASH("ATM corruption");
1853 }
1854 TAILQ_REMOVE(vam_atms(hash), vatm, vatm_list);
1855 _TAILQ_MARK_NOT_ENQUEUED(vatm, vatm_list);
1856 vatm->vatm_list.tqe_next = (void*)~0ull;
1857 _dispatch_voucher_atm_debug("remove", vatm);
1858 _voucher_activity_lock_unlock(vam_atms_lock());
1859 }
1860
1861 DISPATCH_NOINLINE
1862 static void
1863 _voucher_atm_fault(mach_voucher_attr_command_t kvc_cmd)
1864 {
1865 mach_voucher_t kv = _voucher_get_atm_mach_voucher(_voucher_get());
1866 if (!kv) return;
1867
1868 mach_atm_subaid_t subaid = 0;
1869 voucher_t v = _voucher_get();
1870 if (v) {
1871 unsigned int activities = v->v_activities;
1872 voucher_activity_id_t *activity_ids = _voucher_activity_ids(v);
1873 if (activities) {
1874 subaid = activity_ids[0];
1875 }
1876 }
1877
1878 kern_return_t kr;
1879 mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&subaid;
1880 mach_voucher_attr_content_size_t kvc_in_size = sizeof(mach_atm_subaid_t);
1881 mach_voucher_attr_content_t kvc_out = (mach_voucher_attr_content_t)&subaid;
1882 mach_voucher_attr_content_size_t kvc_out_size = sizeof(mach_atm_subaid_t);
1883 kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM,
1884 kvc_cmd, kvc_in, kvc_in_size, kvc_out, &kvc_out_size);
1885 DISPATCH_VERIFY_MIG(kr);
1886 (void)dispatch_assume_zero(kr);
1887 }
1888
1889 static atm_aid_t
1890 _voucher_mach_voucher_get_atm_id(mach_voucher_t kv)
1891 {
1892 kern_return_t kr;
1893 atm_aid_t atm_id = 0;
1894 mach_voucher_attr_content_t kvc = (mach_voucher_attr_content_t)&atm_id;
1895 mach_voucher_attr_content_size_t kvc_size = sizeof(atm_id);
1896 kr = mach_voucher_extract_attr_content(kv, MACH_VOUCHER_ATTR_KEY_ATM, kvc,
1897 &kvc_size);
1898 DISPATCH_VERIFY_MIG(kr);
1899 (void)dispatch_assume_zero(kr);
1900 return atm_id;
1901 }
1902
1903 static mach_voucher_t
1904 _voucher_atm_mach_voucher_create(atm_aid_t *atm_id_ptr)
1905 {
1906 kern_return_t kr;
1907 mach_voucher_t kv;
1908 static const mach_voucher_attr_recipe_data_t atm_create_recipe = {
1909 .key = MACH_VOUCHER_ATTR_KEY_ATM,
1910 .command = MACH_VOUCHER_ATTR_ATM_CREATE,
1911 };
1912 kr = _voucher_create_mach_voucher(&atm_create_recipe,
1913 sizeof(atm_create_recipe), &kv);
1914 if (dispatch_assume_zero(kr)) {
1915 DISPATCH_CLIENT_CRASH("Could not create ATM mach voucher");
1916 }
1917 atm_aid_t atm_id = _voucher_mach_voucher_get_atm_id(kv);
1918 if (!dispatch_assume(atm_id)) {
1919 DISPATCH_CLIENT_CRASH("Could not extract ATM ID");
1920 }
1921 _dispatch_kvoucher_debug("atm create <%lld>", kv, atm_id);
1922 *atm_id_ptr = atm_id;
1923 return kv;
1924 }
1925
1926 static mach_voucher_t
1927 _voucher_atm_mach_voucher_copy(mach_voucher_t akv)
1928 {
1929 kern_return_t kr;
1930 mach_voucher_t kv;
1931 const mach_voucher_attr_recipe_data_t atm_copy_recipe = {
1932 .key = MACH_VOUCHER_ATTR_KEY_ATM,
1933 .command = MACH_VOUCHER_ATTR_COPY,
1934 .previous_voucher = akv,
1935 };
1936 kr = _voucher_create_mach_voucher(&atm_copy_recipe,
1937 sizeof(atm_copy_recipe), &kv);
1938 if (dispatch_assume_zero(kr)) {
1939 DISPATCH_CLIENT_CRASH("Could not copy ATM mach voucher");
1940 }
1941 _dispatch_kvoucher_debug("copy atm voucher from [0x%08x]", kv, akv);
1942 return kv;
1943 }
1944
1945 static void
1946 _voucher_atm_register(_voucher_atm_t vatm)
1947 {
1948 mach_voucher_t kv = vatm->vatm_kvoucher;
1949 if (!kv) return;
1950 kern_return_t kr;
1951 atm_guard_t gen =
1952 dispatch_atomic_inc(&_voucher_atm_generation, relaxed);
1953 _dispatch_voucher_atm_debug("atm register %lld", vatm, gen);
1954 mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&gen;
1955 mach_voucher_attr_content_size_t kvc_in_size = sizeof(gen);
1956 mach_voucher_attr_content_t kvc_out = NULL;
1957 mach_voucher_attr_content_size_t kvc_out_size = 0;
1958 kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM,
1959 ATM_ACTION_REGISTER, kvc_in, kvc_in_size, kvc_out,
1960 &kvc_out_size);
1961 DISPATCH_VERIFY_MIG(kr);
1962 if (kr) {
1963 DISPATCH_CLIENT_CRASH("Could not register ATM ID");
1964 }
1965 vatm->vatm_generation = gen;
1966 _dispatch_voucher_atm_debug("atm registered %lld", vatm,
1967 vatm->vatm_generation);
1968 }
1969
1970 static void
1971 _voucher_atm_unregister(_voucher_atm_t vatm)
1972 {
1973 _dispatch_voucher_atm_debug("atm unregister %lld", vatm,
1974 vatm->vatm_generation);
1975 mach_voucher_t kv = vatm->vatm_kvoucher;
1976 dispatch_assert(kv);
1977 kern_return_t kr;
1978 atm_guard_t gen = vatm->vatm_generation;
1979 mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&gen;
1980 mach_voucher_attr_content_size_t kvc_in_size = sizeof(gen);
1981 mach_voucher_attr_content_t kvc_out = NULL;
1982 mach_voucher_attr_content_size_t kvc_out_size = 0;
1983 kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM,
1984 ATM_ACTION_UNREGISTER, kvc_in, kvc_in_size, kvc_out, &kvc_out_size);
1985 DISPATCH_VERIFY_MIG(kr);
1986 if (kr && kr != KERN_INVALID_VALUE) {
1987 (void)dispatch_assume_zero(kr);
1988 }
1989 _dispatch_voucher_atm_debug("atm unregistered %lld", vatm,
1990 vatm->vatm_generation);
1991 }
1992
1993 static _voucher_atm_t
1994 _voucher_atm_create(mach_voucher_t kv, atm_aid_t atm_id)
1995 {
1996 _voucher_atm_t vatm = _dispatch_calloc(1ul, sizeof(struct _voucher_atm_s));
1997 kv = kv ? _voucher_atm_mach_voucher_copy(kv) :
1998 _voucher_atm_mach_voucher_create(&atm_id);
1999 vatm->vatm_kvoucher = kv;
2000 vatm->vatm_id = atm_id;
2001 _voucher_atm_t vatmx = _voucher_atm_try_insert(vatm);
2002 if (vatmx) {
2003 _voucher_atm_dispose(vatm, false);
2004 vatm = vatmx;
2005 } else {
2006 _voucher_atm_register(vatm);
2007 }
2008 _dispatch_voucher_atm_debug("create with kvoucher[0x%08x]", vatm, kv);
2009 return vatm;
2010 }
2011
2012 static void
2013 _voucher_atm_dispose(_voucher_atm_t vatm, bool unregister)
2014 {
2015 _dispatch_voucher_atm_debug("dispose", vatm);
2016 if (slowpath(_TAILQ_IS_ENQUEUED(vatm, vatm_list))) {
2017 _dispatch_voucher_atm_debug("corruption", vatm);
2018 DISPATCH_CRASH("ATM corruption");
2019 }
2020 vatm->vatm_list.tqe_next = DISPATCH_OBJECT_LISTLESS;
2021 if (vatm->vatm_kvoucher) {
2022 if (unregister) _voucher_atm_unregister(vatm);
2023 _voucher_dealloc_mach_voucher(vatm->vatm_kvoucher);
2024 vatm->vatm_kvoucher = MACH_VOUCHER_NULL;
2025 }
2026 free(vatm);
2027 }
2028
2029 DISPATCH_NOINLINE
2030 static voucher_activity_id_t
2031 _voucher_atm_subid_make(_voucher_atm_t vatm, voucher_activity_flag_t flags)
2032 {
2033 mach_voucher_t kv = vatm->vatm_kvoucher;
2034 _dispatch_voucher_atm_debug("create subid from atm", vatm);
2035 kern_return_t kr;
2036 mach_atm_subaid_t naid;
2037 mach_voucher_attr_content_t kvc_in = NULL;
2038 mach_voucher_attr_content_size_t kvc_in_size = 0;
2039 mach_voucher_attr_content_t kvc_out = (mach_voucher_attr_content_t)&naid;
2040 mach_voucher_attr_content_size_t kvc_out_size = sizeof(naid);
2041 kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM,
2042 ATM_ACTION_GETSUBAID, kvc_in, kvc_in_size, kvc_out, &kvc_out_size);
2043 DISPATCH_VERIFY_MIG(kr);
2044 if (dispatch_assume_zero(kr)) {
2045 DISPATCH_CLIENT_CRASH("Could not get next ATM ID");
2046 }
2047 _dispatch_voucher_atm_debug("created subid from atm %lld", vatm, naid);
2048 return VATMID2ACTID(naid, flags);
2049 }
2050
2051 #pragma mark -
2052 #pragma mark voucher_activity_id_t
2053
2054 static const size_t _voucher_activity_maxsize =
2055 _voucher_activity_buffer_size - _voucher_activity_buffer_header_size -
2056 _voucher_activity_strings_header_size;
2057
2058 voucher_activity_id_t
2059 voucher_activity_start_with_location(voucher_activity_trace_id_t trace_id,
2060 uint64_t location, voucher_activity_flag_t flags)
2061 {
2062 dispatch_once_f(&_voucher_activity_heap_pred, NULL,
2063 _voucher_activity_heap_init);
2064 if (!_voucher_activity_trace_id_enabled(trace_id)) return 0;
2065 voucher_activity_id_t va_id = 0;
2066 _voucher_atm_t vatm = NULL;
2067 _voucher_activity_t act = NULL;
2068 _voucher_activity_tracepoint_t vat = NULL;
2069 unsigned int activities = 1, oactivities = 0;
2070 voucher_t ov = _voucher_get();
2071 vatm = _voucher_get_atm(ov);
2072 if (!(flags & voucher_activity_flag_force) && ov && ov->v_activities) {
2073 oactivities = ov->v_activities;
2074 activities += oactivities;
2075 if (activities > _voucher_max_activities) {
2076 va_id = _voucher_atm_subid_make(vatm, flags);
2077 goto out;
2078 }
2079 }
2080 va_id = _voucher_atm_subid_make(vatm, flags);
2081 if (activities == 1) {
2082 // consumes vatm reference:
2083 act = _voucher_activity_create_with_atm(_voucher_atm_retain(vatm),
2084 va_id, trace_id, location, NULL);
2085 vat = (_voucher_activity_tracepoint_t)act;
2086 } else if (ov && ov->v_activity) {
2087 act = _voucher_activity_retain(ov->v_activity);
2088 }
2089 pthread_priority_t priority = _voucher_get_priority(ov);
2090 mach_voucher_attr_recipe_size_t extra = ov ? _voucher_extra_size(ov) : 0;
2091 voucher_t v = _voucher_alloc(activities, priority, extra);
2092 if (extra) {
2093 memcpy(_voucher_extra_recipes(v), _voucher_extra_recipes(ov), extra);
2094 }
2095 if (ov && ov->v_kvoucher) {
2096 voucher_t kvb = ov->v_kvbase ? ov->v_kvbase : ov;
2097 v->v_kvbase = _voucher_retain(kvb);
2098 v->v_kvoucher = kvb->v_kvoucher;
2099 }
2100 voucher_activity_id_t *activity_ids = _voucher_activity_ids(v);
2101 if (oactivities) {
2102 memcpy(activity_ids, _voucher_activity_ids(ov),
2103 oactivities * sizeof(voucher_activity_id_t));
2104 }
2105 activity_ids[activities-1] = va_id;
2106 v->v_atm = _voucher_atm_retain(vatm);
2107 v->v_activity = act;
2108 _voucher_swap(ov, v);
2109 if (vat) return va_id; // new activity buffer contains trace info
2110 out:
2111 _voucher_activity_trace_activity_event(trace_id, va_id, start);
2112 return va_id;
2113 }
2114
2115 voucher_activity_id_t
2116 voucher_activity_start(voucher_activity_trace_id_t trace_id,
2117 voucher_activity_flag_t flags)
2118 {
2119 return voucher_activity_start_with_location(trace_id, 0, flags);
2120 }
2121
2122 void
2123 voucher_activity_end(voucher_activity_id_t va_id)
2124 {
2125 if (!va_id) return;
2126 _voucher_activity_trace_activity_event(_voucher_activity_trace_id_release,
2127 va_id, end);
2128 voucher_t v = _voucher_get();
2129 if (!v) return;
2130 unsigned int activities = v->v_activities, act_idx = activities;
2131 voucher_activity_id_t *activity_ids = _voucher_activity_ids(v);
2132 while (act_idx) {
2133 if (activity_ids[act_idx-1] == va_id) break;
2134 act_idx--;
2135 }
2136 if (!act_idx) return; // activity_id not found
2137 pthread_priority_t priority = _voucher_get_priority(v);
2138 mach_voucher_attr_recipe_size_t extra = _voucher_extra_size(v);
2139 voucher_t nv = NULL;
2140 if (act_idx > 1 || activities == 1) --activities;
2141 if (priority || activities || extra || v->v_kvoucher) {
2142 nv = _voucher_alloc(activities, priority, extra);
2143 if (extra) {
2144 memcpy(_voucher_extra_recipes(nv), _voucher_extra_recipes(v),extra);
2145 }
2146 }
2147 if (v->v_kvoucher) {
2148 voucher_t kvb = v->v_kvbase ? v->v_kvbase : v;
2149 nv->v_kvbase = _voucher_retain(kvb);
2150 nv->v_kvoucher = kvb->v_kvoucher;
2151 }
2152 bool atm_collect = !activities;
2153 if (activities) {
2154 voucher_activity_id_t *new_activity_ids = _voucher_activity_ids(nv);
2155 if (act_idx == 1 && _voucher_activity_default) {
2156 atm_collect = true;
2157 // default to _voucher_activity_default base activity
2158 new_activity_ids[0] = _voucher_activity_default->va_id;
2159 memcpy(&new_activity_ids[1], &activity_ids[1],
2160 (activities - 1) * sizeof(voucher_activity_id_t));
2161 } else {
2162 if (v->v_activity) {
2163 nv->v_activity = _voucher_activity_retain(v->v_activity);
2164 nv->v_atm = _voucher_atm_retain(v->v_atm);
2165 }
2166 memcpy(new_activity_ids, activity_ids,
2167 --act_idx * sizeof(voucher_activity_id_t));
2168 if (act_idx < activities) {
2169 memcpy(&new_activity_ids[act_idx], &activity_ids[act_idx+1],
2170 (activities - act_idx) * sizeof(voucher_activity_id_t));
2171 }
2172 }
2173 }
2174 _voucher_swap(v, nv);
2175 }
2176
2177 unsigned int
2178 voucher_get_activities(voucher_activity_id_t *entries, unsigned int *count)
2179 {
2180 voucher_t v = _voucher_get();
2181 if (!v || !count) return 0;
2182 unsigned int activities = v->v_activities;
2183 if (*count < activities) activities = *count;
2184 *count = v->v_activities;
2185 voucher_activity_id_t *activity_ids = _voucher_activity_ids(v);
2186 if (activities && entries) {
2187 memcpy(entries, activity_ids, activities *
2188 sizeof(voucher_activity_id_t));
2189 }
2190 return activities;
2191 }
2192
2193 uint8_t
2194 voucher_activity_get_namespace(void)
2195 {
2196 voucher_t v = _voucher_get();
2197 if (!v || !v->v_activity) return 0;
2198 voucher_activity_trace_id_t trace_id = v->v_activity->va_trace_id;
2199 uint8_t cns = (uint8_t)(trace_id >>
2200 _voucher_activity_trace_id_code_namespace_shift);
2201 return cns;
2202 }
2203
2204 DISPATCH_NOINLINE
2205 _voucher_activity_tracepoint_t
2206 _voucher_activity_buffer_tracepoint_acquire_slow(_voucher_activity_t *vap,
2207 _voucher_activity_buffer_header_t *vabp, unsigned int slots,
2208 size_t strsize, uint16_t *stroffsetp)
2209 {
2210 _voucher_activity_t act;
2211 _voucher_activity_buffer_header_t vab;
2212 _voucher_activity_tracepoint_t vat = NULL;
2213 voucher_t v = _voucher_get();
2214 if (v && v->v_activity) {
2215 act = v->v_activity;
2216 } else {
2217 dispatch_once_f(&_voucher_activity_heap_pred, NULL,
2218 _voucher_activity_heap_init);
2219 if (_voucher_activity_disabled()) return NULL;
2220 act = _voucher_activity_default;
2221 }
2222 vab = act->va_current_buffer;
2223 if (act == *vap && vab != *vabp) {
2224 goto retry; // another slowpath raced us
2225 }
2226 do {
2227 vab = _voucher_activity_buffer_alloc(act, vab);
2228 if (!vab) break;
2229 retry:
2230 vat = _voucher_activity_buffer_tracepoint_acquire(vab, slots, strsize,
2231 stroffsetp);
2232 } while (!vat);
2233 *vap = act;
2234 *vabp = vab;
2235 return vat;
2236 }
2237
2238 static inline void
2239 _voucher_activity_trace_fault(voucher_activity_trace_id_t trace_id)
2240 {
2241 if (!slowpath(_voucher_activity_trace_id_is_subtype(trace_id, error))) {
2242 return;
2243 }
2244 mach_voucher_attr_command_t atm_cmd = ATM_ACTION_COLLECT;
2245 if (_voucher_activity_trace_id_is_subtype(trace_id, fault)) {
2246 atm_cmd = ATM_ACTION_LOGFAIL;
2247 }
2248 return _voucher_atm_fault(atm_cmd);
2249 }
2250
2251 uint64_t
2252 voucher_activity_trace(voucher_activity_trace_id_t trace_id, uint64_t location,
2253 void *buffer, size_t length)
2254 {
2255 if (!_voucher_activity_trace_id_enabled(trace_id)) return 0;
2256 _voucher_activity_t act;
2257 _voucher_activity_buffer_header_t vab;
2258 _voucher_activity_tracepoint_t vat;
2259 const unsigned int slots = length <= sizeof(vat->vat_data) ? 1 : 2;
2260 act = _voucher_activity_get();
2261 vab = _voucher_activity_buffer_get_from_activity(act);
2262 vat = _voucher_activity_buffer_tracepoint_acquire(vab, slots, 0, NULL);
2263 if (!vat) {
2264 vat = _voucher_activity_buffer_tracepoint_acquire_slow(&act, &vab,
2265 slots, 0, NULL);
2266 }
2267 if (!vat) return 0;
2268 uint64_t timestamp = _voucher_activity_tracepoint_init_with_id(vat,
2269 trace_id, location, true);
2270 void *tbuf = vat->vat_data;
2271 size_t tlen = sizeof(vat->vat_data);
2272 if (length < tlen) {
2273 memcpy(tbuf, buffer, length);
2274 } else {
2275 memcpy(tbuf, buffer, tlen);
2276 }
2277 if (length > tlen) {
2278 vat->vat_flags |= _voucher_activity_trace_flag_wide_first;
2279 buffer += tlen;
2280 length -= tlen;
2281 (++vat)->vat_flags = _voucher_activity_trace_flag_tracepoint |
2282 _voucher_activity_trace_flag_wide_second;
2283 vat->vat_type = 0; vat->vat_namespace = 0;
2284 tbuf = (void*)vat + offsetof(typeof(*vat), vat_code);
2285 tlen = sizeof(*vat) - offsetof(typeof(*vat), vat_code);
2286 if (length < tlen) {
2287 memcpy(tbuf, buffer, length);
2288 } else {
2289 memcpy(tbuf, buffer, tlen);
2290 }
2291 }
2292 _voucher_activity_trace_fault(trace_id);
2293 if (_voucher_activity_buffer_tracepoint_release(vab)) {
2294 _voucher_activity_firehose_push(act, vab);
2295 }
2296 return timestamp;
2297 }
2298
2299 uint64_t
2300 voucher_activity_trace_strings(voucher_activity_trace_id_t trace_id,
2301 uint64_t location, void *buffer, size_t length, const char *strings[],
2302 size_t string_lengths[], size_t strings_size)
2303 {
2304 if (!_voucher_activity_trace_id_enabled(trace_id)) return 0;
2305 _voucher_activity_t act;
2306 _voucher_activity_buffer_header_t vab;
2307 _voucher_activity_tracepoint_t vat;
2308 uint16_t offset;
2309 const unsigned int slots = length <= sizeof(vat->vat_data) ? 1 : 2;
2310 strings_size = MIN(strings_size, _voucher_activity_maxsize -
2311 slots * sizeof(struct _voucher_activity_tracepoint_s));
2312 act = _voucher_activity_get();
2313 vab = _voucher_activity_buffer_get_from_activity(act);
2314 vat = _voucher_activity_buffer_tracepoint_acquire(vab, slots, strings_size,
2315 &offset);
2316 if (!vat) {
2317 vat = _voucher_activity_buffer_tracepoint_acquire_slow(&act, &vab,
2318 slots, strings_size, &offset);
2319 }
2320 if (!vat) return 0;
2321 uint64_t timestamp = _voucher_activity_tracepoint_init_with_id(vat,
2322 trace_id, location, false);
2323 vat->vat_flags |= _voucher_activity_trace_flag_tracepoint_strings;
2324 vat->vat_stroff.vats_offset = offset;
2325 void *tbuf = vat->vat_stroff.vats_data;
2326 size_t tlen = sizeof(vat->vat_stroff.vats_data);
2327 if (length < tlen) {
2328 memcpy(tbuf, buffer, length);
2329 } else {
2330 memcpy(tbuf, buffer, tlen);
2331 }
2332 if (length > tlen) {
2333 vat->vat_flags |= _voucher_activity_trace_flag_wide_first;
2334 buffer += tlen;
2335 length -= tlen;
2336 (++vat)->vat_flags = _voucher_activity_trace_flag_tracepoint |
2337 _voucher_activity_trace_flag_wide_second;
2338 vat->vat_type = 0; vat->vat_namespace = 0;
2339 tbuf = (void*)vat + offsetof(typeof(*vat), vat_code);
2340 tlen = sizeof(*vat) - offsetof(typeof(*vat), vat_code);
2341 if (length < tlen) {
2342 memcpy(tbuf, buffer, length);
2343 } else {
2344 memcpy(tbuf, buffer, tlen);
2345 }
2346 }
2347 const uint16_t offsetend = offset - (uint16_t)strings_size;
2348 char *b = (char*)vab + _voucher_activity_buffer_size;
2349 int i = 0;
2350 while (offset > offsetend && strings[i]) {
2351 size_t maxsize = MIN(string_lengths[i] + 1, offset - offsetend);
2352 size_t len = strlcpy(b - offset, strings[i++], maxsize);
2353 offset -= MIN(len + 1, maxsize);
2354 }
2355 _voucher_activity_trace_fault(trace_id);
2356 if (_voucher_activity_buffer_tracepoint_release(vab)) {
2357 _voucher_activity_firehose_push(act, vab);
2358 }
2359 return timestamp;
2360 }
2361
2362 uint64_t
2363 voucher_activity_trace_args(voucher_activity_trace_id_t trace_id,
2364 uint64_t location, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3,
2365 uintptr_t arg4)
2366 {
2367 if (!_voucher_activity_trace_id_enabled(trace_id)) return 0;
2368 _voucher_activity_t act;
2369 _voucher_activity_buffer_header_t vab;
2370 _voucher_activity_tracepoint_t vat;
2371 act = _voucher_activity_get();
2372 vab = _voucher_activity_buffer_get_from_activity(act);
2373 vat = _voucher_activity_buffer_tracepoint_acquire(vab, 1, 0, NULL);
2374 if (!vat) {
2375 vat = _voucher_activity_buffer_tracepoint_acquire_slow(&act, &vab, 1,
2376 0, NULL);
2377 }
2378 if (!vat) return 0;
2379 uint64_t timestamp = _voucher_activity_tracepoint_init_with_id(vat,
2380 trace_id, location, true);
2381 vat->vat_flags |= _voucher_activity_trace_flag_tracepoint_args;
2382 vat->vat_data[0] = arg1;
2383 vat->vat_data[1] = arg2;
2384 vat->vat_data[2] = arg3;
2385 vat->vat_data[3] = arg4;
2386 _voucher_activity_trace_fault(trace_id);
2387 if (_voucher_activity_buffer_tracepoint_release(vab)) {
2388 _voucher_activity_firehose_push(act, vab);
2389 }
2390 return timestamp;
2391 }
2392
2393 #pragma mark -
2394 #pragma mark _voucher_debug
2395
2396 size_t
2397 _voucher_debug(voucher_t v, char* buf, size_t bufsiz)
2398 {
2399 size_t offset = 0;
2400 #define bufprintf(...) \
2401 offset += dsnprintf(&buf[offset], bufsiz - offset, ##__VA_ARGS__)
2402 bufprintf("voucher[%p] = { xrefcnt = 0x%x, refcnt = 0x%x, ", v,
2403 v->os_obj_xref_cnt + 1, v->os_obj_ref_cnt + 1);
2404
2405 if (v->v_kvbase) {
2406 bufprintf("base voucher %p, ", v->v_kvbase);
2407 }
2408 if (v->v_kvoucher) {
2409 bufprintf("kvoucher%s 0x%x, ", v->v_kvoucher == v->v_ipc_kvoucher ?
2410 " & ipc kvoucher" : "", v->v_kvoucher);
2411 }
2412 if (v->v_ipc_kvoucher && v->v_ipc_kvoucher != v->v_kvoucher) {
2413 bufprintf("ipc kvoucher 0x%x, ", v->v_ipc_kvoucher);
2414 }
2415 if (v->v_has_priority) {
2416 bufprintf("QOS 0x%x, ", *_voucher_priority(v));
2417 }
2418 if (v->v_activities) {
2419 voucher_activity_id_t *activity_ids = _voucher_activity_ids(v);
2420 bufprintf("activity IDs = { ");
2421 unsigned int i;
2422 for (i = 0; i < v->v_activities; i++) {
2423 bufprintf("0x%llx, ", *activity_ids++);
2424 }
2425 bufprintf("}, ");
2426 }
2427 if (v->v_activity) {
2428 _voucher_activity_t va = v->v_activity;
2429 _voucher_atm_t vatm = va->va_atm;
2430 bufprintf("activity[%p] = { ID 0x%llx, ref %d, atm[%p] = { "
2431 "AID 0x%llx, ref %d, kvoucher 0x%x } }, ", va, va->va_id,
2432 va->va_refcnt + 1, va->va_atm, vatm->vatm_id,
2433 vatm->vatm_refcnt + 1, vatm->vatm_kvoucher);
2434 }
2435 bufprintf("}");
2436 return offset;
2437 }
2438
2439 #else // VOUCHER_USE_MACH_VOUCHER
2440
2441 #pragma mark -
2442 #pragma mark Simulator / vouchers disabled
2443
2444 #if VOUCHER_ENABLE_RECIPE_OBJECTS
2445 voucher_t
2446 voucher_create(voucher_recipe_t recipe)
2447 {
2448 (void)recipe;
2449 return NULL;
2450 }
2451 #endif
2452
2453 voucher_t
2454 voucher_adopt(voucher_t voucher)
2455 {
2456 return voucher;
2457 }
2458
2459 voucher_t
2460 voucher_copy(void)
2461 {
2462 return NULL;
2463 }
2464
2465 voucher_t
2466 voucher_copy_without_importance(void)
2467 {
2468 return NULL;
2469 }
2470
2471 voucher_t
2472 voucher_retain(voucher_t voucher)
2473 {
2474 return voucher;
2475 }
2476
2477 void
2478 voucher_release(voucher_t voucher)
2479 {
2480 (void)voucher;
2481 }
2482
2483 void
2484 voucher_replace_default_voucher(void)
2485 {
2486 }
2487
2488 void
2489 voucher_decrement_importance_count4CF(voucher_t v)
2490 {
2491 (void)v;
2492 }
2493
2494 void
2495 _voucher_thread_cleanup(void *voucher)
2496 {
2497 (void)voucher;
2498 }
2499
2500 void
2501 _voucher_dealloc_mach_voucher(mach_voucher_t kv)
2502 {
2503 (void)kv;
2504 }
2505
2506 mach_voucher_t
2507 _voucher_get_mach_voucher(voucher_t voucher)
2508 {
2509 (void)voucher;
2510 return MACH_VOUCHER_NULL;
2511 }
2512
2513 mach_voucher_t
2514 _voucher_create_mach_voucher_with_priority(voucher_t voucher,
2515 pthread_priority_t priority)
2516 {
2517 (void)voucher; (void)priority;
2518 return MACH_VOUCHER_NULL;
2519 }
2520
2521 voucher_t
2522 _voucher_create_with_priority_and_mach_voucher(voucher_t voucher,
2523 pthread_priority_t priority, mach_voucher_t kv)
2524 {
2525 (void)voucher; (void)priority; (void)kv;
2526 return NULL;
2527 }
2528
2529 voucher_t
2530 _voucher_create_accounting_voucher(voucher_t voucher)
2531 {
2532 (void)voucher;
2533 return NULL;
2534 }
2535
2536 voucher_t
2537 voucher_create_with_mach_msg(mach_msg_header_t *msg)
2538 {
2539 (void)msg;
2540 return NULL;
2541 }
2542
2543 #if VOUCHER_ENABLE_GET_MACH_VOUCHER
2544 mach_voucher_t
2545 voucher_get_mach_voucher(voucher_t voucher)
2546 {
2547 (void)voucher;
2548 return 0;
2549 }
2550 #endif
2551
2552 void
2553 _voucher_xref_dispose(voucher_t voucher)
2554 {
2555 (void)voucher;
2556 }
2557
2558 void
2559 _voucher_dispose(voucher_t voucher)
2560 {
2561 (void)voucher;
2562 }
2563
2564 #if VOUCHER_EXPORT_PERSONA_SPI
2565 uid_t
2566 voucher_get_current_persona(void)
2567 {
2568 return PERSONA_ID_NONE;
2569 }
2570
2571 int
2572 voucher_get_current_persona_originator_info(struct proc_persona_info *persona_info)
2573 {
2574 (void)persona_info;
2575 return -1;
2576 }
2577
2578 int
2579 voucher_get_current_persona_proximate_info(struct proc_persona_info *persona_info)
2580 {
2581 (void)persona_info;
2582 return -1;
2583 }
2584 #endif
2585
2586 void
2587 _voucher_atfork_child(void)
2588 {
2589 }
2590
2591 void
2592 _voucher_init(void)
2593 {
2594 }
2595
2596 void*
2597 voucher_activity_get_metadata_buffer(size_t *length)
2598 {
2599 *length = 0;
2600 return NULL;
2601 }
2602
2603 void
2604 voucher_activity_buffer_hook_install_4libtrace(
2605 _voucher_activity_buffer_hook_t hook)
2606 {
2607 (void)hook;
2608 }
2609
2610 void
2611 _voucher_activity_heap_pressure_normal(void)
2612 {
2613 }
2614
2615 void
2616 _voucher_activity_heap_pressure_warn(void)
2617 {
2618 }
2619
2620 voucher_activity_id_t
2621 voucher_activity_start_with_location(voucher_activity_trace_id_t trace_id,
2622 uint64_t location, voucher_activity_flag_t flags)
2623 {
2624 (void)trace_id; (void)location; (void)flags;
2625 return 0;
2626 }
2627
2628 voucher_activity_id_t
2629 voucher_activity_start(voucher_activity_trace_id_t trace_id,
2630 voucher_activity_flag_t flags)
2631 {
2632 (void)trace_id; (void)flags;
2633 return 0;
2634 }
2635
2636 void
2637 voucher_activity_end(voucher_activity_id_t activity_id)
2638 {
2639 (void)activity_id;
2640 }
2641
2642 unsigned int
2643 voucher_get_activities(voucher_activity_id_t *entries, unsigned int *count)
2644 {
2645 (void)entries; (void)count;
2646 return 0;
2647 }
2648
2649 uint8_t
2650 voucher_activity_get_namespace(void)
2651 {
2652 return 0;
2653 }
2654
2655 uint64_t
2656 voucher_activity_trace(voucher_activity_trace_id_t trace_id, uint64_t location,
2657 void *buffer, size_t length)
2658 {
2659 (void)trace_id; (void)location; (void)buffer; (void)length;
2660 return 0;
2661 }
2662
2663 uint64_t
2664 voucher_activity_trace_strings(voucher_activity_trace_id_t trace_id,
2665 uint64_t location, void *buffer, size_t length, const char *strings[],
2666 size_t string_lengths[], size_t strings_size)
2667 {
2668 (void)trace_id; (void)location; (void)buffer; (void)length; (void)strings;
2669 (void)string_lengths; (void)strings_size;
2670 return 0;
2671 }
2672
2673 uint64_t
2674 voucher_activity_trace_args(voucher_activity_trace_id_t trace_id,
2675 uint64_t location, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3,
2676 uintptr_t arg4)
2677 {
2678 (void)trace_id; (void)location;
2679 (void)arg1; (void)arg2; (void)arg3; (void)arg4;
2680 return 0;
2681 }
2682
2683 size_t
2684 _voucher_debug(voucher_t v, char* buf, size_t bufsiz)
2685 {
2686 (void)v; (void)buf; (void)bufsiz;
2687 return 0;
2688 }
2689
2690 #endif // VOUCHER_USE_MACH_VOUCHER
mirror of libdispatch