]> git.saurik.com Git - apple/libdispatch.git/blob - src/voucher.c
projects / apple / libdispatch.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
libdispatch-500.10.1.tar.gz
[apple/libdispatch.git] / src / voucher.c
1 /*
2 * Copyright (c) 2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_APACHE_LICENSE_HEADER_START@
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * @APPLE_APACHE_LICENSE_HEADER_END@
19 */
20
21 #include "internal.h"
22
23 #if VOUCHER_USE_MACH_VOUCHER
24
25 #include <mach/mach_voucher.h>
26
27 // <rdar://16363550>
28 #ifndef VM_MEMORY_GENEALOGY
29 #define VM_MEMORY_GENEALOGY 78
30 #endif
31
32 #ifndef VOUCHER_ATM_COLLECT_THRESHOLD
33 #define VOUCHER_ATM_COLLECT_THRESHOLD 1
34 #endif
35 #define VATM_COLLECT_THRESHOLD_VALUE(t) (((t) - 1) * 2)
36 static uint64_t volatile _voucher_atm_generation;
37
38 typedef struct _voucher_atm_s *_voucher_atm_t;
39
40 static void _voucher_activity_atfork_child(void);
41 static _voucher_activity_t _voucher_activity_copy_from_mach_voucher(
42 mach_voucher_t kv, voucher_activity_id_t va_id);
43 static inline _voucher_activity_t _voucher_activity_retain(
44 _voucher_activity_t act);
45 static inline void _voucher_activity_release(_voucher_activity_t act);
46 static void _voucher_activity_remove(_voucher_activity_t act);
47 static inline _voucher_atm_t _voucher_atm_retain(_voucher_atm_t vatm);
48 static inline void _voucher_atm_release(_voucher_atm_t vatm);
49
50 #pragma mark -
51 #pragma mark voucher_t
52
53 #if USE_OBJC
54 OS_OBJECT_OBJC_CLASS_DECL(voucher);
55 #define VOUCHER_CLASS OS_OBJECT_OBJC_CLASS(voucher)
56 #else
57 const _os_object_class_s _voucher_class = {
58 ._os_obj_xref_dispose = (void(*)(_os_object_t))_voucher_xref_dispose,
59 ._os_obj_dispose = (void(*)(_os_object_t))_voucher_dispose,
60 };
61 #define VOUCHER_CLASS &_voucher_class
62 #endif // USE_OBJC
63
64 static const voucher_activity_trace_id_t _voucher_activity_trace_id_release =
65 (voucher_activity_trace_id_t)voucher_activity_tracepoint_type_release <<
66 _voucher_activity_trace_id_type_shift;
67 static const unsigned int _voucher_max_activities = 16;
68
69 DISPATCH_ALWAYS_INLINE
70 static inline void
71 _voucher_recipes_init(mach_voucher_attr_recipe_data_t *recipes,
72 mach_voucher_attr_content_size_t bits_size)
73 {
74 static const mach_voucher_attr_recipe_data_t base_recipe = {
75 .key = MACH_VOUCHER_ATTR_KEY_ALL,
76 .command = MACH_VOUCHER_ATTR_COPY,
77 };
78 _voucher_recipes_base(recipes) = base_recipe;
79 static const mach_voucher_attr_recipe_data_t atm_recipe = {
80 .key = MACH_VOUCHER_ATTR_KEY_ATM,
81 .command = MACH_VOUCHER_ATTR_COPY,
82 };
83 _voucher_recipes_atm(recipes) = atm_recipe;
84 static const mach_voucher_attr_recipe_data_t bits_recipe = {
85 .key = MACH_VOUCHER_ATTR_KEY_USER_DATA,
86 .command = MACH_VOUCHER_ATTR_USER_DATA_STORE,
87 };
88 _voucher_recipes_bits(recipes) = bits_recipe;
89 if (!bits_size) return;
90 _voucher_recipes_bits(recipes).content_size = bits_size;
91 *_voucher_recipes_magic(recipes) = _voucher_magic_v1;
92 }
93
94 static inline voucher_t
95 _voucher_alloc(unsigned int activities, pthread_priority_t priority,
96 mach_voucher_attr_recipe_size_t extra)
97 {
98 if (activities > _voucher_max_activities) {
99 activities = _voucher_max_activities;
100 }
101 voucher_t voucher;
102 size_t voucher_size, recipes_size;
103 mach_voucher_attr_content_size_t bits_size;
104 recipes_size = (priority||activities||extra) ? _voucher_recipes_size() : 0;
105 bits_size = recipes_size ? _voucher_bits_size(activities) : 0;
106 voucher_size = sizeof(voucher_s) + recipes_size + bits_size + extra;
107 voucher = (voucher_t)_os_object_alloc_realized(VOUCHER_CLASS, voucher_size);
108 #if VOUCHER_ENABLE_RECIPE_OBJECTS
109 voucher->v_recipe_extra_size = extra;
110 voucher->v_recipe_extra_offset = voucher_size - extra;
111 #else
112 dispatch_assert(!extra);
113 #endif
114 voucher->v_has_priority = priority ? 1 : 0;
115 voucher->v_activities = activities;
116 if (!recipes_size) return voucher;
117 _voucher_recipes_init(voucher->v_recipes, bits_size);
118 *_voucher_priority(voucher) = (_voucher_priority_t)priority;
119 _dispatch_voucher_debug("alloc", voucher);
120 return voucher;
121 }
122
123 #if VOUCHER_ENABLE_RECIPE_OBJECTS
124 voucher_t
125 voucher_create(voucher_recipe_t recipe)
126 {
127 // TODO: capture current activities or current kvoucher ?
128 mach_voucher_attr_recipe_size_t extra = recipe ? recipe->vr_size : 0;
129 voucher_t voucher = _voucher_alloc(0, 0, extra);
130 if (extra) {
131 memcpy(_voucher_extra_recipes(voucher), recipe->vr_data, extra);
132 }
133 return voucher;
134 }
135 #endif
136
137 voucher_t
138 voucher_adopt(voucher_t voucher)
139 {
140 return _voucher_adopt(voucher);
141 }
142
143 voucher_t
144 voucher_copy(void)
145 {
146 return _voucher_copy();
147 }
148
149 voucher_t
150 voucher_copy_without_importance(void)
151 {
152 return _voucher_copy_without_importance();
153 }
154
155 voucher_t
156 voucher_retain(voucher_t voucher)
157 {
158 return _voucher_retain(voucher);
159 }
160
161 void
162 voucher_release(voucher_t voucher)
163 {
164 return _voucher_release(voucher);
165 }
166
167 void
168 _voucher_thread_cleanup(void *voucher)
169 {
170 _voucher_swap(voucher, NULL);
171 }
172
173 DISPATCH_CACHELINE_ALIGN
174 static TAILQ_HEAD(, voucher_s) _vouchers[VL_HASH_SIZE];
175 #define _vouchers_head(kv) (&_vouchers[VL_HASH((kv))])
176 static os_lock_handoff_s _vouchers_lock = OS_LOCK_HANDOFF_INIT;
177 #define _vouchers_lock_lock() os_lock_lock(&_vouchers_lock)
178 #define _vouchers_lock_unlock() os_lock_unlock(&_vouchers_lock)
179
180 static voucher_t
181 _voucher_find_and_retain(mach_voucher_t kv)
182 {
183 voucher_t v;
184 if (!kv) return NULL;
185 _vouchers_lock_lock();
186 TAILQ_FOREACH(v, _vouchers_head(kv), v_list) {
187 if (v->v_ipc_kvoucher == kv) {
188 int xref_cnt = dispatch_atomic_inc2o(v, os_obj_xref_cnt, relaxed);
189 _dispatch_voucher_debug("retain -> %d", v, xref_cnt + 1);
190 if (slowpath(xref_cnt < 0)) {
191 _dispatch_voucher_debug("overrelease", v);
192 DISPATCH_CRASH("Voucher overrelease");
193 }
194 if (xref_cnt == 0) {
195 // resurrection: raced with _voucher_remove
196 (void)dispatch_atomic_inc2o(v, os_obj_ref_cnt, relaxed);
197 }
198 break;
199 }
200 }
201 _vouchers_lock_unlock();
202 return v;
203 }
204
205 static void
206 _voucher_insert(voucher_t v)
207 {
208 mach_voucher_t kv = v->v_ipc_kvoucher;
209 if (!kv) return;
210 _vouchers_lock_lock();
211 if (slowpath(_TAILQ_IS_ENQUEUED(v, v_list))) {
212 _dispatch_voucher_debug("corruption", v);
213 DISPATCH_CRASH("Voucher corruption");
214 }
215 TAILQ_INSERT_TAIL(_vouchers_head(kv), v, v_list);
216 _vouchers_lock_unlock();
217 }
218
219 static void
220 _voucher_remove(voucher_t v)
221 {
222 mach_voucher_t kv = v->v_ipc_kvoucher;
223 if (!_TAILQ_IS_ENQUEUED(v, v_list)) return;
224 _vouchers_lock_lock();
225 if (slowpath(!kv)) {
226 _dispatch_voucher_debug("corruption", v);
227 DISPATCH_CRASH("Voucher corruption");
228 }
229 // check for resurrection race with _voucher_find_and_retain
230 if (dispatch_atomic_load2o(v, os_obj_xref_cnt, seq_cst) < 0 &&
231 _TAILQ_IS_ENQUEUED(v, v_list)) {
232 TAILQ_REMOVE(_vouchers_head(kv), v, v_list);
233 _TAILQ_MARK_NOT_ENQUEUED(v, v_list);
234 v->v_list.tqe_next = (void*)~0ull;
235 }
236 _vouchers_lock_unlock();
237 }
238
239 void
240 _voucher_dealloc_mach_voucher(mach_voucher_t kv)
241 {
242 _dispatch_kvoucher_debug("dealloc", kv);
243 _dispatch_voucher_debug_machport(kv);
244 kern_return_t kr = mach_voucher_deallocate(kv);
245 DISPATCH_VERIFY_MIG(kr);
246 (void)dispatch_assume_zero(kr);
247 }
248
249 static inline kern_return_t
250 _voucher_create_mach_voucher(const mach_voucher_attr_recipe_data_t *recipes,
251 size_t recipes_size, mach_voucher_t *kvp)
252 {
253 kern_return_t kr;
254 mach_port_t mhp = _dispatch_get_mach_host_port();
255 mach_voucher_t kv = MACH_VOUCHER_NULL;
256 mach_voucher_attr_raw_recipe_array_t kvr;
257 mach_voucher_attr_recipe_size_t kvr_size;
258 kvr = (mach_voucher_attr_raw_recipe_array_t)recipes;
259 kvr_size = (mach_voucher_attr_recipe_size_t)recipes_size;
260 kr = host_create_mach_voucher(mhp, kvr, kvr_size, &kv);
261 DISPATCH_VERIFY_MIG(kr);
262 if (!kr) {
263 _dispatch_kvoucher_debug("create", kv);
264 _dispatch_voucher_debug_machport(kv);
265 }
266 *kvp = kv;
267 return kr;
268 }
269
270 #if __has_include(<bank/bank_types.h>) && !defined(VOUCHER_USE_ATTR_BANK)
271 #include <bank/bank_types.h>
272 #define VOUCHER_USE_ATTR_BANK 1
273 mach_voucher_t _voucher_default_task_mach_voucher;
274 #endif
275
276 void
277 _voucher_task_mach_voucher_init(void* ctxt DISPATCH_UNUSED)
278 {
279 #if VOUCHER_USE_ATTR_BANK
280 kern_return_t kr;
281 mach_voucher_t kv;
282 static const mach_voucher_attr_recipe_data_t task_create_recipe = {
283 .key = MACH_VOUCHER_ATTR_KEY_BANK,
284 .command = MACH_VOUCHER_ATTR_BANK_CREATE,
285 };
286 kr = _voucher_create_mach_voucher(&task_create_recipe,
287 sizeof(task_create_recipe), &kv);
288 if (dispatch_assume_zero(kr)) {
289 DISPATCH_CLIENT_CRASH("Could not create task mach voucher");
290 }
291 _voucher_default_task_mach_voucher = kv;
292 _voucher_task_mach_voucher = kv;
293 #endif
294 }
295
296 void
297 voucher_replace_default_voucher(void)
298 {
299 #if VOUCHER_USE_ATTR_BANK
300 (void)_voucher_get_task_mach_voucher(); // initalize task mach voucher
301 mach_voucher_t kv, tkv = MACH_VOUCHER_NULL;
302 voucher_t v = _voucher_get();
303 if (v && v->v_kvoucher) {
304 kern_return_t kr;
305 kv = v->v_ipc_kvoucher ? v->v_ipc_kvoucher : v->v_kvoucher;
306 const mach_voucher_attr_recipe_data_t task_copy_recipe = {
307 .key = MACH_VOUCHER_ATTR_KEY_BANK,
308 .command = MACH_VOUCHER_ATTR_COPY,
309 .previous_voucher = kv,
310 };
311 kr = _voucher_create_mach_voucher(&task_copy_recipe,
312 sizeof(task_copy_recipe), &tkv);
313 if (dispatch_assume_zero(kr)) {
314 tkv = MACH_VOUCHER_NULL;
315 }
316 }
317 if (!tkv) tkv = _voucher_default_task_mach_voucher;
318 kv = dispatch_atomic_xchg(&_voucher_task_mach_voucher, tkv, relaxed);
319 if (kv && kv != _voucher_default_task_mach_voucher) {
320 _voucher_dealloc_mach_voucher(kv);
321 }
322 _dispatch_voucher_debug("kvoucher[0x%08x] replace default voucher", v, tkv);
323 #endif
324 }
325
326 static inline _voucher_atm_t
327 _voucher_get_atm(voucher_t voucher)
328 {
329 _voucher_atm_t vatm;
330 vatm = voucher && voucher->v_atm ? voucher->v_atm : _voucher_task_atm;
331 return vatm;
332 }
333
334 static inline mach_voucher_t
335 _voucher_get_atm_mach_voucher(voucher_t voucher)
336 {
337 _voucher_atm_t vatm = _voucher_get_atm(voucher);
338 mach_voucher_t kv = vatm ? vatm->vatm_kvoucher : MACH_VOUCHER_NULL;
339 return kv;
340 }
341
342 mach_voucher_t
343 _voucher_get_mach_voucher(voucher_t voucher)
344 {
345 if (!voucher) return MACH_VOUCHER_NULL;
346 if (voucher->v_ipc_kvoucher) return voucher->v_ipc_kvoucher;
347 mach_voucher_t kvb = voucher->v_kvoucher;
348 if (!kvb) kvb = _voucher_get_task_mach_voucher();
349 if (!voucher->v_has_priority && !voucher->v_activities &&
350 !_voucher_extra_size(voucher)) {
351 return kvb;
352 }
353 kern_return_t kr;
354 mach_voucher_t kv, kvo;
355 _voucher_base_recipe(voucher).previous_voucher = kvb;
356 _voucher_atm_recipe(voucher).previous_voucher =
357 _voucher_get_atm_mach_voucher(voucher);
358 kr = _voucher_create_mach_voucher(voucher->v_recipes,
359 _voucher_recipes_size() + _voucher_extra_size(voucher) +
360 _voucher_bits_recipe(voucher).content_size, &kv);
361 if (dispatch_assume_zero(kr) || !kv){
362 return MACH_VOUCHER_NULL;
363 }
364 if (!dispatch_atomic_cmpxchgv2o(voucher, v_ipc_kvoucher, MACH_VOUCHER_NULL,
365 kv, &kvo, relaxed)) {
366 _voucher_dealloc_mach_voucher(kv);
367 kv = kvo;
368 } else {
369 if (kv == voucher->v_kvoucher) {
370 // if v_kvoucher == v_ipc_kvoucher we keep only one reference
371 _voucher_dealloc_mach_voucher(kv);
372 }
373 _voucher_insert(voucher);
374 _dispatch_voucher_debug("kvoucher[0x%08x] create", voucher, kv);
375 }
376 return kv;
377 }
378
379 mach_voucher_t
380 _voucher_create_mach_voucher_with_priority(voucher_t voucher,
381 pthread_priority_t priority)
382 {
383 if (priority == _voucher_get_priority(voucher)) {
384 return MACH_VOUCHER_NULL; // caller will use _voucher_get_mach_voucher
385 }
386 kern_return_t kr;
387 mach_voucher_t kv, kvb = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL;
388 if (!kvb) kvb = _voucher_get_task_mach_voucher();
389 mach_voucher_attr_recipe_data_t *recipes;
390 size_t recipes_size = _voucher_recipes_size();
391 if (voucher && (voucher->v_has_priority || voucher->v_activities ||
392 _voucher_extra_size(voucher))) {
393 recipes_size += _voucher_bits_recipe(voucher).content_size +
394 _voucher_extra_size(voucher);
395 recipes = alloca(recipes_size);
396 memcpy(recipes, voucher->v_recipes, recipes_size);
397 _voucher_recipes_atm(recipes).previous_voucher =
398 _voucher_get_atm_mach_voucher(voucher);
399 } else {
400 mach_voucher_attr_content_size_t bits_size = _voucher_bits_size(0);
401 recipes_size += bits_size;
402 recipes = alloca(recipes_size);
403 _voucher_recipes_init(recipes, bits_size);
404 }
405 _voucher_recipes_base(recipes).previous_voucher = kvb;
406 *_voucher_recipes_priority(recipes) = (_voucher_priority_t)priority;
407 kr = _voucher_create_mach_voucher(recipes, recipes_size, &kv);
408 if (dispatch_assume_zero(kr) || !kv){
409 return MACH_VOUCHER_NULL;
410 }
411 _dispatch_kvoucher_debug("create with priority from voucher[%p]", kv,
412 voucher);
413 return kv;
414 }
415
416 static voucher_t
417 _voucher_create_with_mach_voucher(mach_voucher_t kv)
418 {
419 if (!kv) return NULL;
420 kern_return_t kr;
421 mach_voucher_t rkv;
422 mach_voucher_attr_recipe_t vr;
423 size_t vr_size;
424 mach_voucher_attr_recipe_size_t kvr_size = 0;
425 const mach_voucher_attr_recipe_data_t redeem_recipe[] = {
426 [0] = {
427 .key = MACH_VOUCHER_ATTR_KEY_ALL,
428 .command = MACH_VOUCHER_ATTR_COPY,
429 .previous_voucher = kv,
430 },
431 #if VOUCHER_USE_ATTR_BANK
432 [1] = {
433 .key = MACH_VOUCHER_ATTR_KEY_BANK,
434 .command = MACH_VOUCHER_ATTR_REDEEM,
435 },
436 #endif
437 };
438 kr = _voucher_create_mach_voucher(redeem_recipe, sizeof(redeem_recipe),
439 &rkv);
440 if (!dispatch_assume_zero(kr)) {
441 _voucher_dealloc_mach_voucher(kv);
442 } else {
443 _dispatch_voucher_debug_machport(kv);
444 rkv = kv;
445 }
446 voucher_t v = _voucher_find_and_retain(rkv);
447 if (v) {
448 _dispatch_voucher_debug("kvoucher[0x%08x] find with 0x%08x", v, rkv,kv);
449 _voucher_dealloc_mach_voucher(rkv);
450 return v;
451 }
452 vr_size = sizeof(*vr) + _voucher_bits_size(_voucher_max_activities);
453 vr = alloca(vr_size);
454 if (rkv) {
455 kvr_size = (mach_voucher_attr_recipe_size_t)vr_size;
456 kr = mach_voucher_extract_attr_recipe(rkv,
457 MACH_VOUCHER_ATTR_KEY_USER_DATA, (void*)vr, &kvr_size);
458 DISPATCH_VERIFY_MIG(kr);
459 if (dispatch_assume_zero(kr)) kvr_size = 0;
460 }
461 mach_voucher_attr_content_size_t content_size = vr->content_size;
462 uint8_t *content = vr->content;
463 bool valid = false, has_priority = false;
464 unsigned int activities = 0;
465 if (kvr_size >= sizeof(*vr) + sizeof(_voucher_magic_t)) {
466 valid = (*(_voucher_magic_t*)content == _voucher_magic_v1);
467 content += sizeof(_voucher_magic_t);
468 content_size -= sizeof(_voucher_magic_t);
469 }
470 if (valid) {
471 has_priority = (content_size >= sizeof(_voucher_priority_t));
472 activities = has_priority ? (content_size - sizeof(_voucher_priority_t))
473 / sizeof(voucher_activity_id_t) : 0;
474 }
475 pthread_priority_t priority = 0;
476 if (has_priority) {
477 priority = (pthread_priority_t)*(_voucher_priority_t*)content;
478 content += sizeof(_voucher_priority_t);
479 content_size -= sizeof(_voucher_priority_t);
480 }
481 voucher_activity_id_t va_id = 0, va_base_id = 0;
482 _voucher_activity_t act = NULL;
483 _voucher_atm_t vatm = NULL;
484 if (activities) {
485 va_id = *(voucher_activity_id_t*)content;
486 act = _voucher_activity_copy_from_mach_voucher(rkv, va_id);
487 if (!act && _voucher_activity_default) {
488 activities++;
489 // default to _voucher_activity_default base activity
490 va_base_id = _voucher_activity_default->va_id;
491 } else if (act && act->va_id != va_id) {
492 activities++;
493 va_base_id = act->va_id;
494 }
495 if (act) {
496 vatm = _voucher_atm_retain(act->va_atm);
497 }
498 }
499 v = _voucher_alloc(activities, priority, 0);
500 v->v_atm = vatm;
501 v->v_activity = act;
502 voucher_activity_id_t *activity_ids = _voucher_activity_ids(v);
503 if (activities && va_base_id) {
504 *activity_ids++ = va_base_id;
505 activities--;
506 }
507 if (activities) {
508 memcpy(activity_ids, content, content_size);
509 }
510 v->v_ipc_kvoucher = v->v_kvoucher = rkv;
511 _voucher_insert(v);
512 _dispatch_voucher_debug("kvoucher[0x%08x] create with 0x%08x", v, rkv, kv);
513 return v;
514 }
515
516 voucher_t
517 _voucher_create_with_priority_and_mach_voucher(voucher_t ov,
518 pthread_priority_t priority, mach_voucher_t kv)
519 {
520 if (priority == _voucher_get_priority(ov)) {
521 if (kv) _voucher_dealloc_mach_voucher(kv);
522 return ov ? _voucher_retain(ov) : NULL;
523 }
524 voucher_t v = _voucher_find_and_retain(kv);
525 if (v) {
526 _dispatch_voucher_debug("kvoucher[0x%08x] find", v, kv);
527 _voucher_dealloc_mach_voucher(kv);
528 return v;
529 }
530 unsigned int activities = ov ? ov->v_activities : 0;
531 mach_voucher_attr_recipe_size_t extra = ov ? _voucher_extra_size(ov) : 0;
532 v = _voucher_alloc(activities, priority, extra);
533 if (extra) {
534 memcpy(_voucher_extra_recipes(v), _voucher_extra_recipes(ov), extra);
535 }
536 if (activities) {
537 if (ov->v_activity) {
538 v->v_activity = _voucher_activity_retain(ov->v_activity);
539 v->v_atm = _voucher_atm_retain(ov->v_atm);
540 }
541 memcpy(_voucher_activity_ids(v), _voucher_activity_ids(ov),
542 activities * sizeof(voucher_activity_id_t));
543 }
544 if (kv) {
545 v->v_ipc_kvoucher = v->v_kvoucher = kv;
546 _voucher_insert(v);
547 _dispatch_voucher_debug("kvoucher[0x%08x] create with priority from "
548 "voucher[%p]", v, kv, ov);
549 _dispatch_voucher_debug_machport(kv);
550 } else if (ov && ov->v_kvoucher) {
551 voucher_t kvb = ov->v_kvbase ? ov->v_kvbase : ov;
552 v->v_kvbase = _voucher_retain(kvb);
553 v->v_kvoucher = kvb->v_kvoucher;
554 }
555 return v;
556 }
557
558 voucher_t
559 _voucher_create_without_importance(voucher_t ov)
560 {
561 // Nothing to do unless the old voucher has a kernel voucher. If it
562 // doesn't, it can't have any importance, now or in the future.
563 if (!ov) return NULL;
564 // TODO: 17487167: track presence of importance attribute
565 if (!ov->v_kvoucher) return _voucher_retain(ov);
566 kern_return_t kr;
567 mach_voucher_t kv, okv;
568 // Copy kernel voucher, removing importance.
569 okv = ov->v_ipc_kvoucher ? ov->v_ipc_kvoucher : ov->v_kvoucher;
570 const mach_voucher_attr_recipe_data_t importance_remove_recipe[] = {
571 [0] = {
572 .key = MACH_VOUCHER_ATTR_KEY_ALL,
573 .command = MACH_VOUCHER_ATTR_COPY,
574 .previous_voucher = okv,
575 },
576 [1] = {
577 .key = MACH_VOUCHER_ATTR_KEY_IMPORTANCE,
578 .command = MACH_VOUCHER_ATTR_REMOVE,
579 },
580 };
581 kr = _voucher_create_mach_voucher(importance_remove_recipe,
582 sizeof(importance_remove_recipe), &kv);
583 if (dispatch_assume_zero(kr) || !kv){
584 if (ov->v_ipc_kvoucher) return NULL;
585 kv = MACH_VOUCHER_NULL;
586 }
587 if (kv == okv) {
588 _voucher_dealloc_mach_voucher(kv);
589 return _voucher_retain(ov);
590 }
591 voucher_t v = _voucher_find_and_retain(kv);
592 if (v && ov->v_ipc_kvoucher) {
593 _dispatch_voucher_debug("kvoucher[0x%08x] find without importance "
594 "from voucher[%p]", v, kv, ov);
595 _voucher_dealloc_mach_voucher(kv);
596 return v;
597 }
598 voucher_t kvbase = v;
599 // Copy userspace contents
600 unsigned int activities = ov->v_activities;
601 pthread_priority_t priority = _voucher_get_priority(ov);
602 mach_voucher_attr_recipe_size_t extra = _voucher_extra_size(ov);
603 v = _voucher_alloc(activities, priority, extra);
604 if (extra) {
605 memcpy(_voucher_extra_recipes(v), _voucher_extra_recipes(ov), extra);
606 }
607 if (activities) {
608 if (ov->v_activity) {
609 v->v_activity = _voucher_activity_retain(ov->v_activity);
610 v->v_atm = _voucher_atm_retain(ov->v_atm);
611 }
612 memcpy(_voucher_activity_ids(v), _voucher_activity_ids(ov),
613 activities * sizeof(voucher_activity_id_t));
614 }
615 v->v_kvoucher = kv;
616 if (ov->v_ipc_kvoucher) {
617 v->v_ipc_kvoucher = kv;
618 _voucher_insert(v);
619 } else if (kvbase) {
620 v->v_kvbase = kvbase;
621 _voucher_dealloc_mach_voucher(kv); // borrow base reference
622 }
623 if (!kvbase) {
624 _dispatch_voucher_debug("kvoucher[0x%08x] create without importance "
625 "from voucher[%p]", v, kv, ov);
626 }
627 return v;
628 }
629
630 voucher_t
631 _voucher_create_accounting_voucher(voucher_t ov)
632 {
633 // Nothing to do unless the old voucher has a kernel voucher. If it does
634 // doesn't, it can't have any accounting attributes.
635 if (!ov || !ov->v_kvoucher) return NULL;
636 kern_return_t kr = KERN_SUCCESS;
637 mach_voucher_t okv, kv = MACH_VOUCHER_NULL;
638 okv = ov->v_ipc_kvoucher ? ov->v_ipc_kvoucher : ov->v_kvoucher;
639 #if VOUCHER_USE_ATTR_BANK
640 const mach_voucher_attr_recipe_data_t accounting_copy_recipe = {
641 .key = MACH_VOUCHER_ATTR_KEY_BANK,
642 .command = MACH_VOUCHER_ATTR_COPY,
643 .previous_voucher = okv,
644 };
645 kr = _voucher_create_mach_voucher(&accounting_copy_recipe,
646 sizeof(accounting_copy_recipe), &kv);
647 #endif
648 if (dispatch_assume_zero(kr) || !kv){
649 return NULL;
650 }
651 voucher_t v = _voucher_find_and_retain(kv);
652 if (v) {
653 _dispatch_voucher_debug("kvoucher[0x%08x] find accounting voucher "
654 "from voucher[%p]", v, kv, ov);
655 _voucher_dealloc_mach_voucher(kv);
656 return v;
657 }
658 v = _voucher_alloc(0, 0, 0);
659 v->v_ipc_kvoucher = v->v_kvoucher = kv;
660 if (kv == okv) {
661 v->v_kvbase = _voucher_retain(ov);
662 _voucher_dealloc_mach_voucher(kv); // borrow base reference
663 }
664 _voucher_insert(v);
665 _dispatch_voucher_debug("kvoucher[0x%08x] create accounting voucher "
666 "from voucher[%p]", v, kv, ov);
667 return v;
668 }
669
670 voucher_t
671 voucher_create_with_mach_msg(mach_msg_header_t *msg)
672 {
673 voucher_t v = _voucher_create_with_mach_voucher(_voucher_mach_msg_get(msg));
674 _voucher_activity_trace_msg(v, msg, receive);
675 return v;
676 }
677
678 #ifndef MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL
679 #define MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL 2
680 #endif
681
682 void
683 voucher_decrement_importance_count4CF(voucher_t v)
684 {
685 if (!v || !v->v_kvoucher) return;
686 // TODO: 17487167: track presence of importance attribute
687 kern_return_t kr;
688 mach_voucher_t kv = v->v_ipc_kvoucher ? v->v_ipc_kvoucher : v->v_kvoucher;
689 uint32_t dec = 1;
690 mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&dec;
691 mach_voucher_attr_content_size_t kvc_in_size = sizeof(dec);
692 mach_voucher_attr_content_t kvc_out = NULL;
693 mach_voucher_attr_content_size_t kvc_out_size = 0;
694 #if DISPATCH_DEBUG
695 uint32_t count = UINT32_MAX;
696 kvc_out = (mach_voucher_attr_content_t)&count;
697 kvc_out_size = sizeof(count);
698 #endif
699 kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_IMPORTANCE,
700 MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL, kvc_in, kvc_in_size,
701 kvc_out, &kvc_out_size);
702 DISPATCH_VERIFY_MIG(kr);
703 #if DISPATCH_DEBUG
704 _dispatch_voucher_debug("kvoucher[0x%08x] decrement importance count to %u:"
705 " %s - 0x%x", v, kv, count, mach_error_string(kr), kr);
706 #endif
707 if (kr != KERN_INVALID_ARGUMENT &&
708 dispatch_assume_zero(kr) == KERN_FAILURE) {
709 // TODO: 17487167: skip KERN_INVALID_ARGUMENT check
710 DISPATCH_CLIENT_CRASH("Voucher importance count underflow");
711 }
712 }
713
714 #if VOUCHER_ENABLE_GET_MACH_VOUCHER
715 mach_voucher_t
716 voucher_get_mach_voucher(voucher_t voucher)
717 {
718 return _voucher_get_mach_voucher(voucher);
719 }
720 #endif
721
722 void
723 _voucher_xref_dispose(voucher_t voucher)
724 {
725 _dispatch_voucher_debug("xref_dispose", voucher);
726 _voucher_remove(voucher);
727 return _os_object_release_internal_inline((_os_object_t)voucher);
728 }
729
730 void
731 _voucher_dispose(voucher_t voucher)
732 {
733 _dispatch_voucher_debug("dispose", voucher);
734 if (slowpath(_TAILQ_IS_ENQUEUED(voucher, v_list))) {
735 _dispatch_voucher_debug("corruption", voucher);
736 DISPATCH_CRASH("Voucher corruption");
737 }
738 voucher->v_list.tqe_next = DISPATCH_OBJECT_LISTLESS;
739 if (voucher->v_ipc_kvoucher) {
740 if (voucher->v_ipc_kvoucher != voucher->v_kvoucher) {
741 _voucher_dealloc_mach_voucher(voucher->v_ipc_kvoucher);
742 }
743 voucher->v_ipc_kvoucher = MACH_VOUCHER_NULL;
744 }
745 if (voucher->v_kvoucher) {
746 if (!voucher->v_kvbase) {
747 _voucher_dealloc_mach_voucher(voucher->v_kvoucher);
748 }
749 voucher->v_kvoucher = MACH_VOUCHER_NULL;
750 }
751 if (voucher->v_kvbase) {
752 _voucher_release(voucher->v_kvbase);
753 voucher->v_kvbase = NULL;
754 }
755 if (voucher->v_activity) {
756 _voucher_activity_release(voucher->v_activity);
757 voucher->v_activity = NULL;
758 }
759 if (voucher->v_atm) {
760 _voucher_atm_release(voucher->v_atm);
761 voucher->v_atm = NULL;
762 }
763 voucher->v_has_priority = 0;
764 voucher->v_activities = 0;
765 #if VOUCHER_ENABLE_RECIPE_OBJECTS
766 voucher->v_recipe_extra_size = 0;
767 voucher->v_recipe_extra_offset = 0;
768 #endif
769 return _os_object_dealloc((_os_object_t)voucher);
770 }
771
772 void
773 _voucher_atfork_child(void)
774 {
775 _voucher_activity_atfork_child();
776 _dispatch_thread_setspecific(dispatch_voucher_key, NULL);
777 _voucher_task_mach_voucher_pred = 0;
778 _voucher_task_mach_voucher = MACH_VOUCHER_NULL;
779
780 // TODO: voucher/activity inheritance on fork ?
781 }
782
783 #pragma mark -
784 #pragma mark _voucher_init
785
786 boolean_t
787 voucher_mach_msg_set(mach_msg_header_t *msg)
788 {
789 voucher_t v = _voucher_get();
790 bool clear_voucher = _voucher_mach_msg_set(msg, v);
791 if (clear_voucher) _voucher_activity_trace_msg(v, msg, send);
792 return clear_voucher;
793 }
794
795 void
796 voucher_mach_msg_clear(mach_msg_header_t *msg)
797 {
798 (void)_voucher_mach_msg_clear(msg, false);
799 }
800
801 voucher_mach_msg_state_t
802 voucher_mach_msg_adopt(mach_msg_header_t *msg)
803 {
804 mach_voucher_t kv = _voucher_mach_msg_get(msg);
805 if (!kv) return VOUCHER_MACH_MSG_STATE_UNCHANGED;
806 voucher_t v = _voucher_create_with_mach_voucher(kv);
807 _voucher_activity_trace_msg(v, msg, receive);
808 return (voucher_mach_msg_state_t)_voucher_adopt(v);
809 }
810
811 void
812 voucher_mach_msg_revert(voucher_mach_msg_state_t state)
813 {
814 if (state == VOUCHER_MACH_MSG_STATE_UNCHANGED) return;
815 _voucher_replace((voucher_t)state);
816 }
817
818 #if DISPATCH_USE_LIBKERNEL_VOUCHER_INIT
819 #include <_libkernel_init.h>
820
821 static const struct _libkernel_voucher_functions _voucher_libkernel_functions =
822 {
823 .version = 1,
824 .voucher_mach_msg_set = voucher_mach_msg_set,
825 .voucher_mach_msg_clear = voucher_mach_msg_clear,
826 .voucher_mach_msg_adopt = voucher_mach_msg_adopt,
827 .voucher_mach_msg_revert = voucher_mach_msg_revert,
828 };
829
830 static void
831 _voucher_libkernel_init(void)
832 {
833 kern_return_t kr = __libkernel_voucher_init(&_voucher_libkernel_functions);
834 dispatch_assert(!kr);
835 }
836 #else
837 #define _voucher_libkernel_init()
838 #endif
839
840 void
841 _voucher_init(void)
842 {
843 _voucher_libkernel_init();
844 char *e;
845 unsigned int i;
846 for (i = 0; i < VL_HASH_SIZE; i++) {
847 TAILQ_INIT(&_vouchers[i]);
848 }
849 voucher_activity_mode_t mode;
850 mode = DISPATCH_DEBUG ? voucher_activity_mode_debug
851 : voucher_activity_mode_release;
852 e = getenv("OS_ACTIVITY_MODE");
853 if (e) {
854 if (strcmp(e, "release") == 0) {
855 mode = voucher_activity_mode_release;
856 } else if (strcmp(e, "debug") == 0) {
857 mode = voucher_activity_mode_debug;
858 } else if (strcmp(e, "stream") == 0) {
859 mode = voucher_activity_mode_stream;
860 } else if (strcmp(e, "disable") == 0) {
861 mode = voucher_activity_mode_disable;
862 }
863 }
864 _voucher_activity_mode = mode;
865 if (_voucher_activity_disabled()) return;
866
867 // default task activity
868 bool default_task_activity = DISPATCH_DEBUG;
869 e = getenv("LIBDISPATCH_DEFAULT_TASK_ACTIVITY");
870 if (e) default_task_activity = atoi(e);
871 if (default_task_activity) {
872 (void)voucher_activity_start(_voucher_activity_trace_id_release, 0);
873 }
874 }
875
876 #pragma mark -
877 #pragma mark _voucher_activity_lock_s
878
879 DISPATCH_ALWAYS_INLINE
880 static inline void
881 _voucher_activity_lock_init(_voucher_activity_lock_s *lock) {
882 static const os_lock_handoff_s _os_lock_handoff_init = OS_LOCK_HANDOFF_INIT;
883 *lock = _os_lock_handoff_init;
884 }
885
886 DISPATCH_ALWAYS_INLINE
887 static inline void
888 _voucher_activity_lock_lock(_voucher_activity_lock_s *lock) {
889 return os_lock_lock(lock);
890 }
891
892 DISPATCH_ALWAYS_INLINE
893 static inline void
894 _voucher_activity_lock_unlock(_voucher_activity_lock_s *lock) {
895 return os_lock_unlock(lock);
896 }
897
898 #pragma mark -
899 #pragma mark _voucher_activity_heap
900
901 #if __has_extension(c_static_assert)
902 _Static_assert(sizeof(struct _voucher_activity_tracepoint_s) == 64,
903 "Tracepoint too large");
904 _Static_assert(sizeof(struct _voucher_activity_buffer_header_s) <=
905 sizeof(struct _voucher_activity_tracepoint_s),
906 "Buffer header too large");
907 #if __LP64__
908 _Static_assert(offsetof(struct _voucher_activity_s, va_buffers_lock) % 64 == 0,
909 "Bad activity padding");
910 _Static_assert(sizeof(struct _voucher_atm_s) <= 128,
911 "ATM too large");
912 #else
913 _Static_assert(sizeof(struct _voucher_atm_s) <= 64,
914 "ATM too large");
915 #endif
916 _Static_assert(sizeof(_voucher_activity_buffer_t) ==
917 sizeof(struct {char x[_voucher_activity_buffer_size];}),
918 "Buffer too large");
919 _Static_assert(sizeof(struct _voucher_activity_metadata_s) <=
920 sizeof(struct _voucher_activity_metadata_opaque_s),
921 "Metadata too large");
922 _Static_assert(sizeof(_voucher_activity_bitmap_t) % 64 == 0,
923 "Bad metadata bitmap size");
924 #endif
925
926 #define va_buffers_lock(va) (&(va)->va_buffers_lock)
927 #define vatm_activities(vatm) (&(vatm)->vatm_activities)
928 #define vam_atms_lock() (&_voucher_activity_heap->vam_atms_lock)
929 #define vam_activities_lock() (&_voucher_activity_heap->vam_activities_lock)
930 #define vam_atms(hash) (&_voucher_activity_heap->vam_atms[hash])
931 #define vam_activities(hash) (&_voucher_activity_heap->vam_activities[hash])
932 #define vam_buffer_bitmap() (_voucher_activity_heap->vam_buffer_bitmap)
933 #define vam_pressure_locked_bitmap() \
934 (_voucher_activity_heap->vam_pressure_locked_bitmap)
935 #define vam_buffer(i) ((void*)((char*)_voucher_activity_heap + \
936 (i) * _voucher_activity_buffer_size))
937
938 static _voucher_activity_t _voucher_activity_create_with_atm(
939 _voucher_atm_t vatm, voucher_activity_id_t va_id,
940 voucher_activity_trace_id_t trace_id, uint64_t location,
941 _voucher_activity_buffer_header_t buffer);
942 static _voucher_atm_t _voucher_atm_create(mach_voucher_t kv, atm_aid_t atm_id);
943 static void _voucher_activity_firehose_wait(_voucher_activity_t act,
944 _voucher_activity_buffer_header_t buffer);
945
946 DISPATCH_ALWAYS_INLINE
947 static inline uint32_t
948 _voucher_default_activity_buffer_limit()
949 {
950 #if 0 // FIXME: tune buffer chain sizes
951 switch (_voucher_activity_mode) {
952 case voucher_activity_mode_debug:
953 case voucher_activity_mode_stream:
954 // High-profile modes: Default activity can use 1/32nd of the heap
955 // (twice as much as non-default activities)
956 return MAX(_voucher_activity_buffers_per_heap / 32, 3) - 1;
957 }
958 #endif
959 // Low-profile modes: Default activity can use a total of 4 buffers.
960 return 3;
961 }
962
963 DISPATCH_ALWAYS_INLINE
964 static inline uint32_t
965 _voucher_activity_buffer_limit()
966 {
967 #if 0 // FIXME: tune buffer chain sizes
968 switch (_voucher_activity_mode) {
969 case voucher_activity_mode_debug:
970 case voucher_activity_mode_stream:
971 // High-profile modes: 64 activities, each of which can use 1/64th
972 // of the entire heap.
973 return MAX(_voucher_activity_buffers_per_heap / 64, 2) - 1;
974 }
975 #endif
976 // Low-profile modes: Each activity can use a total of 2 buffers.
977 return 1;
978 }
979
980 // The two functions above return the number of *additional* buffers activities
981 // may allocate, hence the gymnastics with - 1.
982
983 DISPATCH_ALWAYS_INLINE
984 static inline uint32_t
985 _voucher_heap_buffer_limit()
986 {
987 switch (_voucher_activity_mode) {
988 case voucher_activity_mode_debug:
989 case voucher_activity_mode_stream:
990 // High-profile modes: Use it all.
991 return _voucher_activity_buffers_per_heap;
992 }
993 #if TARGET_OS_EMBEDDED
994 // Low-profile modes: 3 activities, each of which can use 2 buffers;
995 // plus the default activity, which can use 3; plus 3 buffers of overhead.
996 return 12;
997 #else
998 // Low-profile modes: 13 activities, each of which can use 4 buffers;
999 // plus the default activity, which can use 8; plus 3 buffers of overhead.
1000 return 64;
1001 #endif
1002 }
1003
1004 #define NO_BITS_WERE_UNSET (UINT_MAX)
1005
1006 DISPATCH_ALWAYS_INLINE
1007 static inline size_t
1008 _voucher_activity_bitmap_set_first_unset_bit_upto(
1009 _voucher_activity_bitmap_t volatile bitmap,
1010 unsigned int max_index)
1011 {
1012 dispatch_assert(max_index != 0);
1013 unsigned int index = NO_BITS_WERE_UNSET, max_map, max_bit, i;
1014 max_map = max_index / _voucher_activity_bits_per_bitmap_base_t;
1015 max_map = MIN(max_map, _voucher_activity_bitmaps_per_heap - 1);
1016 max_bit = max_index % _voucher_activity_bits_per_bitmap_base_t;
1017 for (i = 0; i < max_map; i++) {
1018 index = dispatch_atomic_set_first_bit(&bitmap[i], UINT_MAX);
1019 if (fastpath(index < NO_BITS_WERE_UNSET)) {
1020 return index + i * _voucher_activity_bits_per_bitmap_base_t;
1021 }
1022 }
1023 index = dispatch_atomic_set_first_bit(&bitmap[i], max_bit);
1024 if (fastpath(index < NO_BITS_WERE_UNSET)) {
1025 return index + i * _voucher_activity_bits_per_bitmap_base_t;
1026 }
1027 return index;
1028 }
1029
1030 DISPATCH_ALWAYS_INLINE DISPATCH_UNUSED
1031 static inline size_t
1032 _voucher_activity_bitmap_set_first_unset_bit(
1033 _voucher_activity_bitmap_t volatile bitmap)
1034 {
1035 return _voucher_activity_bitmap_set_first_unset_bit_upto(bitmap, UINT_MAX);
1036 }
1037
1038 DISPATCH_ALWAYS_INLINE
1039 static inline void
1040 _voucher_activity_bitmap_clear_bit(
1041 _voucher_activity_bitmap_t volatile bitmap, size_t index)
1042 {
1043 size_t i = index / _voucher_activity_bits_per_bitmap_base_t;
1044 _voucher_activity_bitmap_base_t mask = ((typeof(mask))1) <<
1045 (index % _voucher_activity_bits_per_bitmap_base_t);
1046 if (slowpath((bitmap[i] & mask) == 0)) {
1047 DISPATCH_CRASH("Corruption: failed to clear bit exclusively");
1048 }
1049 (void)dispatch_atomic_and(&bitmap[i], ~mask, release);
1050 }
1051
1052 _voucher_activity_metadata_t _voucher_activity_heap;
1053 static dispatch_once_t _voucher_activity_heap_pred;
1054
1055 static void
1056 _voucher_activity_heap_init(void *ctxt DISPATCH_UNUSED)
1057 {
1058 if (_voucher_activity_disabled()) return;
1059 kern_return_t kr;
1060 mach_vm_size_t vm_size = _voucher_activity_buffer_size *
1061 _voucher_activity_buffers_per_heap;
1062 mach_vm_address_t vm_addr = vm_page_size;
1063 while (slowpath(kr = mach_vm_map(mach_task_self(), &vm_addr, vm_size,
1064 0, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_MEMORY_GENEALOGY),
1065 MEMORY_OBJECT_NULL, 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL,
1066 VM_INHERIT_NONE))) {
1067 if (kr != KERN_NO_SPACE) {
1068 (void)dispatch_assume_zero(kr);
1069 _voucher_activity_mode = voucher_activity_mode_disable;
1070 return;
1071 }
1072 _dispatch_temporary_resource_shortage();
1073 vm_addr = vm_page_size;
1074 }
1075 _voucher_activity_metadata_t heap;
1076 task_trace_memory_info_data_t trace_memory_info = {
1077 .user_memory_address = vm_addr,
1078 .buffer_size = vm_size,
1079 };
1080 kr = task_set_info(mach_task_self(), TASK_TRACE_MEMORY_INFO,
1081 (task_info_t)&trace_memory_info, TASK_TRACE_MEMORY_INFO_COUNT);
1082 DISPATCH_VERIFY_MIG(kr);
1083 if (kr) {
1084 if (kr != KERN_NOT_SUPPORTED) (void)dispatch_assume_zero(kr);
1085 kr = mach_vm_deallocate(mach_task_self(), vm_addr, vm_size);
1086 (void)dispatch_assume_zero(kr);
1087 _voucher_activity_mode = voucher_activity_mode_disable;
1088 return;
1089 }
1090 heap = (void*)vm_addr;
1091 heap->vasm_baseaddr = (void*)vm_addr;
1092 heap->vam_buffer_bitmap[0] = 0x7; // first three buffers are reserved
1093 uint32_t i;
1094 for (i = 0; i < _voucher_activity_hash_size; i++) {
1095 TAILQ_INIT(&heap->vam_activities[i]);
1096 TAILQ_INIT(&heap->vam_atms[i]);
1097 }
1098 _voucher_activity_lock_init(&heap->vam_atms_lock);
1099 _voucher_activity_lock_init(&heap->vam_activities_lock);
1100 _voucher_activity_heap = heap;
1101
1102 _voucher_atm_t vatm = _voucher_atm_create(0, 0);
1103 dispatch_assert(vatm->vatm_kvoucher);
1104 _voucher_atm_retain(vatm);
1105
1106 _voucher_activity_buffer_header_t buffer = vam_buffer(2); // reserved index
1107 // consumes vatm reference:
1108 _voucher_activity_t va = _voucher_activity_create_with_atm(vatm, 0, 0, 0,
1109 buffer);
1110 dispatch_assert(va);
1111 va->va_buffer_limit = _voucher_default_activity_buffer_limit();
1112 _voucher_activity_default = va;
1113 _voucher_task_atm = vatm;
1114 }
1115
1116 static void
1117 _voucher_activity_atfork_child(void)
1118 {
1119 _voucher_activity_heap_pred = 0;
1120 _voucher_activity_heap = NULL; // activity heap is VM_INHERIT_NONE
1121 _voucher_activity_default = NULL;
1122 }
1123
1124 void*
1125 voucher_activity_get_metadata_buffer(size_t *length)
1126 {
1127 dispatch_once_f(&_voucher_activity_heap_pred, NULL,
1128 _voucher_activity_heap_init);
1129 if (_voucher_activity_disabled()) {
1130 *length = 0;
1131 return NULL;
1132 }
1133 *length = sizeof(_voucher_activity_heap->vam_client_metadata);
1134 return _voucher_activity_heap->vam_client_metadata;
1135 }
1136
1137 static _voucher_activity_buffer_hook_t _voucher_activity_buffer_hook;
1138
1139 void
1140 voucher_activity_buffer_hook_install_4libtrace(
1141 _voucher_activity_buffer_hook_t hook)
1142 {
1143 if (dispatch_atomic_cmpxchg(&_voucher_activity_buffer_hook, NULL,
1144 (void*)hook, release)) return;
1145 DISPATCH_CLIENT_CRASH("_voucher_activity_buffer_hook_install_4libtrace " \
1146 "called more than once");
1147 }
1148
1149 #if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG
1150 #define VOUCHER_ACTIVITY_BUFFER_DEBUG(reason, buffer) \
1151 _dispatch_debug("activity buffer %s (%p)", #reason, buffer)
1152 #else
1153 #define VOUCHER_ACTIVITY_BUFFER_DEBUG(reason, buffer)
1154 #endif
1155
1156 #define VOUCHER_ACTIVITY_BUFFER_HOOK_CALLOUT(reason, buffer) \
1157 if (buffer) { VOUCHER_ACTIVITY_BUFFER_DEBUG(reason, buffer); \
1158 if (slowpath(_voucher_activity_buffer_hook)) { \
1159 _voucher_activity_buffer_hook( \
1160 _voucher_activity_buffer_hook_reason_##reason, (buffer)); \
1161 } }
1162
1163 DISPATCH_ALWAYS_INLINE
1164 static inline _voucher_activity_buffer_header_t
1165 _voucher_activity_heap_buffer_alloc(void)
1166 {
1167 _voucher_activity_buffer_header_t buffer = NULL;
1168 size_t index;
1169 index = _voucher_activity_bitmap_set_first_unset_bit_upto(
1170 vam_buffer_bitmap(), _voucher_heap_buffer_limit() - 1);
1171 if (index < NO_BITS_WERE_UNSET) {
1172 buffer = vam_buffer(index);
1173 }
1174 #if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG
1175 _dispatch_debug("activity heap alloc %zd (%p)", index, buffer);
1176 #endif
1177 return buffer;
1178 }
1179
1180 DISPATCH_ALWAYS_INLINE
1181 static inline void
1182 _voucher_activity_heap_buffer_free(_voucher_activity_buffer_header_t buffer)
1183 {
1184 buffer->vabh_flags = _voucher_activity_trace_flag_buffer_empty;
1185 size_t index = (size_t)((char*)buffer - (char*)_voucher_activity_heap) /
1186 _voucher_activity_buffer_size;
1187 #if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG
1188 _dispatch_debug("activity heap free %zd (%p)", index, buffer);
1189 #endif
1190 _voucher_activity_bitmap_clear_bit(vam_buffer_bitmap(), index);
1191 }
1192
1193 #define _voucher_activity_heap_can_madvise() \
1194 (PAGE_SIZE == _voucher_activity_buffer_size) // <rdar://17445544>
1195
1196 DISPATCH_ALWAYS_INLINE
1197 static inline void
1198 _voucher_activity_heap_madvise(size_t bitmap_num, unsigned int start,
1199 unsigned int len)
1200 {
1201 size_t base = bitmap_num * _voucher_activity_bits_per_bitmap_base_t;
1202 #if DISPATCH_DEBUG
1203 #if DISPATCH_VOUCHER_ACTIVITY_DEBUG
1204 _dispatch_debug("activity heap madvise %zd (%p) -> %zd (%p)", base + start,
1205 vam_buffer(base + start), base + start + len,
1206 vam_buffer(base + start + len));
1207 #endif
1208 dispatch_assert(!(len * _voucher_activity_buffer_size % vm_page_size));
1209 const uint64_t pattern = 0xFACEFACEFACEFACE;
1210 _voucher_activity_buffer_header_t buffer = vam_buffer(base + start);
1211 for (unsigned int i = 0; i < len; i++, buffer++) {
1212 memset_pattern8((char*)buffer + sizeof(buffer->vabh_flags), &pattern,
1213 _voucher_activity_buffer_size - sizeof(buffer->vabh_flags));
1214 }
1215 #endif
1216 (void)dispatch_assume_zero(madvise(vam_buffer(base + start),
1217 len * _voucher_activity_buffer_size, MADV_FREE));
1218 }
1219
1220 DISPATCH_ALWAYS_INLINE
1221 static inline void
1222 _voucher_activity_heap_madvise_contiguous(size_t bitmap_num,
1223 _voucher_activity_bitmap_base_t bits)
1224 {
1225 // TODO: x86 has fast ctz; arm has fast clz; haswell has fast ctz
1226 dispatch_assert(_voucher_activity_heap_can_madvise());
1227 if (bits == 0) {
1228 return;
1229 } else if (~bits == 0) {
1230 _voucher_activity_heap_madvise(bitmap_num, 0,
1231 _voucher_activity_bits_per_bitmap_base_t);
1232 } else while (bits != 0) {
1233 unsigned int start = (typeof(start))__builtin_ctzl(bits), len;
1234 typeof(bits) inverse = ~bits >> start;
1235 if (inverse) {
1236 len = (typeof(len))__builtin_ctzl(inverse);
1237 } else {
1238 len = _voucher_activity_bits_per_bitmap_base_t - start;
1239 }
1240 typeof(bits) mask = ((((typeof(bits))1) << len) - 1) << start;
1241 bits &= ~mask;
1242 _voucher_activity_heap_madvise(bitmap_num, start, len);
1243 }
1244 }
1245
1246 void
1247 _voucher_activity_heap_pressure_warn(void)
1248 {
1249 if (!_voucher_activity_heap_can_madvise() || !_voucher_activity_heap) {
1250 return;
1251 }
1252 volatile _voucher_activity_bitmap_base_t *bitmap, *pressure_locked_bitmap;
1253 bitmap = vam_buffer_bitmap();
1254 pressure_locked_bitmap = vam_pressure_locked_bitmap();
1255
1256 // number of bitmaps needed to map the current buffer limit =
1257 // ceil(buffer limit / bits per bitmap)
1258 size_t nbuffers = _voucher_heap_buffer_limit();
1259 size_t nbitmaps_quot = nbuffers / _voucher_activity_bits_per_bitmap_base_t;
1260 size_t nbitmaps_rem = nbuffers % _voucher_activity_bits_per_bitmap_base_t;
1261 size_t nbitmaps = nbitmaps_quot + ((nbitmaps_rem == 0) ? 0 : 1);
1262
1263 for (size_t i = 0; i < nbitmaps; i++) {
1264 _voucher_activity_bitmap_base_t got_bits;
1265 got_bits = dispatch_atomic_or_orig(&bitmap[i], ~((typeof(bitmap[i]))0),
1266 relaxed);
1267 got_bits = ~got_bits; // Now 1 means 'acquired this one, madvise it'
1268 _voucher_activity_heap_madvise_contiguous(i, got_bits);
1269 pressure_locked_bitmap[i] |= got_bits;
1270 }
1271 }
1272
1273 void
1274 _voucher_activity_heap_pressure_normal(void)
1275 {
1276 if (!_voucher_activity_heap_can_madvise() || !_voucher_activity_heap) {
1277 return;
1278 }
1279 volatile _voucher_activity_bitmap_base_t *bitmap, *pressure_locked_bitmap;
1280 bitmap = vam_buffer_bitmap();
1281 pressure_locked_bitmap = vam_pressure_locked_bitmap();
1282 for (size_t i = 0; i < _voucher_activity_bitmaps_per_heap; i++) {
1283 _voucher_activity_bitmap_base_t free_bits = pressure_locked_bitmap[i];
1284 pressure_locked_bitmap[i] = 0;
1285 if (free_bits != 0) {
1286 (void)dispatch_atomic_and(&bitmap[i], ~free_bits, release);
1287 }
1288 }
1289 }
1290
1291 DISPATCH_ALWAYS_INLINE
1292 static inline void
1293 _voucher_activity_buffer_init(_voucher_activity_t act,
1294 _voucher_activity_buffer_header_t buffer, bool initial)
1295 {
1296 _voucher_activity_tracepoint_t vat = (_voucher_activity_tracepoint_t)buffer;
1297 _voucher_activity_tracepoint_init_with_id(vat, act->va_trace_id,
1298 act->va_location, !initial);
1299 buffer->vabh_flags = _voucher_activity_trace_flag_buffer_header |
1300 _voucher_activity_trace_flag_activity |
1301 (initial ? _voucher_activity_trace_flag_start : 0);
1302 buffer->vabh_activity_id = act->va_id;
1303 buffer->vabh_pos.vabp_atomic_pos = 0;
1304 buffer->vabh_pos.vabp_pos.vabp_next_tracepoint_idx = 1;
1305 }
1306
1307 static _voucher_activity_buffer_header_t
1308 _voucher_activity_buffer_alloc_slow(_voucher_activity_t act,
1309 _voucher_activity_buffer_header_t current)
1310 {
1311 _voucher_activity_buffer_header_t buffer;
1312 _voucher_activity_lock_lock(va_buffers_lock(act)); // TODO: revisit locking
1313 buffer = act->va_current_buffer;
1314 if (buffer != current) {
1315 _voucher_activity_lock_unlock(va_buffers_lock(act));
1316 return buffer;
1317 }
1318 buffer = TAILQ_FIRST(&act->va_buffers);
1319 if (buffer != TAILQ_LAST(&act->va_buffers,
1320 _voucher_activity_buffer_list_s)) {
1321 TAILQ_REMOVE(&act->va_buffers, buffer, vabh_list);
1322 TAILQ_INSERT_TAIL(&act->va_buffers, buffer, vabh_list);
1323 }
1324 _voucher_activity_lock_unlock(va_buffers_lock(act));
1325 if (_voucher_activity_buffer_is_full(buffer)) {
1326 _voucher_activity_firehose_wait(act, buffer);
1327 }
1328 if (dispatch_atomic_cmpxchgv2o(act, va_current_buffer, current, buffer,
1329 &current, release)) {
1330 if (_voucher_activity_buffer_mark_full(current)) {
1331 _voucher_activity_firehose_push(act, current);
1332 }
1333 _dispatch_voucher_activity_debug("buffer reuse %p", act, buffer);
1334 } else {
1335 buffer = current;
1336 }
1337 return buffer;
1338 }
1339
1340 static _voucher_activity_buffer_header_t
1341 _voucher_activity_buffer_alloc(_voucher_activity_t act,
1342 _voucher_activity_buffer_header_t current)
1343 {
1344 _voucher_activity_buffer_header_t buffer = NULL;
1345 if (act->va_buffer_count < act->va_buffer_limit) {
1346 buffer = _voucher_activity_heap_buffer_alloc();
1347 if (buffer && dispatch_atomic_inc2o(act, va_buffer_count, relaxed) >
1348 act->va_buffer_limit) {
1349 dispatch_atomic_dec2o(act, va_buffer_count, relaxed);
1350 _voucher_activity_heap_buffer_free(buffer);
1351 buffer = NULL;
1352 }
1353 }
1354 if (!buffer) return _voucher_activity_buffer_alloc_slow(act, current);
1355 _voucher_activity_buffer_init(act, buffer, false);
1356 if (dispatch_atomic_cmpxchgv2o(act, va_current_buffer, current, buffer,
1357 &current, release)) {
1358 _voucher_activity_lock_lock(va_buffers_lock(act));
1359 TAILQ_INSERT_TAIL(&act->va_buffers, buffer, vabh_list);
1360 _voucher_activity_lock_unlock(va_buffers_lock(act));
1361 if (_voucher_activity_buffer_mark_full(current)) {
1362 _voucher_activity_firehose_push(act, current);
1363 }
1364 _dispatch_voucher_activity_debug("buffer alloc %p", act, buffer);
1365 } else {
1366 dispatch_atomic_dec2o(act, va_buffer_count, relaxed);
1367 _voucher_activity_heap_buffer_free(buffer);
1368 buffer = current;
1369 }
1370 return buffer;
1371 }
1372
1373 #pragma mark -
1374 #pragma mark _voucher_activity_t
1375
1376 #define _voucher_activity_ordered_insert(_act, head, field) do { \
1377 typeof(_act) _vai; \
1378 TAILQ_FOREACH(_vai, (head), field) { \
1379 if (_act->va_id < _vai->va_id) break; \
1380 } \
1381 if (_vai) { \
1382 TAILQ_INSERT_BEFORE(_vai, _act, field); \
1383 } else { \
1384 TAILQ_INSERT_TAIL((head), _act, field); \
1385 } } while (0);
1386
1387 static void _voucher_activity_dispose(_voucher_activity_t act);
1388 static _voucher_atm_t _voucher_atm_copy(atm_aid_t atm_id);
1389 static inline void _voucher_atm_release(_voucher_atm_t vatm);
1390 static atm_aid_t _voucher_mach_voucher_get_atm_id(mach_voucher_t kv);
1391
1392 DISPATCH_ALWAYS_INLINE
1393 static inline bool
1394 _voucher_activity_try_retain(_voucher_activity_t act)
1395 {
1396 // not using _os_object_refcnt* because we don't need barriers:
1397 // activities are immutable and are in a hash table with a lock
1398 int use_cnt = dispatch_atomic_inc2o(act, va_refcnt, relaxed);
1399 _dispatch_voucher_activity_debug("retain -> %d", act, use_cnt + 1);
1400 if (slowpath(use_cnt < 0)) {
1401 _dispatch_voucher_activity_debug("overrelease", act);
1402 DISPATCH_CRASH("Activity overrelease");
1403 }
1404 return use_cnt > 0;
1405 }
1406
1407 DISPATCH_ALWAYS_INLINE
1408 static inline _voucher_activity_t
1409 _voucher_activity_retain(_voucher_activity_t act)
1410 {
1411 if (slowpath(!_voucher_activity_try_retain(act))) {
1412 _dispatch_voucher_activity_debug("resurrection", act);
1413 DISPATCH_CRASH("Activity resurrection");
1414 }
1415 return act;
1416 }
1417
1418 DISPATCH_ALWAYS_INLINE
1419 static inline void
1420 _voucher_activity_release(_voucher_activity_t act)
1421 {
1422 // not using _os_object_refcnt* because we don't need barriers:
1423 // activities are immutable and are in a hash table with a lock
1424 int use_cnt = dispatch_atomic_dec2o(act, va_refcnt, relaxed);
1425 _dispatch_voucher_activity_debug("release -> %d", act, use_cnt + 1);
1426 if (fastpath(use_cnt >= 0)) {
1427 return;
1428 }
1429 if (slowpath(use_cnt < -1)) {
1430 _dispatch_voucher_activity_debug("overrelease", act);
1431 DISPATCH_CRASH("Activity overrelease");
1432 }
1433 _voucher_activity_remove(act);
1434 _voucher_activity_dispose(act);
1435 }
1436
1437 static _voucher_activity_t
1438 _voucher_activity_find_and_retain(voucher_activity_id_t va_id, uint32_t hash)
1439 {
1440 // not using _os_object_refcnt* because we don't need barriers:
1441 // activities are immutable and are in a hash table with a lock
1442 //
1443 // assumes vam_activities_lock held
1444 _voucher_activity_t act;
1445 TAILQ_FOREACH(act, vam_activities(hash), va_list) {
1446 if (act->va_id == va_id) {
1447 if (fastpath(_voucher_activity_try_retain(act))) {
1448 return act;
1449 }
1450
1451 // <rdar://problem/20468375> disallow resurrection
1452 dispatch_atomic_dec2o(act, va_refcnt, relaxed);
1453 _dispatch_voucher_activity_debug("undo resurrection", act);
1454 }
1455 }
1456 return NULL;
1457 }
1458
1459 static _voucher_activity_t
1460 _voucher_activity_copy_from_id(voucher_activity_id_t va_id)
1461 {
1462 uint32_t hash = VACTID_HASH(va_id);
1463 _voucher_activity_lock_lock(vam_activities_lock());
1464 _voucher_activity_t act = _voucher_activity_find_and_retain(va_id, hash);
1465 if (act) {
1466 _dispatch_voucher_activity_debug("copy from id 0x%llx", act, va_id);
1467 }
1468 _voucher_activity_lock_unlock(vam_activities_lock());
1469 return act;
1470 }
1471
1472 static _voucher_activity_t
1473 _voucher_activity_try_insert(_voucher_activity_t act_new)
1474 {
1475 voucher_activity_id_t va_id = act_new->va_id;
1476 uint32_t hash = VACTID_HASH(va_id);
1477 _voucher_activity_lock_lock(vam_activities_lock());
1478 _voucher_activity_t act = _voucher_activity_find_and_retain(va_id, hash);
1479 if (act) {
1480 _dispatch_voucher_activity_debug("try insert: failed (%p)", act,act_new);
1481 } else {
1482 if (slowpath(_TAILQ_IS_ENQUEUED(act_new, va_list))) {
1483 _dispatch_voucher_activity_debug("corruption", act_new);
1484 DISPATCH_CRASH("Activity corruption");
1485 }
1486 TAILQ_INSERT_TAIL(vam_activities(hash), act_new, va_list);
1487 _dispatch_voucher_activity_debug("try insert: succeeded", act_new);
1488 }
1489 _voucher_activity_lock_unlock(vam_activities_lock());
1490 return act;
1491 }
1492
1493 static void
1494 _voucher_activity_remove(_voucher_activity_t act)
1495 {
1496 voucher_activity_id_t va_id = act->va_id;
1497 uint32_t hash = VACTID_HASH(va_id);
1498
1499 _voucher_activity_lock_lock(vam_activities_lock());
1500 if (slowpath(!va_id || !_TAILQ_IS_ENQUEUED(act, va_list))) {
1501 _dispatch_voucher_activity_debug("corruption", act);
1502 DISPATCH_CRASH("Activity corruption");
1503 }
1504 TAILQ_REMOVE(vam_activities(hash), act, va_list);
1505 _TAILQ_MARK_NOT_ENQUEUED(act, va_list);
1506 act->va_list.tqe_next = (void*)~0ull;
1507 _dispatch_voucher_activity_debug("remove", act);
1508 _voucher_activity_lock_unlock(vam_activities_lock());
1509 }
1510
1511 static _voucher_activity_t
1512 _voucher_activity_create_with_atm(_voucher_atm_t vatm,
1513 voucher_activity_id_t va_id, voucher_activity_trace_id_t trace_id,
1514 uint64_t location, _voucher_activity_buffer_header_t buffer)
1515 {
1516 if (!buffer) buffer = _voucher_activity_heap_buffer_alloc();
1517 if (!buffer) {
1518 _dispatch_voucher_atm_debug("no buffer", vatm);
1519 _voucher_atm_release(vatm); // consume vatm reference
1520 return NULL;
1521 }
1522 _voucher_activity_t act = _dispatch_calloc(1ul,
1523 sizeof(struct _voucher_activity_s));
1524 act->va_id = va_id;
1525 act->va_trace_id = trace_id ? trace_id : _voucher_activity_trace_id_release;
1526 act->va_location = location;
1527 act->va_buffer_limit = _voucher_activity_buffer_limit();
1528 TAILQ_INIT(&act->va_buffers);
1529 act->va_current_buffer = buffer;
1530 act->va_atm = vatm; // transfer vatm reference
1531 _voucher_activity_lock_init(va_buffers_lock(act));
1532 if (dispatch_assume_zero(pthread_mutex_init(&act->va_mutex, NULL)) ||
1533 dispatch_assume_zero(pthread_cond_init(&act->va_cond, NULL))) {
1534 DISPATCH_CLIENT_CRASH("Could not initialize activity");
1535 }
1536 _TAILQ_MARK_NOT_ENQUEUED(act, va_list);
1537 _TAILQ_MARK_NOT_ENQUEUED(act, va_atm_list);
1538 _TAILQ_MARK_NOT_ENQUEUED(act, va_atm_used_list);
1539
1540 _voucher_activity_buffer_init(act, buffer, true);
1541 TAILQ_INSERT_TAIL(&act->va_buffers, buffer, vabh_list);
1542 _voucher_activity_t actx = _voucher_activity_try_insert(act);
1543 if (actx) {
1544 _voucher_activity_dispose(act);
1545 act = actx;
1546 }
1547 _dispatch_voucher_activity_debug("create", act);
1548 return act;
1549 }
1550
1551 static void
1552 _voucher_activity_dispose(_voucher_activity_t act)
1553 {
1554 _dispatch_voucher_activity_debug("dispose", act);
1555 _voucher_atm_release(act->va_atm);
1556 if (slowpath(_TAILQ_IS_ENQUEUED(act, va_list))) {
1557 _dispatch_voucher_activity_debug("corruption", act);
1558 DISPATCH_CRASH("Activity corruption");
1559 }
1560 act->va_list.tqe_next = DISPATCH_OBJECT_LISTLESS;
1561 dispatch_assert(!_TAILQ_IS_ENQUEUED(act, va_atm_list));
1562 dispatch_assert(!_TAILQ_IS_ENQUEUED(act, va_atm_used_list));
1563 _voucher_activity_buffer_header_t buffer, tmp;
1564 TAILQ_FOREACH_SAFE(buffer, &act->va_buffers, vabh_list, tmp) {
1565 if (buffer->vabh_pos.vabp_pos.vabp_next_tracepoint_idx > 1) {
1566 dispatch_assert(_voucher_activity_buffer_mark_full(buffer));
1567 _voucher_activity_firehose_push(act, buffer);
1568 }
1569 TAILQ_REMOVE(&act->va_buffers, buffer, vabh_list);
1570 _dispatch_voucher_activity_debug("buffer free %p", act, buffer);
1571 _voucher_activity_heap_buffer_free(buffer);
1572 }
1573 (void)dispatch_assume_zero(pthread_mutex_destroy(&act->va_mutex));
1574 (void)dispatch_assume_zero(pthread_cond_destroy(&act->va_cond));
1575 free(act);
1576 }
1577
1578 DISPATCH_NOINLINE
1579 void
1580 _voucher_activity_firehose_push(_voucher_activity_t act,
1581 _voucher_activity_buffer_header_t buffer)
1582 {
1583 if (dispatch_assume_zero(pthread_mutex_lock(&act->va_mutex))) {
1584 DISPATCH_CLIENT_CRASH("Activity corruption: mutex_lock");
1585 }
1586 _dispatch_voucher_activity_debug("firehose push %p", act, buffer);
1587 // TODO: call firehose_push
1588 VOUCHER_ACTIVITY_BUFFER_HOOK_CALLOUT(full, buffer);
1589 _voucher_activity_buffer_init(act, buffer, false);
1590 if (dispatch_assume_zero(pthread_cond_broadcast(&act->va_cond))) {
1591 DISPATCH_CLIENT_CRASH("Activity corruption: cond_broadcast");
1592 }
1593 if (dispatch_assume_zero(pthread_mutex_unlock(&act->va_mutex))) {
1594 DISPATCH_CLIENT_CRASH("Activity corruption: mutex_unlock");
1595 }
1596 }
1597
1598 DISPATCH_NOINLINE
1599 static void
1600 _voucher_activity_firehose_wait(_voucher_activity_t act,
1601 _voucher_activity_buffer_header_t buffer)
1602 {
1603 if (dispatch_assume_zero(pthread_mutex_lock(&act->va_mutex))) {
1604 DISPATCH_CLIENT_CRASH("Activity corruption: mutex_lock");
1605 }
1606 while (_voucher_activity_buffer_is_full(buffer)) {
1607 _dispatch_voucher_activity_debug("firehose wait %p", act, buffer);
1608 if (dispatch_assume_zero(pthread_cond_wait(&act->va_cond,
1609 &act->va_mutex))){
1610 DISPATCH_CLIENT_CRASH("Activity corruption: cond_wait");
1611 }
1612 }
1613 if (dispatch_assume_zero(pthread_mutex_unlock(&act->va_mutex))) {
1614 DISPATCH_CLIENT_CRASH("Activity corruption: mutex_unlock");
1615 }
1616 }
1617
1618 static _voucher_activity_t
1619 _voucher_activity_copy_from_mach_voucher(mach_voucher_t kv,
1620 voucher_activity_id_t va_id)
1621 {
1622 dispatch_once_f(&_voucher_activity_heap_pred, NULL,
1623 _voucher_activity_heap_init);
1624 if (_voucher_activity_disabled()) return NULL;
1625 _voucher_activity_t act = NULL;
1626 if (dispatch_assume(va_id)) {
1627 if ((act = _voucher_activity_copy_from_id(va_id))) return act;
1628 }
1629 atm_aid_t atm_id = _voucher_mach_voucher_get_atm_id(kv);
1630 if (!dispatch_assume(atm_id)) return NULL;
1631 _voucher_activity_buffer_header_t buffer;
1632 buffer = _voucher_activity_heap_buffer_alloc();
1633 if (!buffer) return NULL;
1634 _dispatch_kvoucher_debug("atm copy/create from <%lld>", kv, atm_id);
1635 _voucher_atm_t vatm = _voucher_atm_copy(atm_id);
1636 if (!vatm) vatm = _voucher_atm_create(kv, atm_id);
1637 if (!vatm) {
1638 _voucher_activity_heap_buffer_free(buffer);
1639 return NULL;
1640 }
1641 // consumes vatm reference:
1642 act = _voucher_activity_create_with_atm(vatm, va_id, 0, 0, buffer);
1643 _dispatch_voucher_activity_debug("copy from kvoucher[0x%08x]", act, kv);
1644 return act;
1645 }
1646
1647 #pragma mark -
1648 #pragma mark _voucher_atm_t
1649
1650 static void _voucher_atm_remove(_voucher_atm_t vatm);
1651 static void _voucher_atm_dispose(_voucher_atm_t vatm, bool unregister);
1652
1653 DISPATCH_ALWAYS_INLINE
1654 static inline bool
1655 _voucher_atm_try_retain(_voucher_atm_t vatm)
1656 {
1657 // not using _os_object_refcnt* because we don't need barriers:
1658 // vouchers atm are immutable and are in a hash table with a lock
1659 //
1660 // assumes vam_atms_lock held
1661 int refcnt = dispatch_atomic_inc2o(vatm, vatm_refcnt, relaxed);
1662 _dispatch_voucher_atm_debug("retain -> %d", vatm, refcnt + 1);
1663 if (slowpath(refcnt < 0)) {
1664 _dispatch_voucher_atm_debug("overrelease", vatm);
1665 DISPATCH_CRASH("ATM overrelease");
1666 }
1667 return refcnt > 0;
1668 }
1669
1670 DISPATCH_ALWAYS_INLINE
1671 static inline _voucher_atm_t
1672 _voucher_atm_retain(_voucher_atm_t vatm)
1673 {
1674 if (slowpath(!_voucher_atm_try_retain(vatm))) {
1675 _dispatch_voucher_atm_debug("resurrection", vatm);
1676 DISPATCH_CRASH("ATM resurrection");
1677 }
1678 return vatm;
1679 }
1680
1681 DISPATCH_ALWAYS_INLINE
1682 static inline void
1683 _voucher_atm_release(_voucher_atm_t vatm)
1684 {
1685 // not using _os_object_refcnt* because we don't need barriers:
1686 // vouchers atm are immutable are into a hash table with a lock
1687 int refcnt = dispatch_atomic_dec2o(vatm, vatm_refcnt, relaxed);
1688 _dispatch_voucher_atm_debug("release -> %d", vatm, refcnt + 1);
1689 if (fastpath(refcnt >= 0)) {
1690 return;
1691 }
1692 if (slowpath(refcnt < -1)) {
1693 _dispatch_voucher_atm_debug("overrelease", vatm);
1694 DISPATCH_CRASH("ATM overrelease");
1695 }
1696 _voucher_atm_remove(vatm);
1697 _voucher_atm_dispose(vatm, true);
1698 }
1699
1700 static _voucher_atm_t
1701 _voucher_atm_find_and_retain(atm_aid_t atm_id, uint32_t hash)
1702 {
1703 // not using _os_object_refcnt* because we don't need barriers:
1704 // vouchers atm are immutable are into a hash table with a lock
1705 //
1706 // assumes vam_atms_lock held
1707 _voucher_atm_t vatm;
1708 TAILQ_FOREACH(vatm, vam_atms(hash), vatm_list){
1709 if (vatm->vatm_id == atm_id) {
1710 if (fastpath(_voucher_atm_try_retain(vatm))) {
1711 return vatm;
1712 }
1713
1714 // <rdar://problem/20468375> disallow resurrection
1715 dispatch_atomic_dec2o(vatm, vatm_refcnt, relaxed);
1716 _dispatch_voucher_atm_debug("undo resurrection", vatm);
1717 }
1718 }
1719 return NULL;
1720 }
1721
1722 static _voucher_atm_t
1723 _voucher_atm_copy(atm_aid_t atm_id)
1724 {
1725 uint32_t hash = VATMID_HASH(atm_id);
1726 _voucher_activity_lock_lock(vam_atms_lock());
1727 _voucher_atm_t vatm = _voucher_atm_find_and_retain(atm_id, hash);
1728 if (vatm) {
1729 _dispatch_voucher_atm_debug("copy", vatm);
1730 }
1731 _voucher_activity_lock_unlock(vam_atms_lock());
1732 return vatm;
1733 }
1734
1735 static _voucher_atm_t
1736 _voucher_atm_try_insert(_voucher_atm_t vatm_new)
1737 {
1738 atm_aid_t atm_id = vatm_new->vatm_id;
1739 uint32_t hash = VATMID_HASH(atm_id);
1740 _voucher_activity_lock_lock(vam_atms_lock());
1741 _voucher_atm_t vatm = _voucher_atm_find_and_retain(atm_id, hash);
1742 if (vatm) {
1743 _dispatch_voucher_atm_debug("try insert: failed (%p)", vatm, vatm_new);
1744 } else {
1745 if (slowpath(_TAILQ_IS_ENQUEUED(vatm_new, vatm_list))) {
1746 _dispatch_voucher_atm_debug("corruption", vatm_new);
1747 DISPATCH_CRASH("ATM corruption");
1748 }
1749 TAILQ_INSERT_TAIL(vam_atms(hash), vatm_new, vatm_list);
1750 _dispatch_voucher_atm_debug("try insert: succeeded", vatm_new);
1751 }
1752 _voucher_activity_lock_unlock(vam_atms_lock());
1753 return vatm;
1754 }
1755
1756 static void
1757 _voucher_atm_remove(_voucher_atm_t vatm)
1758 {
1759 atm_aid_t atm_id = vatm->vatm_id;
1760 uint32_t hash = VATMID_HASH(atm_id);
1761
1762 _voucher_activity_lock_lock(vam_atms_lock());
1763 if (slowpath(!atm_id || !_TAILQ_IS_ENQUEUED(vatm, vatm_list))) {
1764 _dispatch_voucher_atm_debug("corruption", vatm);
1765 DISPATCH_CRASH("ATM corruption");
1766 }
1767 TAILQ_REMOVE(vam_atms(hash), vatm, vatm_list);
1768 _TAILQ_MARK_NOT_ENQUEUED(vatm, vatm_list);
1769 vatm->vatm_list.tqe_next = (void*)~0ull;
1770 _dispatch_voucher_atm_debug("remove", vatm);
1771 _voucher_activity_lock_unlock(vam_atms_lock());
1772 }
1773
1774 DISPATCH_NOINLINE
1775 static void
1776 _voucher_atm_fault(mach_voucher_attr_command_t kvc_cmd)
1777 {
1778 mach_voucher_t kv = _voucher_get_atm_mach_voucher(_voucher_get());
1779 if (!kv) return;
1780
1781 mach_atm_subaid_t subaid = 0;
1782 voucher_t v = _voucher_get();
1783 if (v) {
1784 unsigned int activities = v->v_activities;
1785 voucher_activity_id_t *activity_ids = _voucher_activity_ids(v);
1786 if (activities) {
1787 subaid = activity_ids[0];
1788 }
1789 }
1790
1791 kern_return_t kr;
1792 mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&subaid;
1793 mach_voucher_attr_content_size_t kvc_in_size = sizeof(mach_atm_subaid_t);
1794 mach_voucher_attr_content_t kvc_out = (mach_voucher_attr_content_t)&subaid;
1795 mach_voucher_attr_content_size_t kvc_out_size = sizeof(mach_atm_subaid_t);
1796 kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM,
1797 kvc_cmd, kvc_in, kvc_in_size, kvc_out, &kvc_out_size);
1798 DISPATCH_VERIFY_MIG(kr);
1799 (void)dispatch_assume_zero(kr);
1800 }
1801
1802 static atm_aid_t
1803 _voucher_mach_voucher_get_atm_id(mach_voucher_t kv)
1804 {
1805 kern_return_t kr;
1806 atm_aid_t atm_id = 0;
1807 mach_voucher_attr_content_t kvc = (mach_voucher_attr_content_t)&atm_id;
1808 mach_voucher_attr_content_size_t kvc_size = sizeof(atm_id);
1809 kr = mach_voucher_extract_attr_content(kv, MACH_VOUCHER_ATTR_KEY_ATM, kvc,
1810 &kvc_size);
1811 DISPATCH_VERIFY_MIG(kr);
1812 (void)dispatch_assume_zero(kr);
1813 return atm_id;
1814 }
1815
1816 static mach_voucher_t
1817 _voucher_atm_mach_voucher_create(atm_aid_t *atm_id_ptr)
1818 {
1819 kern_return_t kr;
1820 mach_voucher_t kv;
1821 static const mach_voucher_attr_recipe_data_t atm_create_recipe = {
1822 .key = MACH_VOUCHER_ATTR_KEY_ATM,
1823 .command = MACH_VOUCHER_ATTR_ATM_CREATE,
1824 };
1825 kr = _voucher_create_mach_voucher(&atm_create_recipe,
1826 sizeof(atm_create_recipe), &kv);
1827 if (dispatch_assume_zero(kr)) {
1828 DISPATCH_CLIENT_CRASH("Could not create ATM mach voucher");
1829 }
1830 atm_aid_t atm_id = _voucher_mach_voucher_get_atm_id(kv);
1831 if (!dispatch_assume(atm_id)) {
1832 DISPATCH_CLIENT_CRASH("Could not extract ATM ID");
1833 }
1834 _dispatch_kvoucher_debug("atm create <%lld>", kv, atm_id);
1835 *atm_id_ptr = atm_id;
1836 return kv;
1837 }
1838
1839 static mach_voucher_t
1840 _voucher_atm_mach_voucher_copy(mach_voucher_t akv)
1841 {
1842 kern_return_t kr;
1843 mach_voucher_t kv;
1844 const mach_voucher_attr_recipe_data_t atm_copy_recipe = {
1845 .key = MACH_VOUCHER_ATTR_KEY_ATM,
1846 .command = MACH_VOUCHER_ATTR_COPY,
1847 .previous_voucher = akv,
1848 };
1849 kr = _voucher_create_mach_voucher(&atm_copy_recipe,
1850 sizeof(atm_copy_recipe), &kv);
1851 if (dispatch_assume_zero(kr)) {
1852 DISPATCH_CLIENT_CRASH("Could not copy ATM mach voucher");
1853 }
1854 _dispatch_kvoucher_debug("copy atm voucher from [0x%08x]", kv, akv);
1855 return kv;
1856 }
1857
1858 static void
1859 _voucher_atm_register(_voucher_atm_t vatm)
1860 {
1861 mach_voucher_t kv = vatm->vatm_kvoucher;
1862 if (!kv) return;
1863 kern_return_t kr;
1864 atm_guard_t gen =
1865 dispatch_atomic_inc(&_voucher_atm_generation, relaxed);
1866 _dispatch_voucher_atm_debug("atm register %lld", vatm, gen);
1867 mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&gen;
1868 mach_voucher_attr_content_size_t kvc_in_size = sizeof(gen);
1869 mach_voucher_attr_content_t kvc_out = NULL;
1870 mach_voucher_attr_content_size_t kvc_out_size = 0;
1871 kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM,
1872 ATM_ACTION_REGISTER, kvc_in, kvc_in_size, kvc_out,
1873 &kvc_out_size);
1874 DISPATCH_VERIFY_MIG(kr);
1875 if (kr) {
1876 DISPATCH_CLIENT_CRASH("Could not register ATM ID");
1877 }
1878 vatm->vatm_generation = gen;
1879 _dispatch_voucher_atm_debug("atm registered %lld", vatm,
1880 vatm->vatm_generation);
1881 }
1882
1883 static void
1884 _voucher_atm_unregister(_voucher_atm_t vatm)
1885 {
1886 _dispatch_voucher_atm_debug("atm unregister %lld", vatm,
1887 vatm->vatm_generation);
1888 mach_voucher_t kv = vatm->vatm_kvoucher;
1889 dispatch_assert(kv);
1890 kern_return_t kr;
1891 atm_guard_t gen = vatm->vatm_generation;
1892 mach_voucher_attr_content_t kvc_in = (mach_voucher_attr_content_t)&gen;
1893 mach_voucher_attr_content_size_t kvc_in_size = sizeof(gen);
1894 mach_voucher_attr_content_t kvc_out = NULL;
1895 mach_voucher_attr_content_size_t kvc_out_size = 0;
1896 kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM,
1897 ATM_ACTION_UNREGISTER, kvc_in, kvc_in_size, kvc_out, &kvc_out_size);
1898 DISPATCH_VERIFY_MIG(kr);
1899 if (kr && kr != KERN_INVALID_VALUE) {
1900 (void)dispatch_assume_zero(kr);
1901 }
1902 _dispatch_voucher_atm_debug("atm unregistered %lld", vatm,
1903 vatm->vatm_generation);
1904 }
1905
1906 static _voucher_atm_t
1907 _voucher_atm_create(mach_voucher_t kv, atm_aid_t atm_id)
1908 {
1909 _voucher_atm_t vatm = _dispatch_calloc(1ul, sizeof(struct _voucher_atm_s));
1910 kv = kv ? _voucher_atm_mach_voucher_copy(kv) :
1911 _voucher_atm_mach_voucher_create(&atm_id);
1912 vatm->vatm_kvoucher = kv;
1913 vatm->vatm_id = atm_id;
1914 _voucher_atm_t vatmx = _voucher_atm_try_insert(vatm);
1915 if (vatmx) {
1916 _voucher_atm_dispose(vatm, false);
1917 vatm = vatmx;
1918 } else {
1919 _voucher_atm_register(vatm);
1920 }
1921 _dispatch_voucher_atm_debug("create with kvoucher[0x%08x]", vatm, kv);
1922 return vatm;
1923 }
1924
1925 static void
1926 _voucher_atm_dispose(_voucher_atm_t vatm, bool unregister)
1927 {
1928 _dispatch_voucher_atm_debug("dispose", vatm);
1929 if (slowpath(_TAILQ_IS_ENQUEUED(vatm, vatm_list))) {
1930 _dispatch_voucher_atm_debug("corruption", vatm);
1931 DISPATCH_CRASH("ATM corruption");
1932 }
1933 vatm->vatm_list.tqe_next = DISPATCH_OBJECT_LISTLESS;
1934 if (vatm->vatm_kvoucher) {
1935 if (unregister) _voucher_atm_unregister(vatm);
1936 _voucher_dealloc_mach_voucher(vatm->vatm_kvoucher);
1937 vatm->vatm_kvoucher = MACH_VOUCHER_NULL;
1938 }
1939 free(vatm);
1940 }
1941
1942 DISPATCH_NOINLINE
1943 static voucher_activity_id_t
1944 _voucher_atm_subid_make(_voucher_atm_t vatm, voucher_activity_flag_t flags)
1945 {
1946 mach_voucher_t kv = vatm->vatm_kvoucher;
1947 _dispatch_voucher_atm_debug("create subid from atm", vatm);
1948 kern_return_t kr;
1949 mach_atm_subaid_t naid;
1950 mach_voucher_attr_content_t kvc_in = NULL;
1951 mach_voucher_attr_content_size_t kvc_in_size = 0;
1952 mach_voucher_attr_content_t kvc_out = (mach_voucher_attr_content_t)&naid;
1953 mach_voucher_attr_content_size_t kvc_out_size = sizeof(naid);
1954 kr = mach_voucher_attr_command(kv, MACH_VOUCHER_ATTR_KEY_ATM,
1955 ATM_ACTION_GETSUBAID, kvc_in, kvc_in_size, kvc_out, &kvc_out_size);
1956 DISPATCH_VERIFY_MIG(kr);
1957 if (dispatch_assume_zero(kr)) {
1958 DISPATCH_CLIENT_CRASH("Could not get next ATM ID");
1959 }
1960 _dispatch_voucher_atm_debug("created subid from atm %lld", vatm, naid);
1961 return VATMID2ACTID(naid, flags);
1962 }
1963
1964 #pragma mark -
1965 #pragma mark voucher_activity_id_t
1966
1967 static const size_t _voucher_activity_maxsize =
1968 _voucher_activity_buffer_size - _voucher_activity_buffer_header_size -
1969 _voucher_activity_strings_header_size;
1970
1971 voucher_activity_id_t
1972 voucher_activity_start_with_location(voucher_activity_trace_id_t trace_id,
1973 uint64_t location, voucher_activity_flag_t flags)
1974 {
1975 dispatch_once_f(&_voucher_activity_heap_pred, NULL,
1976 _voucher_activity_heap_init);
1977 if (!_voucher_activity_trace_id_enabled(trace_id)) return 0;
1978 voucher_activity_id_t va_id = 0;
1979 _voucher_atm_t vatm = NULL;
1980 _voucher_activity_t act = NULL;
1981 _voucher_activity_tracepoint_t vat = NULL;
1982 unsigned int activities = 1, oactivities = 0;
1983 voucher_t ov = _voucher_get();
1984 vatm = _voucher_get_atm(ov);
1985 if (!(flags & voucher_activity_flag_force) && ov && ov->v_activities) {
1986 oactivities = ov->v_activities;
1987 activities += oactivities;
1988 if (activities > _voucher_max_activities) {
1989 va_id = _voucher_atm_subid_make(vatm, flags);
1990 goto out;
1991 }
1992 }
1993 _voucher_atm_retain(vatm);
1994 // required for v->v_atm = vatm below
1995 _voucher_atm_retain(vatm);
1996 va_id = _voucher_atm_subid_make(vatm, flags);
1997 // consumes vatm reference:
1998 act = _voucher_activity_create_with_atm(vatm, va_id, trace_id, location,
1999 NULL);
2000 vat = (_voucher_activity_tracepoint_t)act;
2001 pthread_priority_t priority = _voucher_get_priority(ov);
2002 mach_voucher_attr_recipe_size_t extra = ov ? _voucher_extra_size(ov) : 0;
2003 voucher_t v = _voucher_alloc(activities, priority, extra);
2004 if (extra) {
2005 memcpy(_voucher_extra_recipes(v), _voucher_extra_recipes(ov), extra);
2006 }
2007 if (ov && ov->v_kvoucher) {
2008 voucher_t kvb = ov->v_kvbase ? ov->v_kvbase : ov;
2009 v->v_kvbase = _voucher_retain(kvb);
2010 v->v_kvoucher = kvb->v_kvoucher;
2011 }
2012 voucher_activity_id_t *activity_ids = _voucher_activity_ids(v);
2013 if (oactivities) {
2014 memcpy(activity_ids, _voucher_activity_ids(ov),
2015 oactivities * sizeof(voucher_activity_id_t));
2016 }
2017 activity_ids[activities-1] = va_id;
2018 v->v_atm = vatm;
2019 v->v_activity = act;
2020 _voucher_swap(ov, v);
2021 return va_id; // new activity buffer contains trace info
2022 out:
2023 _voucher_activity_trace_activity_event(trace_id, va_id, start);
2024 return va_id;
2025 }
2026
2027 voucher_activity_id_t
2028 voucher_activity_start(voucher_activity_trace_id_t trace_id,
2029 voucher_activity_flag_t flags)
2030 {
2031 return voucher_activity_start_with_location(trace_id, 0, flags);
2032 }
2033
2034 void
2035 voucher_activity_end(voucher_activity_id_t va_id)
2036 {
2037 if (!va_id) return;
2038 _voucher_activity_trace_activity_event(_voucher_activity_trace_id_release,
2039 va_id, end);
2040 voucher_t v = _voucher_get();
2041 if (!v) return;
2042 unsigned int activities = v->v_activities, act_idx = activities;
2043 voucher_activity_id_t *activity_ids = _voucher_activity_ids(v);
2044 while (act_idx) {
2045 if (activity_ids[act_idx-1] == va_id) break;
2046 act_idx--;
2047 }
2048 if (!act_idx) return; // activity_id not found
2049 pthread_priority_t priority = _voucher_get_priority(v);
2050 mach_voucher_attr_recipe_size_t extra = _voucher_extra_size(v);
2051 voucher_t nv = NULL;
2052 if (act_idx > 1 || activities == 1) --activities;
2053 if (priority || activities || extra || v->v_kvoucher) {
2054 nv = _voucher_alloc(activities, priority, extra);
2055 if (extra) {
2056 memcpy(_voucher_extra_recipes(nv), _voucher_extra_recipes(v),extra);
2057 }
2058 }
2059 if (v->v_kvoucher) {
2060 voucher_t kvb = v->v_kvbase ? v->v_kvbase : v;
2061 nv->v_kvbase = _voucher_retain(kvb);
2062 nv->v_kvoucher = kvb->v_kvoucher;
2063 }
2064 bool atm_collect = !activities;
2065 if (activities) {
2066 voucher_activity_id_t *new_activity_ids = _voucher_activity_ids(nv);
2067 if (act_idx == 1 && _voucher_activity_default) {
2068 atm_collect = true;
2069 // default to _voucher_activity_default base activity
2070 new_activity_ids[0] = _voucher_activity_default->va_id;
2071 memcpy(&new_activity_ids[1], &activity_ids[1],
2072 (activities - 1) * sizeof(voucher_activity_id_t));
2073 } else {
2074 if (v->v_activity) {
2075 nv->v_activity = _voucher_activity_retain(v->v_activity);
2076 nv->v_atm = _voucher_atm_retain(v->v_atm);
2077 }
2078 memcpy(new_activity_ids, activity_ids,
2079 --act_idx * sizeof(voucher_activity_id_t));
2080 if (act_idx < activities) {
2081 memcpy(&new_activity_ids[act_idx], &activity_ids[act_idx+1],
2082 (activities - act_idx) * sizeof(voucher_activity_id_t));
2083 }
2084 }
2085 }
2086 _voucher_swap(v, nv);
2087 }
2088
2089 unsigned int
2090 voucher_get_activities(voucher_activity_id_t *entries, unsigned int *count)
2091 {
2092 voucher_t v = _voucher_get();
2093 if (!v || !count) return 0;
2094 unsigned int activities = v->v_activities;
2095 if (*count < activities) activities = *count;
2096 *count = v->v_activities;
2097 voucher_activity_id_t *activity_ids = _voucher_activity_ids(v);
2098 if (activities && entries) {
2099 memcpy(entries, activity_ids, activities *
2100 sizeof(voucher_activity_id_t));
2101 }
2102 return activities;
2103 }
2104
2105 uint8_t
2106 voucher_activity_get_namespace(void)
2107 {
2108 voucher_t v = _voucher_get();
2109 if (!v || !v->v_activity) return 0;
2110 voucher_activity_trace_id_t trace_id = v->v_activity->va_trace_id;
2111 uint8_t cns = (uint8_t)(trace_id >>
2112 _voucher_activity_trace_id_code_namespace_shift);
2113 return cns;
2114 }
2115
2116 DISPATCH_NOINLINE
2117 _voucher_activity_tracepoint_t
2118 _voucher_activity_buffer_tracepoint_acquire_slow(_voucher_activity_t *vap,
2119 _voucher_activity_buffer_header_t *vabp, unsigned int slots,
2120 size_t strsize, uint16_t *stroffsetp)
2121 {
2122 _voucher_activity_t act;
2123 _voucher_activity_buffer_header_t vab;
2124 _voucher_activity_tracepoint_t vat = NULL;
2125 voucher_t v = _voucher_get();
2126 if (v && v->v_activity) {
2127 act = v->v_activity;
2128 } else {
2129 dispatch_once_f(&_voucher_activity_heap_pred, NULL,
2130 _voucher_activity_heap_init);
2131 if (_voucher_activity_disabled()) return NULL;
2132 act = _voucher_activity_default;
2133 }
2134 vab = act->va_current_buffer;
2135 if (act == *vap && vab != *vabp) {
2136 goto retry; // another slowpath raced us
2137 }
2138 do {
2139 vab = _voucher_activity_buffer_alloc(act, vab);
2140 if (!vab) break;
2141 retry:
2142 vat = _voucher_activity_buffer_tracepoint_acquire(vab, slots, strsize,
2143 stroffsetp);
2144 } while (!vat);
2145 *vap = act;
2146 *vabp = vab;
2147 return vat;
2148 }
2149
2150 static inline void
2151 _voucher_activity_trace_fault(voucher_activity_trace_id_t trace_id)
2152 {
2153 if (!slowpath(_voucher_activity_trace_id_is_subtype(trace_id, error))) {
2154 return;
2155 }
2156 mach_voucher_attr_command_t atm_cmd = ATM_ACTION_COLLECT;
2157 if (_voucher_activity_trace_id_is_subtype(trace_id, fault)) {
2158 atm_cmd = ATM_ACTION_LOGFAIL;
2159 }
2160 return _voucher_atm_fault(atm_cmd);
2161 }
2162
2163 uint64_t
2164 voucher_activity_trace(voucher_activity_trace_id_t trace_id, uint64_t location,
2165 void *buffer, size_t length)
2166 {
2167 if (!_voucher_activity_trace_id_enabled(trace_id)) return 0;
2168 _voucher_activity_t act;
2169 _voucher_activity_buffer_header_t vab;
2170 _voucher_activity_tracepoint_t vat;
2171 const unsigned int slots = length <= sizeof(vat->vat_data) ? 1 : 2;
2172 act = _voucher_activity_get();
2173 vab = _voucher_activity_buffer_get_from_activity(act);
2174 vat = _voucher_activity_buffer_tracepoint_acquire(vab, slots, 0, NULL);
2175 if (!vat) {
2176 vat = _voucher_activity_buffer_tracepoint_acquire_slow(&act, &vab,
2177 slots, 0, NULL);
2178 }
2179 if (!vat) return 0;
2180 uint64_t timestamp = _voucher_activity_tracepoint_init_with_id(vat,
2181 trace_id, location, true);
2182 void *tbuf = vat->vat_data;
2183 size_t tlen = sizeof(vat->vat_data);
2184 if (length < tlen) {
2185 memcpy(tbuf, buffer, length);
2186 } else {
2187 memcpy(tbuf, buffer, tlen);
2188 }
2189 if (length > tlen) {
2190 vat->vat_flags |= _voucher_activity_trace_flag_wide_first;
2191 buffer += tlen;
2192 length -= tlen;
2193 (++vat)->vat_flags = _voucher_activity_trace_flag_tracepoint |
2194 _voucher_activity_trace_flag_wide_second;
2195 vat->vat_type = 0; vat->vat_namespace = 0;
2196 tbuf = (void*)vat + offsetof(typeof(*vat), vat_code);
2197 tlen = sizeof(*vat) - offsetof(typeof(*vat), vat_code);
2198 if (length < tlen) {
2199 memcpy(tbuf, buffer, length);
2200 } else {
2201 memcpy(tbuf, buffer, tlen);
2202 }
2203 }
2204 _voucher_activity_trace_fault(trace_id);
2205 if (_voucher_activity_buffer_tracepoint_release(vab)) {
2206 _voucher_activity_firehose_push(act, vab);
2207 }
2208 return timestamp;
2209 }
2210
2211 uint64_t
2212 voucher_activity_trace_strings(voucher_activity_trace_id_t trace_id,
2213 uint64_t location, void *buffer, size_t length, const char *strings[],
2214 size_t string_lengths[], size_t strings_size)
2215 {
2216 if (!_voucher_activity_trace_id_enabled(trace_id)) return 0;
2217 _voucher_activity_t act;
2218 _voucher_activity_buffer_header_t vab;
2219 _voucher_activity_tracepoint_t vat;
2220 uint16_t offset;
2221 const unsigned int slots = length <= sizeof(vat->vat_data) ? 1 : 2;
2222 strings_size = MIN(strings_size, _voucher_activity_maxsize -
2223 slots * sizeof(struct _voucher_activity_tracepoint_s));
2224 act = _voucher_activity_get();
2225 vab = _voucher_activity_buffer_get_from_activity(act);
2226 vat = _voucher_activity_buffer_tracepoint_acquire(vab, slots, strings_size,
2227 &offset);
2228 if (!vat) {
2229 vat = _voucher_activity_buffer_tracepoint_acquire_slow(&act, &vab,
2230 slots, strings_size, &offset);
2231 }
2232 if (!vat) return 0;
2233 uint64_t timestamp = _voucher_activity_tracepoint_init_with_id(vat,
2234 trace_id, location, false);
2235 vat->vat_flags |= _voucher_activity_trace_flag_tracepoint_strings;
2236 vat->vat_stroff.vats_offset = offset;
2237 void *tbuf = vat->vat_stroff.vats_data;
2238 size_t tlen = sizeof(vat->vat_stroff.vats_data);
2239 if (length < tlen) {
2240 memcpy(tbuf, buffer, length);
2241 } else {
2242 memcpy(tbuf, buffer, tlen);
2243 }
2244 if (length > tlen) {
2245 vat->vat_flags |= _voucher_activity_trace_flag_wide_first;
2246 buffer += tlen;
2247 length -= tlen;
2248 (++vat)->vat_flags = _voucher_activity_trace_flag_tracepoint |
2249 _voucher_activity_trace_flag_wide_second;
2250 vat->vat_type = 0; vat->vat_namespace = 0;
2251 tbuf = (void*)vat + offsetof(typeof(*vat), vat_code);
2252 tlen = sizeof(*vat) - offsetof(typeof(*vat), vat_code);
2253 if (length < tlen) {
2254 memcpy(tbuf, buffer, length);
2255 } else {
2256 memcpy(tbuf, buffer, tlen);
2257 }
2258 }
2259 const uint16_t offsetend = offset - (uint16_t)strings_size;
2260 char *b = (char*)vab + _voucher_activity_buffer_size;
2261 int i = 0;
2262 while (offset > offsetend && strings[i]) {
2263 size_t maxsize = MIN(string_lengths[i] + 1, offset - offsetend);
2264 size_t len = strlcpy(b - offset, strings[i++], maxsize);
2265 offset -= MIN(len + 1, maxsize);
2266 }
2267 _voucher_activity_trace_fault(trace_id);
2268 if (_voucher_activity_buffer_tracepoint_release(vab)) {
2269 _voucher_activity_firehose_push(act, vab);
2270 }
2271 return timestamp;
2272 }
2273
2274 uint64_t
2275 voucher_activity_trace_args(voucher_activity_trace_id_t trace_id,
2276 uint64_t location, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3,
2277 uintptr_t arg4)
2278 {
2279 if (!_voucher_activity_trace_id_enabled(trace_id)) return 0;
2280 _voucher_activity_t act;
2281 _voucher_activity_buffer_header_t vab;
2282 _voucher_activity_tracepoint_t vat;
2283 act = _voucher_activity_get();
2284 vab = _voucher_activity_buffer_get_from_activity(act);
2285 vat = _voucher_activity_buffer_tracepoint_acquire(vab, 1, 0, NULL);
2286 if (!vat) {
2287 vat = _voucher_activity_buffer_tracepoint_acquire_slow(&act, &vab, 1,
2288 0, NULL);
2289 }
2290 if (!vat) return 0;
2291 uint64_t timestamp = _voucher_activity_tracepoint_init_with_id(vat,
2292 trace_id, location, true);
2293 vat->vat_flags |= _voucher_activity_trace_flag_tracepoint_args;
2294 vat->vat_data[0] = arg1;
2295 vat->vat_data[1] = arg2;
2296 vat->vat_data[2] = arg3;
2297 vat->vat_data[3] = arg4;
2298 _voucher_activity_trace_fault(trace_id);
2299 if (_voucher_activity_buffer_tracepoint_release(vab)) {
2300 _voucher_activity_firehose_push(act, vab);
2301 }
2302 return timestamp;
2303 }
2304
2305 #pragma mark -
2306 #pragma mark _voucher_debug
2307
2308 size_t
2309 _voucher_debug(voucher_t v, char* buf, size_t bufsiz)
2310 {
2311 size_t offset = 0;
2312 #define bufprintf(...) \
2313 offset += dsnprintf(&buf[offset], bufsiz - offset, ##__VA_ARGS__)
2314 bufprintf("voucher[%p] = { xrefcnt = 0x%x, refcnt = 0x%x, ", v,
2315 v->os_obj_xref_cnt + 1, v->os_obj_ref_cnt + 1);
2316
2317 if (v->v_kvbase) {
2318 bufprintf("base voucher %p, ", v->v_kvbase);
2319 }
2320 if (v->v_kvoucher) {
2321 bufprintf("kvoucher%s 0x%x, ", v->v_kvoucher == v->v_ipc_kvoucher ?
2322 " & ipc kvoucher" : "", v->v_kvoucher);
2323 }
2324 if (v->v_ipc_kvoucher && v->v_ipc_kvoucher != v->v_kvoucher) {
2325 bufprintf("ipc kvoucher 0x%x, ", v->v_ipc_kvoucher);
2326 }
2327 if (v->v_has_priority) {
2328 bufprintf("QOS 0x%x, ", *_voucher_priority(v));
2329 }
2330 if (v->v_activities) {
2331 voucher_activity_id_t *activity_ids = _voucher_activity_ids(v);
2332 bufprintf("activity IDs = { ");
2333 unsigned int i;
2334 for (i = 0; i < v->v_activities; i++) {
2335 bufprintf("0x%llx, ", *activity_ids++);
2336 }
2337 bufprintf("}, ");
2338 }
2339 if (v->v_activity) {
2340 _voucher_activity_t va = v->v_activity;
2341 _voucher_atm_t vatm = va->va_atm;
2342 bufprintf("activity[%p] = { ID 0x%llx, ref %d, atm[%p] = { "
2343 "AID 0x%llx, ref %d, kvoucher 0x%x } }, ", va, va->va_id,
2344 va->va_refcnt + 1, va->va_atm, vatm->vatm_id,
2345 vatm->vatm_refcnt + 1, vatm->vatm_kvoucher);
2346 }
2347 bufprintf("}");
2348 return offset;
2349 }
2350
2351 #else // VOUCHER_USE_MACH_VOUCHER
2352
2353 #pragma mark -
2354 #pragma mark Simulator / vouchers disabled
2355
2356 #if VOUCHER_ENABLE_RECIPE_OBJECTS
2357 voucher_t
2358 voucher_create(voucher_recipe_t recipe)
2359 {
2360 (void)recipe;
2361 return NULL;
2362 }
2363 #endif
2364
2365 voucher_t
2366 voucher_adopt(voucher_t voucher)
2367 {
2368 return voucher;
2369 }
2370
2371 voucher_t
2372 voucher_copy(void)
2373 {
2374 return NULL;
2375 }
2376
2377 voucher_t
2378 voucher_copy_without_importance(void)
2379 {
2380 return NULL;
2381 }
2382
2383 voucher_t
2384 voucher_retain(voucher_t voucher)
2385 {
2386 return voucher;
2387 }
2388
2389 void
2390 voucher_release(voucher_t voucher)
2391 {
2392 (void)voucher;
2393 }
2394
2395 void
2396 voucher_replace_default_voucher(void)
2397 {
2398 }
2399
2400 void
2401 voucher_decrement_importance_count4CF(voucher_t v)
2402 {
2403 (void)v;
2404 }
2405
2406 void
2407 _voucher_thread_cleanup(void *voucher)
2408 {
2409 (void)voucher;
2410 }
2411
2412 void
2413 _voucher_dealloc_mach_voucher(mach_voucher_t kv)
2414 {
2415 (void)kv;
2416 }
2417
2418 mach_voucher_t
2419 _voucher_create_mach_voucher_with_priority(voucher_t voucher,
2420 pthread_priority_t priority)
2421 {
2422 (void)voucher; (void)priority;
2423 return MACH_VOUCHER_NULL;
2424 }
2425
2426 voucher_t
2427 _voucher_create_with_priority_and_mach_voucher(voucher_t voucher,
2428 pthread_priority_t priority, mach_voucher_t kv)
2429 {
2430 (void)voucher; (void)priority; (void)kv;
2431 return NULL;
2432 }
2433
2434 voucher_t
2435 _voucher_create_accounting_voucher(voucher_t voucher)
2436 {
2437 (void)voucher;
2438 return NULL;
2439 }
2440
2441 voucher_t
2442 voucher_create_with_mach_msg(mach_msg_header_t *msg)
2443 {
2444 (void)msg;
2445 return NULL;
2446 }
2447
2448 #if VOUCHER_ENABLE_GET_MACH_VOUCHER
2449 mach_voucher_t
2450 voucher_get_mach_voucher(voucher_t voucher)
2451 {
2452 (void)voucher;
2453 return 0;
2454 }
2455 #endif
2456
2457 void
2458 _voucher_xref_dispose(voucher_t voucher)
2459 {
2460 (void)voucher;
2461 }
2462
2463 void
2464 _voucher_dispose(voucher_t voucher)
2465 {
2466 (void)voucher;
2467 }
2468
2469 void
2470 _voucher_atfork_child(void)
2471 {
2472 }
2473
2474 void
2475 _voucher_init(void)
2476 {
2477 }
2478
2479 void*
2480 voucher_activity_get_metadata_buffer(size_t *length)
2481 {
2482 *length = 0;
2483 return NULL;
2484 }
2485
2486 void
2487 voucher_activity_buffer_hook_install_4libtrace(
2488 _voucher_activity_buffer_hook_t hook)
2489 {
2490 (void)hook;
2491 }
2492
2493 void
2494 _voucher_activity_heap_pressure_normal(void)
2495 {
2496 }
2497
2498 void
2499 _voucher_activity_heap_pressure_warn(void)
2500 {
2501 }
2502
2503 voucher_activity_id_t
2504 voucher_activity_start_with_location(voucher_activity_trace_id_t trace_id,
2505 uint64_t location, voucher_activity_flag_t flags)
2506 {
2507 (void)trace_id; (void)location; (void)flags;
2508 return 0;
2509 }
2510
2511 voucher_activity_id_t
2512 voucher_activity_start(voucher_activity_trace_id_t trace_id,
2513 voucher_activity_flag_t flags)
2514 {
2515 (void)trace_id; (void)flags;
2516 return 0;
2517 }
2518
2519 void
2520 voucher_activity_end(voucher_activity_id_t activity_id)
2521 {
2522 (void)activity_id;
2523 }
2524
2525 unsigned int
2526 voucher_get_activities(voucher_activity_id_t *entries, unsigned int *count)
2527 {
2528 (void)entries; (void)count;
2529 return 0;
2530 }
2531
2532 uint8_t
2533 voucher_activity_get_namespace(void)
2534 {
2535 return 0;
2536 }
2537
2538 uint64_t
2539 voucher_activity_trace(voucher_activity_trace_id_t trace_id, uint64_t location,
2540 void *buffer, size_t length)
2541 {
2542 (void)trace_id; (void)location; (void)buffer; (void)length;
2543 return 0;
2544 }
2545
2546 uint64_t
2547 voucher_activity_trace_strings(voucher_activity_trace_id_t trace_id,
2548 uint64_t location, void *buffer, size_t length, const char *strings[],
2549 size_t string_lengths[], size_t strings_size)
2550 {
2551 (void)trace_id; (void)location; (void)buffer; (void)length; (void)strings;
2552 (void)string_lengths; (void)strings_size;
2553 return 0;
2554 }
2555
2556 uint64_t
2557 voucher_activity_trace_args(voucher_activity_trace_id_t trace_id,
2558 uint64_t location, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3,
2559 uintptr_t arg4)
2560 {
2561 (void)trace_id; (void)location;
2562 (void)arg1; (void)arg2; (void)arg3; (void)arg4;
2563 return 0;
2564 }
2565
2566 size_t
2567 _voucher_debug(voucher_t v, char* buf, size_t bufsiz)
2568 {
2569 (void)v; (void)buf; (void)bufsiz;
2570 return 0;
2571 }
2572
2573 #endif // VOUCHER_USE_MACH_VOUCHER
mirror of libdispatch