]> git.saurik.com Git - apple/xnu.git/blob - tools/lldbmacros/waitq.py
xnu-3789.21.4.tar.gz
[apple/xnu.git] / tools / lldbmacros / waitq.py
1 from xnu import *
2 from utils import *
3 from core.configuration import *
4
5 import sys
6
7 def GetWaitqStateStr(waitq):
8 wq_types = {
9 0: 'INV',
10 1: '???',
11 2: ' Q',
12 3: 'SET'
13 }
14 return wq_types[int(waitq.waitq_type)]
15
16 def GetWaitqBitsStr(waitq):
17 out_str = ""
18 if (Cast(waitq.waitq_interlock, 'int') != 0):
19 if waitq.waitq_irq:
20 out_str += '!'
21 else:
22 out_str += '*'
23 if waitq.waitq_fifo:
24 out_str += 'F'
25 if waitq.waitq_prepost:
26 out_str += 'P'
27 if waitq.waitq_irq:
28 out_str += 'I'
29 return out_str
30
31 def WaitqTableElemType(e):
32 type = (e.wqte.lt_bits >> 29) & 0x3
33 wqe_type = {
34 0: 'FREE',
35 1: 'ELEM',
36 2: 'LINK',
37 3: 'RSVD'
38 }
39 return wqe_type[type]
40
41 def WaitqTableElemId(e):
42 return e.wqte.lt_id.id
43
44 def WaitqTableElemValid(e):
45 if unsigned(e) == 0:
46 return 0
47 return (e.wqte.lt_bits & 0x80000000) == 0x80000000
48
49 def WaitqTableElemRefcnt(e):
50 return (e.wqte.lt_bits & 0x1fffffff)
51
52 def WaitqTableIdxFromId(id):
53 if hasattr(kern.globals, 'g_lt_idx_max'):
54 idx = id & unsigned(kern.globals.g_lt_idx_max)
55 else:
56 # best guess
57 idx = id & 0x000000000003ffff
58 return int(idx)
59
60 def WaitqTableGenFromId(id):
61 if hasattr(kern.globals, 'g_lt_idx_max'):
62 msk = ~unsigned(kern.globals.g_lt_idx_max)
63 else:
64 # best guess
65 msk = ~0x000000000003ffff
66 shift = 0
67 while (msk & 0x1) == 0:
68 msk >>= 1
69 shift += 1
70 return (unsigned(id) >> shift) & msk
71
72 def GetWaitqLink(id):
73 if int(id) == 0:
74 return 0, "NULL link id"
75 idx = WaitqTableIdxFromId(id)
76 if idx >= kern.globals.g_wqlinktable.nelem:
77 return 0, "Invalid waitq link table id: {:d}".format(id)
78 slab_slot = idx / kern.globals.g_wqlinktable.slab_elem;
79 slab = kern.globals.g_wqlinktable.table[int(slab_slot)]
80 if slab == 0:
81 print "Invalid waitq link table id:", str(id), " (invalid slab)"
82 first_elem = Cast(slab, 'lt_elem *')
83 addr = int(slab) + ((idx - first_elem.lt_id.idx) * int(kern.globals.g_wqlinktable.elem_sz))
84 link = kern.GetValueFromAddress(addr, 'waitq_link *')
85 gen = WaitqTableGenFromId(id)
86 warn_str = ''
87 if gen > 0 and link.wqte.lt_id.generation != gen:
88 warn_str = "WARNING: found idx:{:d}/gen:{:d}, but requested idx:{:d}/gen:{:d}".format(link.wqte.lt_id.idx, link.wqte.lt_id.generation, idx, gen)
89 link = 0
90 return link, warn_str
91
92 def GetWaitqPrepost(id):
93 idx = WaitqTableIdxFromId(id)
94 if idx > int(kern.globals.g_prepost_table.nelem):
95 warn_str = "Invalid waitq prepost table id {:s}".format(str(id))
96 return 0, warn_str
97 slab_slot = idx / kern.globals.g_prepost_table.slab_elem;
98 slab = kern.globals.g_prepost_table.table[int(slab_slot)]
99 if slab == 0:
100 warn_str = "Invalid waitq prepost table id:", str(id), " (invalid slab)"
101 return 0, warn_str
102 first_elem = Cast(slab, 'lt_elem *')
103 addr = int(slab) + ((idx - first_elem.lt_id.idx) * int(kern.globals.g_prepost_table.elem_sz))
104 wqp = kern.GetValueFromAddress(addr, 'wq_prepost *')
105 gen = WaitqTableGenFromId(id)
106 warn_str = ''
107 if gen > 0 and wqp.wqte.lt_id.generation != gen:
108 warn_str = "WARNING: found idx:{:d}/gen:{:d}, but requested idx:{:d}/gen:{:d}".format(wqp.wqte.lt_id.idx, wqp.wqte.lt_id.generation, idx, gen)
109 wqp = 0
110 return wqp, warn_str
111
112
113 def GetWaitqSetidString(setid):
114 idx = WaitqTableIdxFromId(setid)
115 gen = WaitqTableGenFromId(setid)
116 # This must match the format used in WaitqSetsFromLink
117 str = "{:>7d}/{:<#14x}".format(unsigned(idx), unsigned(gen))
118 return str
119
120
121 def WaitqSetsFromLink(link, sets, depth):
122 if int(link) == 0:
123 sets.append("{: <22s}".format("<link:NULL>"))
124 return
125 if WaitqTableElemType(link) == "ELEM":
126 #sets.append("{: <#18x}".format(unsigned(link.wql_wqs.wql_set)))
127 #sets.append("{:>7d}/{:<#14x}".format(unsigned(id.idx),unsigned(id.generation)))
128 sets.append(GetWaitqSetidString(link.wqte.lt_id.id))
129 return
130 if depth >= 950:
131 sets.append("{: <22s}".format("!recursion limit!"))
132 return
133 left_link = GetWaitqLink(link.wql_link.left_setid)[0]
134 right_link = GetWaitqLink(link.wql_link.right_setid)[0]
135 WaitqSetsFromLink(left_link, sets, depth + 1)
136 WaitqSetsFromLink(right_link, sets, depth + 1)
137 return
138
139 def GetWaitqSets(waitq):
140 sets = []
141 if int(waitq) == 0:
142 return sets
143 if waitq.waitq_set_id == 0:
144 return sets
145 link = GetWaitqLink(waitq.waitq_set_id)[0]
146 WaitqSetsFromLink(link, sets, 0)
147 return sets
148
149 def GetFrameString(pc, compact=True):
150 str = GetSourceInformationForAddress(unsigned(pc))
151 if compact:
152 return re.sub(r'.*0x[0-9a-f]+\s+<(\w+)( \+ 0x[0-9a-f]+)*>.*', r'\1', str, re.UNICODE)
153 else:
154 return re.sub(r'.*(0x[0-9a-f]+)\s+<(\w+)( \+ 0x[0-9a-f]+)*>.*', r'\2(\1)', str, re.UNICODE)
155
156 @lldb_type_summary(['waitq_link', 'waitq_link *'])
157 @header("{:<18s} {:<18s} {:<19s} {:<10s} {:<1s} {:<4s} {:<10s} {:<20s}".format('addr','id','idx','gen','V','type','refcnt','info'))
158 def GetWaitqSetidLinkSummary(link, verbose=False):
159 has_stats = 0
160 if not link:
161 return ""
162 fmt_str = "{l: <#18x} {l.wqte.lt_id.id: <#18x} {l.wqte.lt_id.idx: <7d} (->{l.wqte.lt_next_idx: <7d}) {l.wqte.lt_id.generation: <#10x} {v: <1s} {t: <4s} {rcnt: <10d} "
163 if hasattr(link, 'sl_alloc_task'):
164 has_stats = 1
165 fmt_str += "owner:{l.sl_alloc_task: <#x}/th:{l.sl_alloc_th: <#x}\n"
166 fmt_str += ' '*87
167 try:
168 pid = GetProcPIDForTask(link.sl_alloc_task)
169 except:
170 pid = unsigned(link.sl_alloc_task.audit_token.val[5])
171 pidnm = ""
172 if pid < 0:
173 pidnm = "DEAD:{:s}".format(GetProcNameForTask(link.sl_alloc_task))
174 else:
175 pidnm += GetProcNameForPid(pid)
176 fmt_str += " ({:d}/{:s}), ".format(pid, pidnm)
177 type = WaitqTableElemType(link)
178 if type == "ELEM":
179 type = "WQS"
180 v = "F"
181 if WaitqTableElemValid(link):
182 v = "T"
183 refcnt = WaitqTableElemRefcnt(link)
184 out_str = fmt_str.format(l=link, v=v, t=type, rcnt=refcnt)
185 if type == "WQS":
186 out_str += "wqs:{0: <#18x}".format(unsigned(link.wql_wqs.wql_set))
187 elif type == "LINK":
188 lID = link.wql_link.left_setid
189 rID = link.wql_link.right_setid
190 left = GetWaitqLink(lID)[0]
191 right = GetWaitqLink(rID)[0]
192 ltype = "<invalid>"
193 if WaitqTableElemValid(left):
194 ltype = WaitqTableElemType(left)
195 if ltype == "ELEM":
196 ltype = "WQS"
197 rtype = "<invalid>"
198 if WaitqTableElemValid(right):
199 rtype = WaitqTableElemType(right)
200 if rtype == "ELEM":
201 rtype = "WQS"
202 out_str += "left:{:<#x}({:s}), right:{:<#x}({:s})".format(lID, ltype, rID, rtype)
203 if hasattr(link, 'sl_alloc_bt') and unsigned(link.sl_alloc_bt[0]) > 0:
204 fmt_str = "\n{:s}alloc_bt({:d}):[".format(' '*87, link.sl_alloc_ts)
205 f = 0
206 while f < kern.globals.g_nwaitq_btframes:
207 fstr = GetFrameString(link.sl_alloc_bt[f], not verbose)
208 f += 1
209 if f == kern.globals.g_nwaitq_btframes:
210 fmt_str += "{:<s}".format(fstr)
211 else:
212 fmt_str += "{:<s} <- ".format(fstr)
213 fmt_str += "]"
214 out_str += fmt_str
215 if hasattr(link, 'sl_mkvalid_bt') and unsigned(link.sl_mkvalid_bt[0]) > 0:
216 fmt_str = "\n{:s}mkvalid_bt({:d}):[".format(' '*87, link.sl_mkvalid_ts)
217 f = 0
218 while f < kern.globals.g_nwaitq_btframes:
219 fstr = GetFrameString(link.sl_mkvalid_bt[f], not verbose)
220 f += 1
221 if f == kern.globals.g_nwaitq_btframes:
222 fmt_str += "{:<s}".format(fstr)
223 else:
224 fmt_str += "{:<s} <- ".format(fstr)
225 fmt_str += "]"
226 out_str += fmt_str
227 if hasattr(link, 'sl_invalidate_bt') and unsigned(link.sl_invalidate_bt[0]) > 0:
228 fmt_str = "\n{:s}invalidate_bt({:d}):[".format(' '*87, link.sl_invalidate_ts)
229 f = 0
230 while f < kern.globals.g_nwaitq_btframes:
231 fstr = GetFrameString(link.sl_invalidate_bt[f], not verbose)
232 f += 1
233 if f == kern.globals.g_nwaitq_btframes:
234 fmt_str += "{:<s}".format(fstr)
235 else:
236 fmt_str += "{:<s} <- ".format(fstr)
237 fmt_str += "]"
238 out_str += fmt_str
239 return out_str
240
241 def PrintWaitqSetidLinkTree(link, verbose, sets, indent=87):
242 if not WaitqTableElemType(link) == "LINK":
243 return
244 lID = link.wql_link.left_setid
245 rID = link.wql_link.right_setid
246 left = GetWaitqLink(lID)[0]
247 right = GetWaitqLink(rID)[0]
248
249 ltype = "<invalid>"
250 if WaitqTableElemValid(left):
251 ltype = WaitqTableElemType(left)
252 if ltype == "ELEM":
253 ltype = "WQS"
254 lstr = "L:{:<#x}({:s})".format(lID, ltype)
255
256 rtype = "<invalid>"
257 if WaitqTableElemValid(right):
258 rtype = WaitqTableElemType(right)
259 if rtype == "ELEM":
260 rtype = "WQS"
261 rstr = "R:{:<#x}({:s})".format(rID, rtype)
262
263 if ltype == "WQS":
264 sets.append(addressof(left.wql_wqs.wql_set.wqset_q))
265 if rtype == "WQS":
266 sets.append(addressof(right.wql_wqs.wql_set.wqset_q))
267
268 print "{:s}`->{:s}, {:s}".format(' '*indent, lstr, rstr)
269 if ltype == "WQS":
270 PrintWaitqSetidLinkTree(right, verbose, sets, indent + len(lstr) + 6);
271 else:
272 print "{:s}`->{:s}, {:s}".format(' '*indent, lstr, rstr)
273 PrintWaitqSetidLinkTree(left, verbose, sets, indent + 4);
274 PrintWaitqSetidLinkTree(right, verbose, sets, indent + len(lstr) + 6)
275 return
276
277 # Macro: showsetidlink
278 @lldb_command('showsetidlink', "S:FT")
279 def ShowSetidLink(cmd_args=None, cmd_options={}):
280 """ Print waitq_link structure summary
281
282 Note: you can pass either a complete ID (generation + index), or
283 just the index to the -S argument.
284
285 usage: showsetidlink [-F] [-S ID] [0xaddr]
286 -S {ID} : show the setid link whose ID is {ID}
287 -F : follow the chain of setid structures
288 and print a summary of each one
289 -T : print the tree of setidlinks in table format
290 """
291 link = 0
292 followchain = 0
293 showtree = 0
294 verbose = False
295 if config['verbosity'] > vHUMAN:
296 verbose = True
297 if "-T" in cmd_options:
298 showtree = 1
299 if "-S" in cmd_options:
300 id = unsigned(kern.GetValueFromAddress(cmd_options["-S"], 'uint64_t *'))
301 link, warn_str = GetWaitqLink(id)
302 if not link:
303 if warn_str != '':
304 raise LookupError(warn_str)
305 else:
306 raise ArgumentError("Invalid link ID {:d}({:<#x}".format(id, id))
307 if "-F" in cmd_options:
308 followchain = 1
309 if link == 0:
310 if not cmd_args:
311 raise ArgumentError("Please pass the address of a waitq_link object")
312 link = kern.GetValueFromAddress(cmd_args[0], 'waitq_link *')
313 if not link:
314 raise ArgumentError("Invalid waitq_link {:s}".format(cmd_args[0]))
315
316 print GetWaitqSetidLinkSummary.header
317 print GetWaitqSetidLinkSummary(link, verbose)
318 if followchain == 1:
319 next_id = link.wqte.lt_next_idx
320 max_elem = int(kern.globals.g_wqlinktable.nelem)
321 if hasattr(kern.globals, 'g_lt_idx_max'):
322 max_elem = unsigned(kern.globals.g_lt_idx_max)
323 while link != 0 and next_id < max_elem:
324 link, warn_str = GetWaitqLink(unsigned(next_id))
325 if link != 0:
326 print GetWaitqSetidLinkSummary(link, verbose)
327 next_id = link.wqte.lt_next_idx
328 if showtree == 1:
329 sets = []
330 print "\nLinkTree:{:<#x}({:s})".format(link.wqte.lt_id.id, WaitqTableElemType(link))
331 PrintWaitqSetidLinkTree(link, verbose, sets, 9)
332 if len(sets) > 0:
333 print "{:d} Sets:".format(len(sets))
334 for wq in sets:
335 pp_str = GetWaitqPreposts(wq)
336 npreposts = len(pp_str)
337 nps = ""
338 if npreposts > 0:
339 if npreposts > 1:
340 nps = "s: "
341 else:
342 nps = ": "
343 nps += ';'.join(pp_str)
344 else:
345 nps = "s"
346 print "\tWQS:{:<#x} ({:d} prepost{:s})".format(unsigned(wq),npreposts,nps)
347 # EndMacro: showsetidlink
348 @lldb_command('showwaitqlink', "S:FT")
349 def ShowWaitqLink(cmd_args=None, cmd_options={}):
350 """ Print waitq_link structure summary
351 """
352 ShowSetidLink(cmd_args, cmd_options)
353
354
355 # Macro: showallsetidlinks
356 @lldb_command('showallsetidlinks', 'V:T:S:F:XQ')
357 def ShowAllSetidLinks(cmd_args=None, cmd_options={}):
358 """ Dump / summarize all waitq set linktable elements
359
360 usage: showallsetidlinks [options]
361 -V {0,1} : only show [1 == valid/live links, 0 == invalid links]
362 -T {type} : only display objects of type {type}
363 -S {desc} : only display objects of type {type} which fit {desc}
364 -T LINK -S {desc} can be:
365 iL : Invalid left-link pointer (only)
366 iR : Invalid right-link pointer (only)
367 iLR : Invalid left+right link pointers
368 iLRI : Invalid left+right link pointers AND dead allocating process
369 w/o "-T" -S {desc} can be:
370 iP : Invalid / Dead allocating process
371 -F n : summarize the backtraces at frame level 'n'
372 -X : cross-check waitq pointers in link table
373 -Q : be quiet, only summarize
374 """
375 opt_summary = 0
376 opt_type_filt = ""
377 opt_valid_only = 0
378 opt_invalid_only = 0
379 opt_bt_idx = 0
380 opt_cross_check = 0
381 opt_validate_links = 0
382 opt_subtype_filter = 0
383 verbose = False
384 if config['verbosity'] > vHUMAN:
385 verbose = True
386 if "-Q" in cmd_options:
387 opt_summary = 1
388 if "-V" in cmd_options:
389 if int(cmd_options["-V"]) == 1:
390 opt_valid_only = 1
391 elif int(cmd_options["-V"]) == 0:
392 opt_invalid_only = 1
393 else:
394 raise ArgumentError("Invalid parameter to -V '{:s}': expecting 0 or 1".format(cmd_options["-V"]))
395 if "-X" in cmd_options:
396 opt_cross_check = 1
397 nunique_wqs = 0
398 nduplicated_wqs = 0
399 max_wqs_dupes = 0
400 if "-F" in cmd_options:
401 opt_bt_idx = unsigned(cmd_options["-F"])
402 if hasattr(kern.globals, "g_nwaitq_btframes"):
403 if opt_bt_idx >= unsigned(kern.globals.g_nwaitq_btframes):
404 raise ArgumentError("Invalid BT index '{:s}' max:{:d}".format(cmd_options["-F"], unsigned(kern.globals.g_nwaitq_btframes) - 1))
405 if "-T" in cmd_options:
406 opt_type_filt = cmd_options["-T"]
407 if opt_type_filt == "FREE" or opt_type_filt == "RSVD" or opt_type_filt == "LINK":
408 pass
409 elif opt_type_filt == "WQS":
410 opt_type_filt = "ELEM"
411 else:
412 raise ArgumentError("Invalid type filter'{:s}'".format(cmd_options["-T"]))
413 if "-S" in cmd_options:
414 opt_subtype_filter = cmd_options["-S"]
415 if opt_type_filt == "LINK":
416 if not (opt_subtype_filter == "iL" or \
417 opt_subtype_filter == "iR" or \
418 opt_subtype_filter == "iLR" or \
419 opt_subtype_filter == "iLRI"):
420 raise ArgumentError("Invalid LINK sub-type filter \{desc\}: {:s}".format(opt_subtype_filter))
421 elif opt_type_filt == "":
422 if not opt_subtype_filter == "iP":
423 raise ArgumentError("Invalid sub-type filter \{desc\}: {:s}".format(opt_subtype_filter))
424 table = kern.globals.g_wqlinktable
425 nelem = int(table.nelem)
426 wq_ptr = {}
427 bt_summary = {}
428 nfree = 0
429 ninv = 0
430 nwqs = 0
431 nlink = 0
432 nrsvd = 0
433 hdr_str = "Looking through {:d} waitq_link objects from g_wqlinktable@{:<#x}".format(nelem, addressof(kern.globals.g_wqlinktable))
434 if opt_type_filt != "" or opt_valid_only != 0:
435 hdr_str += "\n\t`-> for "
436 if opt_valid_only:
437 hdr_str += "valid "
438 else:
439 hdr_str += "all "
440 if opt_type_filt == "":
441 hdr_str += "objects"
442 else:
443 hdr_str += "{:s} objects".format(opt_type_filt)
444 else:
445 if opt_valid_only:
446 hdr_str += "\n\t`-> showing only VALID links"
447 elif opt_invalid_only:
448 hdr_str += "\n\t`-> showing only INVALID links"
449 if opt_subtype_filter != 0:
450 if opt_type_filt != "LINK" and opt_type_filt != "":
451 raise ArgumentError("Subtype (-S {desc}) can only be used with (-T LINK) or no type filter at all")
452 hdr_str += "\n\t`-> filtering {:s} objects through '{:s}'".format(opt_type_filt, opt_subtype_filter)
453 if opt_cross_check:
454 hdr_str += "\n\t`-> cross-checking WQS elements for duplicates"
455 hdr_str += "\n\n"
456 print hdr_str
457 if not opt_summary:
458 print GetWaitqSetidLinkSummary.header
459 id = 0
460 while id < nelem:
461 if id == 0:
462 # Set a generation count to differentiate from an invalid ID
463 first_entry = Cast(kern.globals.g_wqlinktable.table[0], 'lt_elem *')
464 link = GetWaitqLink(first_entry.lt_id.id)[0]
465 else:
466 link = GetWaitqLink(id)[0]
467 if not link:
468 print "<<<invalid link:{:d}>>>".format(id)
469 ninv += 1
470 else:
471 lt = WaitqTableElemType(link)
472 isvalid = WaitqTableElemValid(link)
473 inconsistent = 0
474 do_print = not ( (isvalid and opt_invalid_only) or (not isvalid and opt_valid_only) )
475 if do_print and opt_subtype_filter != 0 and lt == "LINK":
476 lID = link.wql_link.left_setid
477 rID = link.wql_link.right_setid
478 left = GetWaitqLink(lID)[0]
479 right = GetWaitqLink(rID)[0]
480 lValid = WaitqTableElemValid(left)
481 rValid = WaitqTableElemValid(right)
482 if opt_subtype_filter == "iL":
483 if lValid or (not lValid and not rValid):
484 do_print = False
485 elif opt_subtype_filter == "iR":
486 if rValid or (not rValid and not lValid):
487 do_print = False
488 elif opt_subtype_filter == "iLR":
489 if rValid or lValid:
490 do_print = False
491 elif opt_subtype_filter == "iLRI" and hasattr(link, 'sl_alloc_task'):
492 # only print this if both left and right are invalid
493 # and the allocating task is unknown/dead
494 do_print = False
495 is_dead = 0
496 pid = -1
497 try:
498 pid = GetProcPIDForTask(link.sl_alloc_task)
499 except:
500 if link.sl_alloc_task:
501 pid = unsigned(link.sl_alloc_task.audit_token.val[5])
502 if pid < 0:
503 is_dead = 1
504 else:
505 pidnm = GetProcNameForPid(pid)
506 if pidnm == "Unknown":
507 is_dead = 1
508 if (not rValid) and (not lValid) and is_dead:
509 do_print = True
510
511 if do_print and opt_type_filt == "" and opt_subtype_filter == "iP" and hasattr(link, 'sl_alloc_task'):
512 # Only print non-free table objects that were allocated by
513 # dead processes
514 do_print = False
515 is_dead = 0
516 pid = -1
517 try:
518 pid = GetProcPIDForTask(link.sl_alloc_task)
519 except:
520 if link.sl_alloc_task:
521 pid = unsigned(link.sl_alloc_task.audit_token.val[5])
522 if pid < 0:
523 is_dead = 1
524 else:
525 pidnm = GetProcNameForPid(pid)
526 if pidnm == "Unknown":
527 is_dead = 1
528 if is_dead:
529 do_print = True
530
531 if (opt_type_filt == "" or opt_type_filt == lt) and do_print:
532 if lt == "ELEM":
533 nwqs += 1
534 elif lt == "LINK":
535 nlink += 1
536 elif lt == "RSVD":
537 nrsvd += 1
538 elif lt == "FREE":
539 nfree += 1
540 else:
541 ninv += 1
542 inconsistent = 1
543 if hasattr(link, 'sl_alloc_bt'):
544 pc = unsigned(link.sl_alloc_bt[opt_bt_idx])
545 pc_str = str(pc)
546 if pc > 0:
547 if pc_str in bt_summary:
548 bt_summary[pc_str] += 1
549 else:
550 bt_summary[pc_str] = 1
551 if not opt_summary:
552 print GetWaitqSetidLinkSummary(link, verbose)
553 if inconsistent:
554 ninconsistent += 1
555 # print out warnings about inconsistent state as we parse
556 # the list - even if the caller wants a summary
557 print "[WARNING] inconsistent state in idx: {:d} ({:s} element)".format(link.wqte.lt_id.idx, lt)
558 if opt_cross_check == 1 and lt == "ELEM":
559 wq = unsigned(addressof(link.wql_wqs.wql_set.wqset_q))
560 if wq in wq_ptr:
561 wq_ptr[wq].append(id)
562 l = len(wq_ptr[wq])
563 if l == 2:
564 nduplicated_wqs += 1
565 if l > max_wqs_dupes:
566 max_wqs_dupes = l
567 else:
568 wq_ptr[wq] = [ id ]
569 nunique_wqs += 1
570 id += 1
571 if opt_summary or verbose:
572 if verbose and opt_cross_check:
573 sys.stderr.write('[{:d}|{:d}|{:d}] id: {:d}/{:d}... \r'.format(nunique_wqs, nduplicated_wqs, max_wqs_dupes, id, nelem))
574 else:
575 sys.stderr.write('id: {:d}/{:d}... \r'.format(id, nelem))
576
577 nused = nwqs + nlink + nrsvd
578 nfound = nused + nfree + ninv
579 print "\n\nFound {:d} objects: {:d} WQS, {:d} LINK, {:d} RSVD, {:d} FREE".format(nfound, nwqs, nlink, nrsvd, nfree)
580 if (opt_type_filt == "" and opt_valid_only == 0) and (nused != table.used_elem):
581 print"\tWARNING: inconsistent state! Table reports {:d}/{:d} used elem, found {:d}/{:d}".format(table.used_elem, nelem, nused, nfound)
582 if len(bt_summary) > 0:
583 print "Link allocation BT (frame={:d})".format(opt_bt_idx)
584 for k,v in bt_summary.iteritems():
585 print "\t[{:d}] from: {:s}".format(v, GetSourceInformationForAddress(unsigned(k)))
586 if opt_cross_check:
587 print "\n{:d} Duplicated WQS objects:".format(nduplicated_wqs)
588 for wq in wq_ptr:
589 l = len(wq_ptr[wq])
590 if l > 1:
591 print "\tWQS:{:#x} ({:d} {:s}".format(wq, l, str(wq_ptr[wq]))
592 # EndMacro: showallsetidlinks
593
594
595 # Macro: showallpreposts
596 @lldb_command('showallpreposts', 'VQT:F:Y:')
597 def ShowAllPreposts(cmd_args=None, cmd_options={}):
598 """ Dump / summarize all waitq prepost linkage elements
599
600 usage: showallpreposts [-V] [-T {type}] [-Y n] [-F n] [-Q]
601 -V : only show valid / live links
602 -T {type} : only display objects of type {type}
603 -Y {0|1} : only only show POST objects that are
604 valid (-Y 1) or invalid (-Y 0)
605 -F n : summarize the backtraces at frame level 'n'
606 -Q : be quiet, only summarize
607 """
608 opt_summary = 0
609 opt_type_filt = ""
610 opt_valid_only = 0
611 opt_post_type = -1
612 opt_bt_idx = 0
613 verbose = False
614 if config['verbosity'] > vHUMAN:
615 verbose = True
616 if "-Q" in cmd_options:
617 opt_summary = 1
618 if "-V" in cmd_options:
619 opt_valid_only = 1
620 if "-Y" in cmd_options:
621 opt_post_type = unsigned(cmd_options["-Y"])
622 if opt_post_type != 0 and opt_post_type != 1:
623 raise ArgumentError("Invalid POST obj specifier [-Y %d] (expected 0 or 1)" % cmd_options["-Y"])
624 if "-F" in cmd_options:
625 opt_bt_idx = unsigned(cmd_options["-F"])
626 if hasattr(kern.globals, "g_nwaitq_btframes"):
627 if opt_bt_idx >= unsigned(kern.globals.g_nwaitq_btframes):
628 raise ArgumentError("Invalid BT index '{:s}' max:{:d}".format(cmd_options["-F"], unsigned(kern.globals.g_nwaitq_btframes) - 1))
629 if "-T" in cmd_options:
630 opt_type_filt = cmd_options["-T"]
631 if opt_type_filt == "FREE" or opt_type_filt == "RSVD":
632 pass
633 elif opt_type_filt == "POST":
634 opt_type_filt = "LINK"
635 elif opt_type_filt == "WQ":
636 opt_type_filt = "ELEM"
637 else:
638 raise ArgumentError("Invalid type filter'{:s}'".format(cmd_options["-T"]))
639 table = kern.globals.g_prepost_table
640 nelem = int(table.nelem)
641 bt_summary = {}
642 nfree = 0
643 ninv = 0
644 nwq = 0
645 npost = 0
646 nrsvd = 0
647 hdr_str = "Looking through {:d} objects from g_prepost_table@{:<#x}".format(nelem, addressof(kern.globals.g_prepost_table))
648 if opt_type_filt != "" or opt_valid_only != 0:
649 hdr_str += "\n\t`-> for "
650 if opt_valid_only:
651 hdr_str += "valid "
652 else:
653 hdr_str += "all "
654 if opt_type_filt == "":
655 hdr_str += "objects"
656 else:
657 hdr_str += "{:s} objects".format(cmd_options["-T"])
658 print hdr_str
659 if not opt_summary:
660 print GetWaitqPrepostSummary.header
661 id = 0
662 while id < nelem:
663 wqp = GetWaitqPrepost(id)[0]
664 if wqp == 0:
665 print "<<<invalid prepost:{:d}>>>".format(id)
666 ninv += 1
667 else:
668 lt = WaitqTableElemType(wqp)
669 isvalid = WaitqTableElemValid(wqp)
670 should_count = 1
671 if isvalid and opt_post_type > -1 and lt == "LINK":
672 post_wqp = GetWaitqPrepost(wqp.wqp_post.wqp_wq_id)[0]
673 post_valid = WaitqTableElemValid(post_wqp)
674 if opt_post_type == 0 and post_valid: # only count _invalid_ POST objects
675 should_count = 0
676 elif opt_post_type == 1 and not post_valid: # only count _valid_ POST objects
677 should_count = 0
678 if should_count and (opt_type_filt == "" or opt_type_filt == lt) and ((opt_valid_only == 0 or isvalid)):
679 if lt == "ELEM":
680 nwq += 1
681 elif lt == "LINK":
682 npost += 1
683 elif lt == "RSVD":
684 nrsvd += 1
685 elif lt == "FREE":
686 nfree += 1
687 else:
688 ninv += 1
689 if hasattr(wqp, 'wqp_alloc_bt'):
690 pc = unsigned(wqp.wqp_alloc_bt[opt_bt_idx])
691 pc_str = str(pc)
692 if pc > 0:
693 if pc_str in bt_summary:
694 bt_summary[pc_str] += 1
695 else:
696 bt_summary[pc_str] = 1
697 if not opt_summary:
698 print GetWaitqPrepostSummary(wqp)
699 if verbose:
700 sys.stderr.write('id: {:d}/{:d}... \r'.format(id, nelem))
701 id += 1
702 nused = nwq + npost + nrsvd
703 nfound = nused + nfree + ninv
704 print "\nFound {:d} objects: {:d} WQ, {:d} POST, {:d} RSVD, {:d} FREE".format(nfound, nwq, npost, nrsvd, nfree)
705 if (opt_type_filt == "" and opt_valid_only == 0) and (nused != table.used_elem):
706 print"\tWARNING: inconsistent state! Table reports {:d}/{:d} used elem, found {:d}/{:d}".format(table.used_elem, nelem, nused, nfound)
707 if len(bt_summary) > 0:
708 print "Link allocation BT (frame={:d})".format(opt_bt_idx)
709 for k,v in bt_summary.iteritems():
710 print "\t[{:d}] from: {:s}".format(v, GetSourceInformationForAddress(unsigned(k)))
711 # EndMacro: showallpreposts
712
713
714 @lldb_type_summary(['wq_prepost', 'wq_prepost *'])
715 @header("{:<18s} {:<18s} {:<19s} {:<10s} {:<1s} {:<4s} {:<10s} {:<20s}".format('addr','id','idx','gen','V','type','refcnt','info'))
716 def GetWaitqPrepostSummary(wqp):
717 if not wqp:
718 return
719 fmt_str = "{w: <#18x} {w.wqte.lt_id.id: <#18x} {w.wqte.lt_id.idx: <7d} (->{w.wqte.lt_next_idx: <7d}) {w.wqte.lt_id.generation: <#10x} {v: <1s} {t: <4s} {rcnt: <10d} "
720 type = WaitqTableElemType(wqp)
721 if type == "ELEM":
722 type = "WQ"
723 elif type == "LINK":
724 type = "POST"
725 v = "F"
726 if WaitqTableElemValid(wqp):
727 v = "T"
728 refcnt = WaitqTableElemRefcnt(wqp)
729 out_str = fmt_str.format(w=wqp, v=v, t=type, rcnt=refcnt)
730 if type == "WQ":
731 out_str += "wq:{0: <#18x}".format(unsigned(wqp.wqp_wq.wqp_wq_ptr))
732 elif type == "POST":
733 out_str += "next:{0: <#18x}, wqid:{1: <#18x}".format(wqp.wqp_post.wqp_next_id, wqp.wqp_post.wqp_wq_id)
734 post_wqp = GetWaitqPrepost(wqp.wqp_post.wqp_wq_id)[0]
735 if not WaitqTableElemValid(post_wqp):
736 out_str += "(<invalid>)"
737 else:
738 if WaitqTableElemType(post_wqp) != "ELEM":
739 out_str += "(!WQP_WQ?)"
740 else:
741 out_str += "({0: <#18x})".format(unsigned(post_wqp.wqp_wq.wqp_wq_ptr))
742 return out_str
743
744
745 # Macro: showprepost
746 @lldb_command('showprepost', "P:")
747 def ShowPrepost(cmd_args=None, cmd_options={}):
748 """ Print prepost structure summary
749
750 Note: you can pass either a complete ID (generation + index), or
751 just the index to the -P argument.
752
753 usage: showprepost [-P ID] [0xaddr]
754 -P {ID} : show prepost structure whose ID is {ID}
755 """
756 wqp = 0
757 if "-P" in cmd_options:
758 wqp, warn_str = GetWaitqPrepost(unsigned(kern.GetValueFromAddress(cmd_options["-P"], 'uint64_t *')))
759 if wqp == 0:
760 if warn_str != '':
761 raise LookupError(warn_str)
762 else:
763 raise ArgumentError("Invalid prepost ID {:s}".format(cmd_options["-P"]))
764 if wqp == 0:
765 if not cmd_args:
766 raise ArgumentError("Please pass the address of a prepost object")
767 wqp = kern.GetValueFromAddress(cmd_args[0], 'wq_prepost *')
768 if not wqp:
769 raise ArgumentError("Invalid prepost {:s}".format(cmd_args[0]))
770
771 print GetWaitqPrepostSummary.header
772 print GetWaitqPrepostSummary(wqp)
773 # EndMacro: showprepost
774
775
776 def WaitqPrepostFromObj(wqp, head_id, inv_ok, prepost_str, pp_arr = 0, depth = 0):
777 if pp_arr != 0:
778 pp_arr.append(wqp)
779 etype = WaitqTableElemType(wqp)
780 if not WaitqTableElemValid(wqp) and not inv_ok:
781 id = 0
782 if wqp:
783 id = wqp.wqte.lt_id.id
784 prepost_str.append("{0: <#18x}:{1: <18s}".format(id, "<invalid>"))
785 return
786 if etype == "ELEM": # WQP_WQ
787 prepost_str.append("{0: <#18x}:{1: <#18x}".format(wqp.wqte.lt_id.id, unsigned(wqp.wqp_wq.wqp_wq_ptr)))
788 return
789
790 post_wq = 0
791
792 if etype == "LINK": # WQP_POST
793 next_id = wqp.wqp_post.wqp_next_id
794 post_wq = GetWaitqPrepost(wqp.wqp_post.wqp_wq_id)[0]
795 if WaitqTableElemValid(post_wq):
796 if WaitqTableElemType(post_wq) != "ELEM":
797 prepost_str.append("{0: <#18x}:{1: <18s}".format(post_wq.wqte.lt_id.id, "<invalid post>"))
798 else:
799 prepost_str.append("{0: <#18x}:{1: <#18x}".format(wqp.wqte.lt_id.id, unsigned(post_wq.wqp_wq.wqp_wq_ptr)))
800 if next_id > 0 and next_id != head_id:
801 if depth >= 950:
802 prepost_str.append("{: <37s}".format("!recursion limit!"))
803 return
804 WaitqPrepostFromObj(GetWaitqPrepost(next_id)[0], head_id, inv_ok, prepost_str, pp_arr, depth + 1)
805 else: # "RSVD" or "FREE":
806 prepost_str.append("{0: <#18x} -> {1: <15d}".format(wqp.wqte.lt_id.id, wqp.wqte.lt_next_idx))
807 next_id = wqp.wqte.lt_next_idx
808 max_elem = int(kern.globals.g_prepost_table.nelem)
809 if hasattr(kern.globals, 'g_lt_idx_max'):
810 max_elem = unsigned(kern.globals.g_lt_idx_max)
811 if next_id < max_elem:
812 if depth >= 950:
813 prepost_str.append("{: <37s}".format("!recursion limit!"))
814 return
815 WaitqPrepostFromObj(GetWaitqPrepost(next_id)[0], head_id, inv_ok, prepost_str, pp_arr, depth + 1)
816 return
817
818 def GetPrepostChain(head_id, inv_ok = False, pp_arr = 0):
819 pp = []
820 if unsigned(head_id) == 0:
821 return [ "{0: <#18x}:{1: <18s}".format(head_id, "<invalid>") ]
822 wqp = GetWaitqPrepost(head_id)[0]
823 if wqp != 0:
824 WaitqPrepostFromObj(wqp, head_id, inv_ok, pp, pp_arr)
825 else:
826 return [ "{0: <#18x}:{1: <18s}".format(head_id, "<invalid>") ]
827 return pp
828
829 def GetWaitqPreposts(waitq):
830 if GetWaitqStateStr(waitq) != "SET":
831 return []
832 wqset = Cast(waitq, 'waitq_set *')
833 if wqset.wqset_prepost_id == 0:
834 return []
835 return GetPrepostChain(wqset.wqset_prepost_id)
836
837
838 # Macro: showprepostchain
839 @lldb_command('showprepostchain', "P:")
840 def ShowPrepostChain(cmd_args=None, cmd_options={}):
841 """ Follow a chain of preposts, printing each one.
842 Note that prepost chains are circular, so this will print
843 the entire chain given a single element.
844
845 Note: you can pass either a complete ID (generation + index), or
846 just the index to the -P argument.
847
848 usage: showprepostchain [-P ID] [0xaddr]
849 -P {ID} : start printing with the prepost whose ID is {ID}
850 """
851 wqp = 0
852 if "-P" in cmd_options:
853 wqp, warn_str = GetWaitqPrepost(unsigned(kern.GetValueFromAddress(cmd_options["-P"], 'uint64_t *')))
854 if wqp == 0:
855 if warn_str != '':
856 raise LookupError(warn_str)
857 else:
858 raise ArgumentError("Invalid prepost ID {:s}".format(cmd_options["-P"]))
859 if wqp == 0:
860 if not cmd_args:
861 raise ArgumentError("Please pass the address of a prepost object")
862 wqp = kern.GetValueFromAddress(cmd_args[0], 'wq_prepost *')
863 if not wqp:
864 raise ArgumentError("Invalid prepost {:s}".format(cmd_args[0]))
865
866 pp_arr = []
867 GetPrepostChain(wqp.wqte.lt_id.id, True, pp_arr)
868 pp_cnt = len(pp_arr)
869 idx = 0
870 nvalid = 0
871 ninvalid = 0
872 print GetWaitqPrepostSummary.header
873 while idx < pp_cnt:
874 print GetWaitqPrepostSummary(pp_arr[idx])
875 if pp_arr[idx] != 0:
876 type = WaitqTableElemType(pp_arr[idx])
877 if type == "LINK":
878 post_wqp = GetWaitqPrepost(pp_arr[idx].wqp_post.wqp_wq_id)[0]
879 if not WaitqTableElemValid(post_wqp):
880 ninvalid += 1
881 else:
882 nvalid += 1
883 else:
884 nvalid += 1
885 idx += 1
886 print "%s" % '-'*86
887 print "Total: {:d} ({:d} valid, {:d} invalid)".format(len(pp_arr), nvalid, ninvalid)
888 # EndMacro: showprepostchain
889
890
891 @lldb_type_summary(['waitq', 'waitq *'])
892 @header("{: <16s} {: <3s} {: <4s} {: <17s} {: <18s} {: <18s} {: <37s} {: <22s} {: <10s}".format('waitq', 'typ', 'bits', 'evtmask', 'setid', 'wq_wqp', 'preposts', 'member_of', 'threads'))
893 def GetWaitqSummary(waitq):
894 fmt_str = "{q: <16x} {state: <3s} {bits: <4s} {q.waitq_eventmask: <#17x} {setid: <#18x} {q.waitq_prepost_id: <#18x}"
895 th_str = []
896 if waitq.waitq_queue.next and waitq.waitq_queue.prev:
897 for thread in IterateLinkageChain(addressof(waitq.waitq_queue), 'thread *', 'wait_links'):
898 th_str.append("{: <18s} e:{: <#18x}".format(hex(thread), thread.wait_event))
899 else:
900 th_str.append("{: <39s}".format('<invalid (NULL) queue>'))
901 th_cnt = len(th_str)
902 set_str = GetWaitqSets(waitq)
903 set_cnt = len(set_str)
904 pp_str = GetWaitqPreposts(waitq)
905 pp_cnt = len(pp_str)
906 last_str = ''
907 idx = 0;
908 while idx < pp_cnt or idx < set_cnt or idx < th_cnt:
909 p = ""
910 s = ""
911 t = ""
912 if idx < pp_cnt:
913 p = pp_str[idx]
914 if idx < set_cnt:
915 s = set_str[idx]
916 if idx < th_cnt:
917 t = th_str[idx]
918 if idx == 0:
919 last_str += "{0: <37s} {1: <22s} {2: <39s}".format(p, s, t)
920 else:
921 last_str += "\n{0: <80s} {1: <37s} {2: <22s} {3: <39s}".format('', p, s, t)
922 idx += 1
923 if pp_cnt > 0 or set_cnt > 0 or th_cnt > 0:
924 last_str += "\n{:<80s} {: <37s} {: <22s} {: <39s}".format('', '-'*37, '-'*20, '-'*39)
925 last_str += "\n{0: <80s} {1: <37d} {2: <22d} {3: <39d}".format('', pp_cnt, set_cnt, th_cnt)
926
927 state = GetWaitqStateStr(waitq)
928 setid = 0
929 if state == "SET":
930 setid = Cast(waitq, 'waitq_set *').wqset_id
931 out_str = fmt_str.format(q=waitq, state=state, bits=GetWaitqBitsStr(waitq), setid=setid)
932 out_str += last_str
933 return out_str
934
935 # Macro: showwaitq
936 @lldb_command('showwaitq', "P:S:")
937 def ShowWaitq(cmd_args=None, cmd_options={}):
938 """ Print waitq structure summary.
939 Lookup the waitq either by address, by Set ID, or indirectly
940 through a prepost object that points to the waitq.
941
942 Note: you can pass either a complete ID (generation + index), or
943 just the index to the -P and -S arguments.
944
945 usage: showwaitq [-P PrePostID] [-S SetID] [0xaddr]
946 -P {ID} : prepost ID that points to a waitq
947 -S {ID} : waitq_set ID
948 """
949 waitq = 0
950 if "-P" in cmd_options:
951 wqp, warn_str = GetWaitqPrepost(unsigned(kern.GetValueFromAddress(cmd_options["-P"], 'uint64_t *')))
952 if wqp == 0:
953 if warn_str:
954 raise LookupError(warn_str)
955 else:
956 raise ArgumentError("Invalid prepost ID {:s}".format(cmd_options["-P"]))
957 if WaitqTableElemType(wqp) != "ELEM":
958 raise ArgumentError("Prepost ID {:s} points to a WQP_POST object, not a WQP_WQ!".format(cmd_options["-P"]))
959 waitq = wqp.wqp_wq.wqp_wq_ptr
960 if "-S" in cmd_options:
961 if waitq:
962 raise ArgumentError("Please pass only one of '-S' or '-P'!")
963 link, warn_str = GetWaitqLink(unsigned(kern.GetValueFromAddress(cmd_options["-S"],'uint64_t *')))
964 if not link:
965 if warn_str != '':
966 raise LookupError(warn_str)
967 else:
968 raise ArgumentError("Invalid link ID {:s}".format(cmd_options["-S"]))
969 if WaitqTableElemType(link) != "ELEM":
970 raise ArgumentError("Link ID {:s} points to a SLT_LINK object, not an SLT_WQS!".format(cmd_options["-S"]))
971 waitq = addressof(link.wql_wqs.wql_set.wqset_q)
972
973 if not waitq and not cmd_args:
974 raise ArgumentError("Please pass the address of a waitq!")
975 if not waitq:
976 waitq = kern.GetValueFromAddress(cmd_args[0], 'waitq *')
977 if not waitq:
978 raise ("Unknown arguments: %r %r" % (cmd_args, cmd_options))
979 print GetWaitqSummary.header
980 print GetWaitqSummary(waitq)
981 # EndMacro: showwaitq
982
983
984 # Macro: showglobalwaitqs
985 @lldb_command('showglobalwaitqs')
986 def ShowGlobalWaitqs(cmd_args=None):
987 """ Summarize global waitq usage
988 """
989 global kern
990 q = 0
991
992 print "Global waitq objects"
993 print GetWaitqSummary.header
994
995 while q < kern.globals.g_num_waitqs:
996 print GetWaitqSummary(addressof(kern.globals.global_waitqs[q]))
997 q = q + 1
998 # EndMacro: showglobalwaitqs
999
1000
1001 # Macro: showglobalqstats
1002 @lldb_command('showglobalqstats', "OF")
1003 def ShowGlobalQStats(cmd_args=None, cmd_options={}):
1004 """ Summarize global waitq statistics
1005
1006 usage: showglobalqstats [-O] [-F]
1007 -O : only output waitqs with outstanding waits
1008 -F : output as much backtrace as was recorded
1009 """
1010 global kern
1011 q = 0
1012
1013 if not hasattr(kern.globals, 'g_waitq_stats'):
1014 print "No waitq stats support (use DEVELOPMENT kernel)!"
1015 return
1016
1017 print "Global waitq stats"
1018 print "{0: <18s} {1: <8s} {2: <8s} {3: <8s} {4: <8s} {5: <8s} {6: <32s}".format('waitq', '#waits', '#wakes', '#diff', '#fails', '#clears', 'backtraces')
1019
1020 waiters_only = False
1021 full_bt = False
1022 if "-O" in cmd_options:
1023 waiters_only = True
1024 if "-F" in cmd_options:
1025 full_bt = True
1026
1027 fmt_str = "{q: <#18x} {stats.waits: <8d} {stats.wakeups: <8d} {diff: <8d} {stats.failed_wakeups: <8d} {stats.clears: <8d} {bt_str: <s}"
1028 while q < kern.globals.g_num_waitqs:
1029 waitq = kern.globals.global_waitqs[q]
1030 stats = kern.globals.g_waitq_stats[q]
1031 diff = stats.waits - stats.wakeups
1032 if diff == 0 and waiters_only:
1033 q = q + 1
1034 continue
1035 last_waitstr = ''
1036 last_wakestr = ''
1037 fw_str = ''
1038 if (stats.last_wait[0]):
1039 last_waitstr = GetSourceInformationForAddress(unsigned(stats.last_wait[0]))
1040 if (stats.last_wakeup[0]):
1041 last_wakestr = GetSourceInformationForAddress(unsigned(stats.last_wakeup[0]))
1042 if (stats.last_failed_wakeup[0]):
1043 fw_str = GetSourceInformationForAddress(unsigned(stats.last_failed_wakeup[0]))
1044
1045 if full_bt:
1046 f = 1
1047 while f < kern.globals.g_nwaitq_btframes:
1048 if stats.last_wait[f]:
1049 last_waitstr = "{0}->{1}".format(GetSourceInformationForAddress(unsigned(stats.last_wait[f])), last_waitstr)
1050 if stats.last_wakeup[f]:
1051 last_wakestr = "{0}->{1}".format(GetSourceInformationForAddress(unsigned(stats.last_wakeup[f])), last_wakestr)
1052 if stats.last_failed_wakeup[f]:
1053 fw_str = "{0}->{1}".format(GetSourceInformationForAddress(unsigned(stats.last_failed_wakeup[f])), fw_str)
1054 f = f + 1
1055 bt_str = ''
1056 if last_waitstr:
1057 bt_str += "wait : " + last_waitstr
1058 if last_wakestr:
1059 if bt_str:
1060 bt_str += "\n{0: <70s} ".format('')
1061 bt_str += "wake : " + last_wakestr
1062 if fw_str:
1063 if bt_str:
1064 bt_str += "\n{0: <70s} ".format('')
1065 bt_str += "fails: " + fw_str
1066
1067 print fmt_str.format(q=addressof(waitq), stats=stats, diff=diff, bt_str=bt_str)
1068 q = q + 1
1069 # EndMacro: showglobalqstats