tools/lldbmacros/kcdata.py

   1 #!/usr/bin/env python
   2 import sys
   3 import struct
   4 import mmap
   5 import json
   6 import cgitb
   7 import copy
   8 import re
   9 import base64
  10 import argparse
  11 import os
  12 import shlex
  13 import subprocess
  14 import logging
  15 import contextlib
  16 import base64
  17
  18 class Globals(object):
  19     pass
  20 G = Globals()
  21 G.accept_incomplete_data = False
  22 G.data_was_incomplete = False
  23
  24 kcdata_type_def = {
  25     'KCDATA_TYPE_INVALID':              0x0,
  26     'KCDATA_TYPE_STRING_DESC':          0x1,
  27     'KCDATA_TYPE_UINT32_DESC':          0x2,
  28     'KCDATA_TYPE_UINT64_DESC':          0x3,
  29     'KCDATA_TYPE_INT32_DESC':           0x4,
  30     'KCDATA_TYPE_INT64_DESC':           0x5,
  31     'KCDATA_TYPE_BINDATA_DESC':         0x6,
  32     'KCDATA_TYPE_ARRAY':                0x11,
  33     'KCDATA_TYPE_TYPEDEFINTION':        0x12,
  34     'KCDATA_TYPE_CONTAINER_BEGIN':      0x13,
  35     'KCDATA_TYPE_CONTIANER_END':        0x14,
  36
  37     'KCDATA_TYPE_ARRAY_PAD0':           0x20,
  38     'KCDATA_TYPE_ARRAY_PAD1':           0x21,
  39     'KCDATA_TYPE_ARRAY_PAD2':           0x22,
  40     'KCDATA_TYPE_ARRAY_PAD3':           0x23,
  41     'KCDATA_TYPE_ARRAY_PAD4':           0x24,
  42     'KCDATA_TYPE_ARRAY_PAD5':           0x25,
  43     'KCDATA_TYPE_ARRAY_PAD6':           0x26,
  44     'KCDATA_TYPE_ARRAY_PAD7':           0x27,
  45     'KCDATA_TYPE_ARRAY_PAD8':           0x28,
  46     'KCDATA_TYPE_ARRAY_PAD9':           0x29,
  47     'KCDATA_TYPE_ARRAY_PADa':           0x2a,
  48     'KCDATA_TYPE_ARRAY_PADb':           0x2b,
  49     'KCDATA_TYPE_ARRAY_PADc':           0x2c,
  50     'KCDATA_TYPE_ARRAY_PADd':           0x2d,
  51     'KCDATA_TYPE_ARRAY_PADe':           0x2e,
  52     'KCDATA_TYPE_ARRAY_PADf':           0x2f,
  53
  54     'KCDATA_TYPE_LIBRARY_LOADINFO':     0x30,
  55     'KCDATA_TYPE_LIBRARY_LOADINFO64':   0x31,
  56     'KCDATA_TYPE_TIMEBASE':             0x32,
  57     'KCDATA_TYPE_MACH_ABSOLUTE_TIME':   0x33,
  58     'KCDATA_TYPE_TIMEVAL':              0x34,
  59     'KCDATA_TYPE_USECS_SINCE_EPOCH':    0x35,
  60     'KCDATA_TYPE_PID':                  0x36,
  61     'KCDATA_TYPE_PROCNAME':             0x37,
  62     'KCDATA_TYPE_NESTED_KCDATA':        0x38,
  63
  64     'STACKSHOT_KCCONTAINER_TASK':       0x903,
  65     'STACKSHOT_KCCONTAINER_THREAD':     0x904,
  66     'STACKSHOT_KCTYPE_DONATING_PIDS':   0x907,
  67     'STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO': 0x908,
  68     'STACKSHOT_KCTYPE_KERN_STACKFRAME': 0x90A,
  69     'STACKSHOT_KCTYPE_KERN_STACKFRAME64': 0x90B,
  70     'STACKSHOT_KCTYPE_USER_STACKFRAME': 0x90C,
  71     'STACKSHOT_KCTYPE_USER_STACKFRAME64': 0x90D,
  72     'STACKSHOT_KCTYPE_BOOTARGS':        0x90E,
  73     'STACKSHOT_KCTYPE_OSVERSION':       0x90F,
  74     'STACKSHOT_KCTYPE_KERN_PAGE_SIZE':  0x910,
  75     'STACKSHOT_KCTYPE_JETSAM_LEVEL':    0x911,
  76     'STACKSHOT_KCTYPE_DELTA_SINCE_TIMESTAMP': 0x912,
  77     'STACKSHOT_KCTYPE_TASK_DELTA_SNAPSHOT': 0x940,
  78     'STACKSHOT_KCTYPE_THREAD_DELTA_SNAPSHOT': 0x941,
  79
  80     'STACKSHOT_KCTYPE_KERN_STACKLR':  0x913,
  81     'STACKSHOT_KCTYPE_KERN_STACKLR64':  0x914,
  82     'STACKSHOT_KCTYPE_USER_STACKLR':  0x915,
  83     'STACKSHOT_KCTYPE_USER_STACKLR64':  0x916,
  84     'STACKSHOT_KCTYPE_NONRUNNABLE_TIDS':  0x917,
  85     'STACKSHOT_KCTYPE_NONRUNNABLE_TASKS':  0x918,
  86     'STACKSHOT_KCTYPE_CPU_TIMES': 0x919,
  87     'STACKSHOT_KCTYPE_STACKSHOT_DURATION': 0x91a,
  88     'STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS': 0x91b,
  89     'STACKSHOT_KCTYPE_KERNELCACHE_LOADINFO': 0x91c,
  90     'STACKSHOT_KCTYPE_THREAD_WAITINFO' : 0x91d,
  91     'STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT' : 0x91e,
  92     'STACKSHOT_KCTYPE_THREAD_GROUP' : 0x91f,
  93     'STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT' : 0x920,
  94     'STACKSHOT_KCTYPE_JETSAM_COALITION' : 0x921,
  95     'STACKSHOT_KCTYPE_INSTRS_CYCLES' : 0x923,
  96
  97     'STACKSHOT_KCTYPE_THREAD_POLICY_VERSION': 0x922,
  98
  99
 100     'KCDATA_TYPE_BUFFER_END':      0xF19158ED,
 101
 102
 103     'TASK_CRASHINFO_EXTMODINFO':           0x801,
 104     'TASK_CRASHINFO_BSDINFOWITHUNIQID':    0x802,
 105     'TASK_CRASHINFO_TASKDYLD_INFO':        0x803,
 106     'TASK_CRASHINFO_UUID':                 0x804,
 107     'TASK_CRASHINFO_PID':                  0x805,
 108     'TASK_CRASHINFO_PPID':                 0x806,
 109
 110     # Don't want anyone using this.  It's struct rusage from whatever machine generated the data
 111     #'TASK_CRASHINFO_RUSAGE':               0x807,
 112     'Type_0x807':               0x807,
 113
 114     'TASK_CRASHINFO_RUSAGE_INFO':          0x808,
 115     'TASK_CRASHINFO_PROC_NAME':            0x809,
 116     'TASK_CRASHINFO_PROC_STARTTIME':       0x80B,
 117     'TASK_CRASHINFO_USERSTACK':            0x80C,
 118     'TASK_CRASHINFO_ARGSLEN':              0x80D,
 119     'TASK_CRASHINFO_EXCEPTION_CODES':      0x80E,
 120     'TASK_CRASHINFO_PROC_PATH':            0x80F,
 121     'TASK_CRASHINFO_PROC_CSFLAGS':         0x810,
 122     'TASK_CRASHINFO_PROC_STATUS':          0x811,
 123     'TASK_CRASHINFO_UID':                  0x812,
 124     'TASK_CRASHINFO_GID':                  0x813,
 125     'TASK_CRASHINFO_PROC_ARGC':            0x814,
 126     'TASK_CRASHINFO_PROC_FLAGS':           0x815,
 127     'TASK_CRASHINFO_CPUTYPE':              0x816,
 128     'TASK_CRASHINFO_WORKQUEUEINFO':        0x817,
 129     'TASK_CRASHINFO_RESPONSIBLE_PID':      0x818,
 130     'TASK_CRASHINFO_DIRTY_FLAGS':          0x819,
 131     'TASK_CRASHINFO_CRASHED_THREADID':     0x81A,
 132     'TASK_CRASHINFO_COALITION_ID':         0x81B,
 133     'EXIT_REASON_SNAPSHOT':                0x1001,
 134     'EXIT_REASON_USER_DESC':               0x1002,
 135     'EXIT_REASON_USER_PAYLOAD':            0x1003,
 136     'EXIT_REASON_CODESIGNING_INFO':        0x1004,
 137     'EXIT_REASON_WORKLOOP_ID':             0x1005,
 138     'EXIT_REASON_DISPATCH_QUEUE_NO':       0x1006,
 139     'KCDATA_BUFFER_BEGIN_CRASHINFO':       0xDEADF157,
 140     'KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT': 0xDE17A59A,
 141     'KCDATA_BUFFER_BEGIN_STACKSHOT':       0x59a25807,
 142     'KCDATA_BUFFER_BEGIN_OS_REASON':       0x53A20900,
 143     'KCDATA_BUFFER_BEGIN_XNUPOST_CONFIG':  0x1E21C09F
 144 }
 145 kcdata_type_def_rev = dict((v, k) for k, v in kcdata_type_def.iteritems())
 146
 147 KNOWN_TYPES_COLLECTION = {}
 148
 149 KNOWN_TOPLEVEL_CONTAINER_TYPES = ()
 150
 151 def enum(**args):
 152     return type('enum', (), args)
 153
 154 KCSUBTYPE_TYPE = enum(KC_ST_CHAR=1, KC_ST_INT8=2, KC_ST_UINT8=3, KC_ST_INT16=4, KC_ST_UINT16=5, KC_ST_INT32=6, KC_ST_UINT32=7, KC_ST_INT64=8, KC_ST_UINT64=9)
 155
 156
 157 LEGAL_OLD_STYLE_ARRAY_TYPE_NAMES = ['KCDATA_TYPE_LIBRARY_LOADINFO',
 158                                     'KCDATA_TYPE_LIBRARY_LOADINFO64',
 159                                     'STACKSHOT_KCTYPE_KERN_STACKFRAME',
 160                                     'STACKSHOT_KCTYPE_USER_STACKFRAME',
 161                                     'STACKSHOT_KCTYPE_KERN_STACKFRAME64',
 162                                     'STACKSHOT_KCTYPE_USER_STACKFRAME64',
 163                                     'STACKSHOT_KCTYPE_DONATING_PIDS',
 164                                     'STACKSHOT_KCTYPE_THREAD_DELTA_SNAPSHOT']
 165
 166 KCDATA_FLAGS_STRUCT_PADDING_MASK = 0xf
 167 KCDATA_FLAGS_STRUCT_HAS_PADDING = 0x80
 168
 169 class KCSubTypeElement(object):
 170     """convert kcdata_subtype_descriptor to """
 171     _unpack_formats = (None, 'c', 'b', 'B', 'h', 'H', 'i', 'I', 'q', 'Q')
 172     _ctypes = ('Unknown', 'char', 'int8_t', 'uint8_t', 'int16_t', 'uint16_t', 'int32_t', 'uint32_t', 'int64_t', 'uint64_t')
 173
 174     def __init__(self, st_name, st_type, st_size, st_offset=0, st_flag=0, custom_repr=None):
 175         self.name = st_name
 176         self.offset = st_offset
 177         self.type_id = st_type
 178         if st_type <= 0 or st_type > KCSUBTYPE_TYPE.KC_ST_UINT64:
 179             raise ValueError("Invalid type passed %d" % st_type)
 180         self.unpack_fmt = KCSubTypeElement._unpack_formats[self.type_id]
 181         self.size = st_size
 182         self.totalsize = st_size
 183         self.count = 1
 184         self.is_array_type = False
 185         self.custom_JsonRepr = custom_repr
 186         if (st_flag & 0x1) == 0x1:
 187             self.is_array_type = True
 188             self.size = st_size & 0xffff
 189             self.count = (st_size >> 16) & 0xffff
 190             self.totalsize = self.size * self.count
 191
 192     @staticmethod
 193     def GetSizeForArray(el_count, el_size):
 194         return ((el_count & 0xffff) << 16) | (el_size & 0xffff)
 195
 196     @staticmethod
 197     def FromBinaryTypeData(byte_data):
 198         (st_flag, st_type, st_offset, st_size, st_name) = struct.unpack_from('=BBHI32s', byte_data)
 199         st_name = st_name.rstrip('\x00')
 200         return KCSubTypeElement(st_name, st_type, st_size, st_offset, st_flag)
 201
 202     @staticmethod
 203     def FromBasicCtype(st_name, st_type, st_offset=0):
 204         if st_type <= 0 or st_type > KCSUBTYPE_TYPE.KC_ST_UINT64:
 205             raise ValueError("Invalid type passed %d" % st_type)
 206         st_size = struct.calcsize(KCSubTypeElement._unpack_formats[st_type])
 207         st_flag = 0
 208         retval = KCSubTypeElement(st_name, st_type, st_size, st_offset, st_flag, KCSubTypeElement._get_naked_element_value)
 209         return retval
 210
 211     @staticmethod
 212     def FromKCSubTypeElement(other, name_override=''):
 213         _copy = copy.copy(other)
 214         if name_override:
 215             _copy.name = name_override
 216         return copy
 217
 218     def GetName(self):
 219         return self.name
 220
 221     def GetTotalSize(self):
 222         return self.totalsize
 223
 224     def GetValueAsString(self, base_data, array_pos=0):
 225         return str(self.GetValue(base_data, array_pos))
 226
 227     def GetValue(self, base_data, array_pos=0):
 228         return struct.unpack_from(self.unpack_fmt, base_data[self.offset + (array_pos * self.size):])[0]
 229
 230     @staticmethod
 231     def _get_naked_element_value(elementValue, elementName):
 232         return json.dumps(elementValue)
 233
 234     def __str__(self):
 235         if self.is_array_type:
 236             return '[%d,%d] %s  %s[%d];' % (self.offset, self.totalsize, self.GetCTypeDesc(), self.name, self.count)
 237         return '[%d,%d] %s  %s;' % (self.offset, self.totalsize, self.GetCTypeDesc(), self.name)
 238
 239     def __repr__(self):
 240         return str(self)
 241
 242     def GetCTypeDesc(self):
 243         return KCSubTypeElement._ctypes[self.type_id]
 244
 245     def GetStringRepr(self, base_data):
 246         if not self.is_array_type:
 247             return self.GetValueAsString(base_data)
 248         if self.type_id == KCSUBTYPE_TYPE.KC_ST_CHAR:
 249             str_len = self.count
 250             if len(base_data) < str_len:
 251                 str_len = len(base_data)
 252             str_arr = []
 253             for i in range(str_len):
 254                 _v = self.GetValue(base_data, i)
 255                 if ord(_v) == 0:
 256                     break
 257                 str_arr.append(self.GetValueAsString(base_data, i))
 258             return json.dumps(''.join(str_arr))
 259
 260         count = self.count
 261         if count > len(base_data)/self.size:
 262             count = len(base_data)/self.size
 263
 264         o = '[' + ','.join([self.GetValueAsString(base_data, i) for i in range(count)]) + ']'
 265
 266         return o
 267
 268     def GetJsonRepr(self, base_data, flags=0):
 269         if (flags & (KCDATA_FLAGS_STRUCT_HAS_PADDING | KCDATA_FLAGS_STRUCT_PADDING_MASK)) != 0:
 270             padding = (flags & KCDATA_FLAGS_STRUCT_PADDING_MASK)
 271             if padding:
 272                 base_data = base_data[:-padding]
 273         if self.custom_JsonRepr:
 274             if self.is_array_type:
 275                 e_data = [self.GetValue(base_data, i) for i in range(self.count)]
 276             else:
 277                 e_data = self.GetValue(base_data)
 278             return self.custom_JsonRepr(e_data, self.name)
 279         return self.GetStringRepr(base_data)
 280
 281     def sizeof(self):
 282         return self.totalsize
 283
 284     def ShouldSkip(self, data):
 285         return len(data) < self.offset + self.totalsize
 286
 287     def ShouldMerge(self):
 288         return False
 289
 290
 291 class KCTypeDescription(object):
 292     def __init__(self, t_type_id, t_elements=[], t_name='anon', custom_repr=None, legacy_size=None, merge=False):
 293         self.type_id = t_type_id
 294         self.elements = t_elements
 295         self.name = t_name
 296         self.totalsize = 0
 297         self.custom_JsonRepr = custom_repr
 298         self.legacy_size = legacy_size
 299         self.merge = merge
 300         for e in self.elements:
 301             self.totalsize += e.GetTotalSize()
 302
 303     def ValidateData(self, base_data):
 304         if len(base_data) >= self.totalsize:
 305             return True
 306         return False
 307
 308     def GetTypeID(self):
 309         return self.type_id
 310
 311     def GetName(self):
 312         return self.name
 313
 314     def __str__(self):
 315         o = '%s {\n\t' % self.name + "\n\t".join([str(e) for e in self.elements]) + '\n};'
 316         return o
 317
 318     @staticmethod
 319     def FromKCTypeDescription(other, t_type_id, t_name):
 320         retval = KCTypeDescription(t_type_id, other.elements, t_name, other.custom_JsonRepr)
 321         return retval
 322
 323     def ShouldMerge(self):
 324         return self.merge
 325
 326     def GetJsonRepr(self, base_data, flags):
 327         if (flags & (KCDATA_FLAGS_STRUCT_HAS_PADDING | KCDATA_FLAGS_STRUCT_PADDING_MASK)) != 0:
 328             padding = (flags & KCDATA_FLAGS_STRUCT_PADDING_MASK)
 329             if padding:
 330                 base_data = base_data[:-padding]
 331         elif self.legacy_size and len(base_data) == self.legacy_size + ((-self.legacy_size) & 0xf):
 332             base_data = base_data[:self.legacy_size]
 333         if self.custom_JsonRepr:
 334             return self.custom_JsonRepr([e.GetValue(base_data) for e in self.elements])
 335         o = ", ".join(['"%s": %s' % (e.GetName(), e.GetJsonRepr(base_data)) for e in self.elements if not e.ShouldSkip(base_data)])
 336         if not self.merge:
 337             o = '{' + o + '}'
 338         return o
 339
 340     def sizeof(self):
 341         return max(st.totalsize + st.offset for st in self.elements)
 342
 343
 344 def GetTypeNameForKey(k):
 345     retval = "0x%x" % k
 346     if k in KNOWN_TYPES_COLLECTION:
 347         retval = KNOWN_TYPES_COLLECTION[k].GetName()
 348     elif k in kcdata_type_def_rev:
 349         retval = kcdata_type_def_rev[k]
 350     return retval
 351
 352
 353 def GetTypeForName(n):
 354     ret = 0
 355     if n in kcdata_type_def:
 356         ret = kcdata_type_def[n]
 357     return ret
 358
 359
 360 LEGAL_OLD_STYLE_ARRAY_TYPES = map(GetTypeForName, LEGAL_OLD_STYLE_ARRAY_TYPE_NAMES)
 361
 362 kcdata_type_def_rev[GetTypeForName('KCDATA_BUFFER_BEGIN_STACKSHOT')] = 'kcdata_stackshot'
 363 kcdata_type_def_rev[GetTypeForName('KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT')] = 'kcdata_delta_stackshot'
 364 kcdata_type_def_rev[GetTypeForName('KCDATA_BUFFER_BEGIN_CRASHINFO')] = 'kcdata_crashinfo'
 365 kcdata_type_def_rev[GetTypeForName('KCDATA_BUFFER_BEGIN_OS_REASON')] = 'kcdata_reason'
 366 kcdata_type_def_rev[GetTypeForName('STACKSHOT_KCCONTAINER_TASK')] = 'task_snapshots'
 367 kcdata_type_def_rev[GetTypeForName('STACKSHOT_KCCONTAINER_THREAD')] = 'thread_snapshots'
 368 kcdata_type_def_rev[GetTypeForName('KCDATA_BUFFER_BEGIN_XNUPOST_CONFIG')] = 'xnupost_testconfig'
 369
 370 class Indent(object):
 371     def __init__(self):
 372         self.n = 0
 373     def __call__(self, end=False):
 374         if end:
 375             return " " * (self.n-4)
 376         else:
 377             return " " * self.n
 378     @contextlib.contextmanager
 379     def indent(self):
 380         self.n += 4
 381         try:
 382             yield
 383         finally:
 384             self.n -= 4
 385
 386 INDENT = Indent()
 387
 388 class KCObject(object):
 389
 390     def __init__(self, type_code, data, offset, flags=0):
 391
 392         self.i_type = type_code
 393         self.i_data = data
 394         self.offset = offset
 395         self.i_size = len(data)
 396         self.i_flags = flags
 397         self.obj_collection = []
 398         self.obj = {}
 399         self.is_container_type = False
 400         self.is_array_type = False
 401         self.is_naked_type = False
 402         self.nested_kcdata = None
 403         self.i_name = GetTypeNameForKey(type_code)
 404
 405         self.ParseData()
 406
 407         if self.i_type == GetTypeForName('KCDATA_TYPE_CONTAINER_BEGIN'):
 408             self.__class__ = KCContainerObject
 409
 410         if self.i_type in KNOWN_TOPLEVEL_CONTAINER_TYPES:
 411             self.__class__ = KCBufferObject
 412
 413         self.InitAfterParse()
 414
 415     def __str__(self):
 416         return "<KCObject at 0x%x>" % self.offset
 417
 418     def InitAfterParse(self):
 419         pass
 420
 421     @staticmethod
 422     def FromKCItem(kcitem):
 423         return KCObject(kcitem.i_type, kcitem.i_data, kcitem.i_offset, kcitem.i_flags)
 424
 425     def IsContainerEnd(self):
 426         return self.i_type == GetTypeForName('KCDATA_TYPE_CONTIANER_END')
 427
 428     def IsBufferEnd(self):
 429         return self.i_type == GetTypeForName('KCDATA_TYPE_BUFFER_END')
 430
 431     def IsArray(self):
 432         return self.is_array_type
 433
 434     def ShouldMerge(self):
 435         if self.nested_kcdata:
 436             return True
 437         elif not self.is_array_type and self.i_type in KNOWN_TYPES_COLLECTION:
 438             return KNOWN_TYPES_COLLECTION[self.i_type].ShouldMerge()
 439         else:
 440             return False
 441
 442     def GetJsonRepr(self):
 443         if self.is_array_type:
 444             return '[' + ', '.join([i.GetJsonRepr() for i in self.obj_collection]) + ']'
 445         if self.i_type in KNOWN_TYPES_COLLECTION:
 446             return KNOWN_TYPES_COLLECTION[self.i_type].GetJsonRepr(self.i_data, self.i_flags)
 447         if self.is_naked_type:
 448             return json.dumps(self.obj)
 449         if self.nested_kcdata:
 450             return self.nested_kcdata.GetJsonRepr()
 451
 452         raise NotImplementedError("Broken GetJsonRepr implementation")
 453
 454     def ParseData(self):
 455
 456
 457         if self.i_type == GetTypeForName('KCDATA_TYPE_CONTAINER_BEGIN'):
 458             self.obj['uniqID'] = self.i_flags
 459             self.i_name = str(self.obj['uniqID'])
 460             self.obj['typeID'] = struct.unpack_from('I', self.i_data)[0]
 461             logging.info("0x%08x: %sCONTAINER: %s(%x)" % (self.offset, INDENT(), GetTypeNameForKey(self.obj['typeID']), self.i_flags))
 462
 463         elif self.i_type in (KNOWN_TOPLEVEL_CONTAINER_TYPES):
 464             self.obj['uniqID'] = self.i_name
 465             self.obj['typeID'] = self.i_type
 466             logging.info("0x%08x: %s%s" % (self.offset, INDENT(), self.i_name))
 467
 468         elif self.i_type == GetTypeForName('KCDATA_TYPE_CONTIANER_END'):
 469             self.obj['uniqID'] = self.i_flags
 470             logging.info("0x%08x: %sEND" % (self.offset, INDENT(end=True)))
 471
 472         elif self.i_type == GetTypeForName('KCDATA_TYPE_BUFFER_END'):
 473             self.obj = ''
 474             logging.info("0x%08x: %sEND_BUFFER" % (self.offset, INDENT(end=True)))
 475
 476         elif self.i_type == GetTypeForName('KCDATA_TYPE_UINT32_DESC'):
 477             self.is_naked_type = True
 478             u_d = struct.unpack_from('32sI', self.i_data)
 479             self.i_name = u_d[0].strip(chr(0))
 480             self.obj = u_d[1]
 481             logging.info("0x%08x: %s%s" % (self.offset, INDENT(), self.i_name))
 482
 483         elif self.i_type == GetTypeForName('KCDATA_TYPE_UINT64_DESC'):
 484             self.is_naked_type = True
 485             u_d = struct.unpack_from('32sQ', self.i_data)
 486             self.i_name = u_d[0].strip(chr(0))
 487             self.obj = u_d[1]
 488             logging.info("0x%08x: %s%s" % (self.offset, INDENT(), self.i_name))
 489
 490         elif self.i_type == GetTypeForName('KCDATA_TYPE_TYPEDEFINTION'):
 491             self.is_naked_type = True
 492             u_d = struct.unpack_from('II32s', self.i_data)
 493             self.obj['name'] = u_d[2].split(chr(0))[0]
 494             self.i_name = "typedef[%s]" % self.obj['name']
 495             self.obj['typeID'] = u_d[0]
 496             self.obj['numOfFields'] = u_d[1]
 497             element_arr = []
 498             for i in range(u_d[1]):
 499                 e = KCSubTypeElement.FromBinaryTypeData(self.i_data[40+(i*40):])
 500                 #print str(e)
 501                 element_arr.append(e)
 502             type_desc = KCTypeDescription(u_d[0], element_arr, self.obj['name'])
 503             #print str(type_desc)
 504             self.obj['fields'] = [str(e) for e in element_arr]
 505             KNOWN_TYPES_COLLECTION[type_desc.GetTypeID()] = type_desc
 506             logging.info("0x%08x: %s%s" % (self.offset, INDENT(), self.i_name))
 507
 508         elif self.i_type == GetTypeForName('KCDATA_TYPE_ARRAY'):
 509             self.is_array_type = True
 510             e_t = (self.i_flags >> 32) & 0xffffffff
 511             if e_t not in LEGAL_OLD_STYLE_ARRAY_TYPES:
 512                 raise Exception, "illegal old-style array type: %s (0x%x)" % (GetTypeNameForKey(e_t), e_t)
 513             e_c = self.i_flags & 0xffffffff
 514             e_s = KNOWN_TYPES_COLLECTION[e_t].sizeof()
 515             if e_s * e_c > self.i_size:
 516                 raise Excpetion, "array too small for its count"
 517             self.obj['typeID'] = e_t
 518             self.i_name = GetTypeNameForKey(e_t)
 519             self.i_type = e_t
 520             self.obj['numOfElements'] = e_c
 521             self.obj['sizeOfElement'] = e_s
 522             logging.info("0x%08x: %sARRAY: %s" % (self.offset, INDENT(), self.i_name))
 523             #populate the array here by recursive creation of KCObject
 524             with INDENT.indent():
 525                 for _i in range(e_c):
 526                     _o = KCObject(e_t, self.i_data[(_i * e_s):(_i * e_s) + e_s], self.offset + _i*e_s)
 527                     self.obj_collection.append(_o)
 528
 529         elif self.i_type >= GetTypeForName('KCDATA_TYPE_ARRAY_PAD0') and self.i_type <= GetTypeForName('KCDATA_TYPE_ARRAY_PADf'):
 530             self.is_array_type = True
 531             e_t = (self.i_flags >> 32) & 0xffffffff
 532             e_c = self.i_flags & 0xffffffff
 533             e_s = (self.i_size - (self.i_type & 0xf)) / e_c if e_c != 0 else None
 534             self.obj['typeID'] = e_t
 535             self.i_name = GetTypeNameForKey(e_t)
 536             self.i_type = e_t
 537             self.obj['numOfElements'] = e_c
 538             self.obj['sizeOfElement'] = e_s
 539             logging.info("0x%08x: %sARRAY: %s" % (self.offset, INDENT(), self.i_name))
 540             #populate the array here by recursive creation of KCObject
 541             with INDENT.indent():
 542                 for _i in range(e_c):
 543                     _o = KCObject(e_t, self.i_data[(_i * e_s):(_i * e_s) + e_s], self.offset + _i*e_s)
 544                     self.obj_collection.append(_o)
 545
 546         elif self.i_type == GetTypeForName('KCDATA_TYPE_NESTED_KCDATA'):
 547             logging.info("0x%08x: %sNESTED_KCDATA" % (self.offset, INDENT()))
 548             with INDENT.indent():
 549                 nested_iterator = kcdata_item_iterator(self.i_data[:self.i_size])
 550                 nested_buffer = KCObject.FromKCItem(nested_iterator.next())
 551                 if not isinstance(nested_buffer, KCBufferObject):
 552                     raise Exception, "nested buffer isn't a KCBufferObject"
 553                 nested_buffer.ReadItems(nested_iterator)
 554             self.nested_kcdata = nested_buffer
 555
 556         elif self.i_type in KNOWN_TYPES_COLLECTION:
 557             self.i_name = KNOWN_TYPES_COLLECTION[self.i_type].GetName()
 558             self.is_naked_type = True
 559             logging.info("0x%08x: %s%s" % (self.offset, INDENT(), self.i_name))
 560         else:
 561             self.is_naked_type = True
 562             #self.obj = "data of len %d" % len(self.i_data)
 563             #self.obj = ''.join(["%x" % ki for ki in struct.unpack('%dB' % len(self.i_data), self.i_data)])
 564             self.obj = map(ord, self.i_data)
 565             logging.info("0x%08x: %s%s" % (self.offset, INDENT(), self.i_name))
 566
 567
 568 class KCContainerObject(KCObject):
 569     def __init__(self, *args, **kwargs):
 570         assert False
 571
 572     def InitAfterParse(self):
 573         self.obj_container_dict = {}
 574         self.obj_nested_objs = {}
 575
 576     def ShouldMerge(self):
 577         return True
 578
 579     def GetJsonRepr(self):
 580         # o = '"%s"' % self.obj['uniqID'] + ' : { "typeID" : %d ,' % self.obj['typeID']
 581         o = '"%s"' % self.obj['uniqID'] + ' : { '
 582         for (k, v) in self.obj_container_dict.items():
 583             if v.ShouldMerge():
 584                 o += v.GetJsonRepr() + ","
 585             else:
 586                 o += ' "%s" : ' % k + v.GetJsonRepr() + ","
 587
 588         for (k, v) in self.obj_nested_objs.items():
 589             o += '"%s" : {' % k + ",".join([vi.GetJsonRepr() for vi in v.values()]) + "} ,"
 590
 591         o = o.rstrip(',') + "}"
 592
 593         return o
 594
 595     def AddObject(self, kco):
 596         assert not kco.IsContainerEnd()
 597         if isinstance(kco, KCContainerObject):
 598             type_name = GetTypeNameForKey(kco.obj['typeID'])
 599             if type_name not in self.obj_nested_objs:
 600                 self.obj_nested_objs[type_name] = {}
 601             self.obj_nested_objs[type_name][kco.i_name] = kco
 602             return
 603         if kco.i_name in self.obj_container_dict:
 604             if kco.IsArray() and self.obj_container_dict[kco.i_name].IsArray():
 605                 self.obj_container_dict[kco.i_name].obj_collection.extend( kco.obj_collection )
 606         else:
 607             self.obj_container_dict[kco.i_name] = kco
 608
 609     def IsEndMarker(self, o):
 610         if not o.IsContainerEnd():
 611             return False
 612         if o.i_flags != self.i_flags:
 613             raise Exception, "container end marker doesn't match"
 614         return True
 615
 616     no_end_message = "could not find container end marker"
 617
 618     def ReadItems(self, iterator):
 619         found_end = False
 620         with INDENT.indent():
 621             for i in iterator:
 622                 o = KCObject.FromKCItem(i)
 623                 if self.IsEndMarker(o):
 624                     found_end = True
 625                     break
 626                 if o.IsBufferEnd():
 627                     break
 628                 if isinstance(o, KCContainerObject):
 629                     o.ReadItems(iterator)
 630                 self.AddObject(o)
 631         if not found_end:
 632             if G.accept_incomplete_data:
 633                 if not G.data_was_incomplete:
 634                     print >>sys.stderr, "kcdata.py WARNING: data is incomplete!"
 635                     G.data_was_incomplete = True
 636             else:
 637                 raise Exception, self.no_end_message
 638
 639
 640
 641 class KCBufferObject(KCContainerObject):
 642
 643     def IsEndMarker(self,o):
 644         if o.IsContainerEnd():
 645             raise Exception, "container end marker at the toplevel"
 646         return o.IsBufferEnd()
 647
 648     no_end_message = "could not find buffer end marker"
 649
 650
 651 class KCData_item:
 652     """ a basic kcdata_item type object.
 653     """
 654     header_size = 16  # (uint32_t + uint32_t + uint64_t)
 655
 656     def __init__(self, item_type, item_size, item_flags, item_data):
 657         self.i_type = item_type
 658         self.i_size = item_size
 659         self.i_flags = item_flags
 660         self.i_data = item_data
 661         self.i_offset = None
 662
 663     def __init__(self, barray, pos=0):
 664         """ create an object by parsing data from bytes array
 665             returns : obj - if data is readable
 666                       raises ValueError if something is not ok.
 667         """
 668         self.i_type = struct.unpack('I', barray[pos:pos+4])[0]     # int.from_bytes(barray[pos:pos+4])
 669         self.i_size = struct.unpack('I', barray[pos+4:pos+8])[0]   # int.from_bytes(barray[pos+4:pos+8])
 670         self.i_flags = struct.unpack('Q', barray[pos+8:pos+16])[0]  # int.from_bytes(barray[pos+8:pos+16])
 671         self.i_data = barray[pos+16: (pos + 16 + self.i_size)]
 672         self.i_offset = pos
 673
 674     def __len__(self):
 675         return self.i_size + KCData_item.header_size
 676
 677     def GetHeaderDescription(self):
 678         outs = "type: 0x%x size: 0x%x flags: 0x%x  (%s)" % (self.i_type, self.i_size, self.i_flags, GetTypeNameForKey(self.i_type))
 679         if not self.i_offset is None:
 680             outs = "pos: 0x%x" % self.i_offset + outs
 681         return outs
 682
 683     def __str__(self):
 684         return self.GetHeaderDescription()
 685
 686 def kcdata_item_iterator(data):
 687     file_len = len(data)
 688     curpos = 0
 689     while curpos < file_len:
 690         item = KCData_item(data, curpos)
 691         yield item
 692         curpos += len(item)
 693
 694 def _get_data_element(elementValues):
 695     return json.dumps(elementValues[-1])
 696
 697 KNOWN_TOPLEVEL_CONTAINER_TYPES = map(GetTypeForName, ('KCDATA_BUFFER_BEGIN_CRASHINFO', 'KCDATA_BUFFER_BEGIN_STACKSHOT', 'KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT', 'KCDATA_BUFFER_BEGIN_OS_REASON','KCDATA_BUFFER_BEGIN_XNUPOST_CONFIG'))
 698
 699 KNOWN_TYPES_COLLECTION[GetTypeForName('KCDATA_TYPE_UINT32_DESC')] = KCTypeDescription(GetTypeForName('KCDATA_TYPE_UINT32_DESC'), (
 700     KCSubTypeElement('desc', KCSUBTYPE_TYPE.KC_ST_CHAR, KCSubTypeElement.GetSizeForArray(32, 1), 0, 1),
 701     KCSubTypeElement('data', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 32, 0)
 702 ),
 703     'KCDATA_TYPE_UINT32_DESC',
 704     _get_data_element
 705 )
 706
 707 KNOWN_TYPES_COLLECTION[GetTypeForName('KCDATA_TYPE_UINT64_DESC')] = KCTypeDescription(GetTypeForName('KCDATA_TYPE_UINT64_DESC'), (
 708     KCSubTypeElement('desc', KCSUBTYPE_TYPE.KC_ST_CHAR, KCSubTypeElement.GetSizeForArray(32, 1), 0, 1),
 709     KCSubTypeElement('data', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 32, 0)
 710 ),
 711     'KCDATA_TYPE_UINT64_DESC',
 712     _get_data_element
 713 )
 714
 715 KNOWN_TYPES_COLLECTION[GetTypeForName('KCDATA_TYPE_TIMEBASE')] = KCTypeDescription(GetTypeForName('KCDATA_TYPE_TIMEBASE'), (
 716     KCSubTypeElement('numer', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 0, 0),
 717     KCSubTypeElement('denom', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 4, 0)
 718 ),
 719     'mach_timebase_info'
 720 )
 721
 722
 723 STACKSHOT_IO_NUM_PRIORITIES = 4
 724 KNOWN_TYPES_COLLECTION[0x901] = KCTypeDescription(0x901, (
 725     KCSubTypeElement.FromBasicCtype('ss_disk_reads_count', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),
 726     KCSubTypeElement.FromBasicCtype('ss_disk_reads_size', KCSUBTYPE_TYPE.KC_ST_UINT64, 8),
 727     KCSubTypeElement.FromBasicCtype('ss_disk_writes_count', KCSUBTYPE_TYPE.KC_ST_UINT64, 16),
 728     KCSubTypeElement.FromBasicCtype('ss_disk_writes_size', KCSUBTYPE_TYPE.KC_ST_UINT64, 24),
 729     KCSubTypeElement('ss_io_priority_count', KCSUBTYPE_TYPE.KC_ST_UINT64, KCSubTypeElement.GetSizeForArray(STACKSHOT_IO_NUM_PRIORITIES, 8), 32, 1),
 730     KCSubTypeElement('ss_io_priority_size', KCSUBTYPE_TYPE.KC_ST_UINT64, KCSubTypeElement.GetSizeForArray(STACKSHOT_IO_NUM_PRIORITIES, 8), 32 + (STACKSHOT_IO_NUM_PRIORITIES * 8), 1),
 731     KCSubTypeElement.FromBasicCtype('ss_paging_count', KCSUBTYPE_TYPE.KC_ST_UINT64, 32 + 2 * (STACKSHOT_IO_NUM_PRIORITIES * 8)),
 732     KCSubTypeElement.FromBasicCtype('ss_paging_size', KCSUBTYPE_TYPE.KC_ST_UINT64, 40 + 2 * (STACKSHOT_IO_NUM_PRIORITIES * 8)),
 733     KCSubTypeElement.FromBasicCtype('ss_non_paging_count', KCSUBTYPE_TYPE.KC_ST_UINT64, 48 + 2 * (STACKSHOT_IO_NUM_PRIORITIES * 8)),
 734     KCSubTypeElement.FromBasicCtype('ss_non_paging_size', KCSUBTYPE_TYPE.KC_ST_UINT64, 56 + 2 * (STACKSHOT_IO_NUM_PRIORITIES * 8)),
 735     KCSubTypeElement.FromBasicCtype('ss_data_count', KCSUBTYPE_TYPE.KC_ST_UINT64, 64 + 2 * (STACKSHOT_IO_NUM_PRIORITIES * 8)),
 736     KCSubTypeElement.FromBasicCtype('ss_data_size', KCSUBTYPE_TYPE.KC_ST_UINT64, 72 + 2 * (STACKSHOT_IO_NUM_PRIORITIES * 8)),
 737     KCSubTypeElement.FromBasicCtype('ss_metadata_count', KCSUBTYPE_TYPE.KC_ST_UINT64, 80 + 2 * (STACKSHOT_IO_NUM_PRIORITIES * 8)),
 738     KCSubTypeElement.FromBasicCtype('ss_metadata_size', KCSUBTYPE_TYPE.KC_ST_UINT64, 88 + 2 * (STACKSHOT_IO_NUM_PRIORITIES * 8))
 739 ),
 740     'io_statistics'
 741 )
 742
 743 KNOWN_TYPES_COLLECTION[0x902] = KCTypeDescription(0x902, (
 744     KCSubTypeElement('snapshot_magic', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 4 * 0, 0),
 745     KCSubTypeElement('free_pages', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 4 * 1, 0),
 746     KCSubTypeElement('active_pages', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 4 * 2, 0),
 747     KCSubTypeElement('inactive_pages', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 4 * 3, 0),
 748     KCSubTypeElement('purgeable_pages', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 4 * 4, 0),
 749     KCSubTypeElement('wired_pages', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 4 * 5, 0),
 750     KCSubTypeElement('speculative_pages', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 4 * 6, 0),
 751     KCSubTypeElement('throttled_pages', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 4 * 7, 0),
 752     KCSubTypeElement('filebacked_pages', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 4 * 8, 0),
 753     KCSubTypeElement('compressions', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 4 * 9, 0),
 754     KCSubTypeElement('decompressions', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 4 * 10, 0),
 755     KCSubTypeElement('compressor_size', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 4 * 11, 0),
 756     KCSubTypeElement('busy_buffer_count', KCSUBTYPE_TYPE.KC_ST_INT32, 4, 4 * 12, 0),
 757     KCSubTypeElement('pages_wanted', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 4 * 13, 0),
 758     KCSubTypeElement('pages_reclaimed', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 4 * 14, 0),
 759     KCSubTypeElement('pages_wanted_reclaimed_valid', KCSUBTYPE_TYPE.KC_ST_UINT8, 1, 4 * 15, 0)
 760 ),
 761     'mem_and_io_snapshot'
 762 )
 763
 764
 765 KNOWN_TYPES_COLLECTION[0x905] = KCTypeDescription(0x905, (
 766     KCSubTypeElement.FromBasicCtype('ts_unique_pid', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),
 767     KCSubTypeElement.FromBasicCtype('ts_ss_flags', KCSUBTYPE_TYPE.KC_ST_UINT64, 8),
 768     KCSubTypeElement.FromBasicCtype('ts_user_time_in_terminated_thre', KCSUBTYPE_TYPE.KC_ST_UINT64, 16),
 769     KCSubTypeElement.FromBasicCtype('ts_system_time_in_terminated_th', KCSUBTYPE_TYPE.KC_ST_UINT64, 24),
 770     KCSubTypeElement.FromBasicCtype('ts_p_start_sec', KCSUBTYPE_TYPE.KC_ST_UINT64, 32),
 771     KCSubTypeElement.FromBasicCtype('ts_task_size', KCSUBTYPE_TYPE.KC_ST_UINT64, 40),
 772     KCSubTypeElement.FromBasicCtype('ts_max_resident_size', KCSUBTYPE_TYPE.KC_ST_UINT64, 48),
 773     KCSubTypeElement.FromBasicCtype('ts_suspend_count', KCSUBTYPE_TYPE.KC_ST_UINT32, 56),
 774     KCSubTypeElement.FromBasicCtype('ts_faults', KCSUBTYPE_TYPE.KC_ST_UINT32, 60),
 775     KCSubTypeElement.FromBasicCtype('ts_pageins', KCSUBTYPE_TYPE.KC_ST_UINT32, 64),
 776     KCSubTypeElement.FromBasicCtype('ts_cow_faults', KCSUBTYPE_TYPE.KC_ST_UINT32, 68),
 777     KCSubTypeElement.FromBasicCtype('ts_was_throttled', KCSUBTYPE_TYPE.KC_ST_UINT32, 72),
 778     KCSubTypeElement.FromBasicCtype('ts_did_throttle', KCSUBTYPE_TYPE.KC_ST_UINT32, 76),
 779     KCSubTypeElement.FromBasicCtype('ts_latency_qos', KCSUBTYPE_TYPE.KC_ST_UINT32, 80),
 780     KCSubTypeElement.FromBasicCtype('ts_pid', KCSUBTYPE_TYPE.KC_ST_INT32, 84),
 781     KCSubTypeElement('ts_p_comm', KCSUBTYPE_TYPE.KC_ST_CHAR, KCSubTypeElement.GetSizeForArray(32, 1), 88, 1)
 782 ),
 783     'task_snapshot'
 784 )
 785
 786 KNOWN_TYPES_COLLECTION[0x906] = KCTypeDescription(0x906, (
 787     KCSubTypeElement.FromBasicCtype('ths_thread_id', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),
 788     KCSubTypeElement.FromBasicCtype('ths_wait_event', KCSUBTYPE_TYPE.KC_ST_UINT64, 8),
 789     KCSubTypeElement.FromBasicCtype('ths_continuation', KCSUBTYPE_TYPE.KC_ST_UINT64, 16),
 790     KCSubTypeElement.FromBasicCtype('ths_total_syscalls', KCSUBTYPE_TYPE.KC_ST_UINT64, 24),
 791     KCSubTypeElement.FromBasicCtype('ths_voucher_identifier', KCSUBTYPE_TYPE.KC_ST_UINT64, 32),
 792     KCSubTypeElement.FromBasicCtype('ths_dqserialnum', KCSUBTYPE_TYPE.KC_ST_UINT64, 40),
 793     KCSubTypeElement.FromBasicCtype('ths_user_time', KCSUBTYPE_TYPE.KC_ST_UINT64, 48),
 794     KCSubTypeElement.FromBasicCtype('ths_sys_time', KCSUBTYPE_TYPE.KC_ST_UINT64, 56),
 795     KCSubTypeElement.FromBasicCtype('ths_ss_flags', KCSUBTYPE_TYPE.KC_ST_UINT64, 64),
 796     KCSubTypeElement.FromBasicCtype('ths_last_run_time', KCSUBTYPE_TYPE.KC_ST_UINT64, 72),
 797     KCSubTypeElement.FromBasicCtype('ths_last_made_runnable_time', KCSUBTYPE_TYPE.KC_ST_UINT64, 80),
 798     KCSubTypeElement.FromBasicCtype('ths_state', KCSUBTYPE_TYPE.KC_ST_UINT32, 88),
 799     KCSubTypeElement.FromBasicCtype('ths_sched_flags', KCSUBTYPE_TYPE.KC_ST_UINT32, 92),
 800     KCSubTypeElement.FromBasicCtype('ths_base_priority', KCSUBTYPE_TYPE.KC_ST_INT16, 96),
 801     KCSubTypeElement.FromBasicCtype('ths_sched_priority', KCSUBTYPE_TYPE.KC_ST_INT16, 98),
 802     KCSubTypeElement.FromBasicCtype('ths_eqos', KCSUBTYPE_TYPE.KC_ST_UINT8, 100),
 803     KCSubTypeElement.FromBasicCtype('ths_rqos', KCSUBTYPE_TYPE.KC_ST_UINT8, 101),
 804     KCSubTypeElement.FromBasicCtype('ths_rqos_override', KCSUBTYPE_TYPE.KC_ST_UINT8, 102),
 805     KCSubTypeElement.FromBasicCtype('ths_io_tier', KCSUBTYPE_TYPE.KC_ST_UINT8, 103),
 806     KCSubTypeElement.FromBasicCtype('ths_thread_t', KCSUBTYPE_TYPE.KC_ST_UINT64, 104),
 807     KCSubTypeElement.FromBasicCtype('ths_requested_policy', KCSUBTYPE_TYPE.KC_ST_UINT64, 112),
 808     KCSubTypeElement.FromBasicCtype('ths_effective_policy', KCSUBTYPE_TYPE.KC_ST_UINT64, 120),
 809 ),
 810     'thread_snapshot',
 811     legacy_size = 0x68
 812 )
 813
 814 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_THREAD_DELTA_SNAPSHOT')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_THREAD_DELTA_SNAPSHOT'), (
 815     KCSubTypeElement.FromBasicCtype('tds_thread_id', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),
 816     KCSubTypeElement.FromBasicCtype('tds_voucher_identifier', KCSUBTYPE_TYPE.KC_ST_UINT64, 8),
 817     KCSubTypeElement.FromBasicCtype('tds_ss_flags', KCSUBTYPE_TYPE.KC_ST_UINT64, 16),
 818     KCSubTypeElement.FromBasicCtype('tds_last_made_runnable_time', KCSUBTYPE_TYPE.KC_ST_UINT64, 24),
 819     KCSubTypeElement.FromBasicCtype('tds_state', KCSUBTYPE_TYPE.KC_ST_UINT32, 32),
 820     KCSubTypeElement.FromBasicCtype('tds_sched_flags', KCSUBTYPE_TYPE.KC_ST_UINT32, 36),
 821     KCSubTypeElement.FromBasicCtype('tds_base_priority', KCSUBTYPE_TYPE.KC_ST_INT16, 40),
 822     KCSubTypeElement.FromBasicCtype('tds_sched_priority', KCSUBTYPE_TYPE.KC_ST_INT16, 42),
 823     KCSubTypeElement.FromBasicCtype('tds_eqos', KCSUBTYPE_TYPE.KC_ST_UINT8, 44),
 824     KCSubTypeElement.FromBasicCtype('tds_rqos', KCSUBTYPE_TYPE.KC_ST_UINT8, 45),
 825     KCSubTypeElement.FromBasicCtype('tds_rqos_override', KCSUBTYPE_TYPE.KC_ST_UINT8, 46),
 826     KCSubTypeElement.FromBasicCtype('tds_io_tier', KCSUBTYPE_TYPE.KC_ST_UINT8, 47),
 827 ),
 828     'thread_delta_snapshot'
 829 )
 830
 831 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_TASK_DELTA_SNAPSHOT')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_TASK_DELTA_SNAPSHOT'), (
 832     KCSubTypeElement.FromBasicCtype('tds_unique_pid', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),
 833     KCSubTypeElement.FromBasicCtype('tds_ss_flags', KCSUBTYPE_TYPE.KC_ST_UINT64, 8),
 834     KCSubTypeElement.FromBasicCtype('tds_user_time_in_terminated_thr', KCSUBTYPE_TYPE.KC_ST_UINT64, 16),
 835     KCSubTypeElement.FromBasicCtype('tds_system_time_in_terminated_t', KCSUBTYPE_TYPE.KC_ST_UINT64, 24),
 836     KCSubTypeElement.FromBasicCtype('tds_task_size', KCSUBTYPE_TYPE.KC_ST_UINT64, 32),
 837     KCSubTypeElement.FromBasicCtype('tds_max_resident_size', KCSUBTYPE_TYPE.KC_ST_UINT64, 40),
 838     KCSubTypeElement.FromBasicCtype('tds_suspend_count', KCSUBTYPE_TYPE.KC_ST_UINT32, 48),
 839     KCSubTypeElement.FromBasicCtype('tds_faults', KCSUBTYPE_TYPE.KC_ST_UINT32, 52),
 840     KCSubTypeElement.FromBasicCtype('tds_pageins', KCSUBTYPE_TYPE.KC_ST_UINT32, 56),
 841     KCSubTypeElement.FromBasicCtype('tds_cow_faults', KCSUBTYPE_TYPE.KC_ST_UINT32, 60),
 842     KCSubTypeElement.FromBasicCtype('tds_was_throttled', KCSUBTYPE_TYPE.KC_ST_UINT32, 64),
 843     KCSubTypeElement.FromBasicCtype('tds_did_throttle', KCSUBTYPE_TYPE.KC_ST_UINT32, 68),
 844     KCSubTypeElement.FromBasicCtype('tds_latency_qos', KCSUBTYPE_TYPE.KC_ST_UINT32, 72),
 845 ),
 846     'task_delta_snapshot'
 847 )
 848
 849
 850 KNOWN_TYPES_COLLECTION[0x909] = KCSubTypeElement('pth_name', KCSUBTYPE_TYPE.KC_ST_CHAR, KCSubTypeElement.GetSizeForArray(64, 1), 0, 1)
 851
 852
 853
 854 KNOWN_TYPES_COLLECTION[GetTypeForName('KCDATA_TYPE_LIBRARY_LOADINFO64')] = KCTypeDescription(GetTypeForName('KCDATA_TYPE_LIBRARY_LOADINFO64'), (
 855     KCSubTypeElement('imageLoadAddress', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0),
 856     KCSubTypeElement('imageUUID', KCSUBTYPE_TYPE.KC_ST_UINT8, KCSubTypeElement.GetSizeForArray(16, 1), 8, 1)
 857 ),
 858     'dyld_load_info'
 859 )
 860
 861 KNOWN_TYPES_COLLECTION[GetTypeForName('KCDATA_TYPE_LIBRARY_LOADINFO')] = KCTypeDescription(GetTypeForName('KCDATA_TYPE_LIBRARY_LOADINFO'), (
 862     KCSubTypeElement('imageLoadAddress', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 0, 0),
 863     KCSubTypeElement('imageUUID', KCSUBTYPE_TYPE.KC_ST_UINT8, KCSubTypeElement.GetSizeForArray(16, 1), 4, 1)
 864 ),
 865     'dyld_load_info'
 866 )
 867
 868 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO'), (
 869     KCSubTypeElement('imageLoadAddress', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0),
 870     KCSubTypeElement('imageUUID', KCSUBTYPE_TYPE.KC_ST_UINT8, KCSubTypeElement.GetSizeForArray(16, 1), 8, 1),
 871     KCSubTypeElement('imageSlidBaseAddress', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 24, 0),
 872 ),
 873     'shared_cache_dyld_load_info',
 874     legacy_size = 0x18
 875 )
 876
 877 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_KERNELCACHE_LOADINFO')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_KERNELCACHE_LOADINFO'), (
 878     KCSubTypeElement('imageLoadAddress', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0),
 879     KCSubTypeElement('imageUUID', KCSUBTYPE_TYPE.KC_ST_UINT8, KCSubTypeElement.GetSizeForArray(16, 1), 8, 1),
 880 ),
 881     'kernelcache_load_info'
 882 )
 883
 884 KNOWN_TYPES_COLLECTION[0x33] = KCSubTypeElement('mach_absolute_time', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0, KCSubTypeElement._get_naked_element_value)
 885 KNOWN_TYPES_COLLECTION[0x907] = KCSubTypeElement.FromBasicCtype('donating_pids', KCSUBTYPE_TYPE.KC_ST_INT32)
 886
 887 KNOWN_TYPES_COLLECTION[GetTypeForName('KCDATA_TYPE_USECS_SINCE_EPOCH')] = KCSubTypeElement('usecs_since_epoch', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0, KCSubTypeElement._get_naked_element_value)
 888
 889 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_KERN_STACKFRAME')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_KERN_STACKFRAME'), (
 890     KCSubTypeElement.FromBasicCtype('lr', KCSUBTYPE_TYPE.KC_ST_UINT32),
 891     KCSubTypeElement.FromBasicCtype('sp', KCSUBTYPE_TYPE.KC_ST_UINT32, 4)
 892 ),
 893     'kernel_stack_frames'
 894 )
 895
 896 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_KERN_STACKLR')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_KERN_STACKLR'), (
 897     KCSubTypeElement.FromBasicCtype('lr', KCSUBTYPE_TYPE.KC_ST_UINT32),
 898 ),
 899     'kernel_stack_frames'
 900 )
 901
 902
 903 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_USER_STACKFRAME')] = KCTypeDescription.FromKCTypeDescription(
 904     KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_KERN_STACKFRAME')],
 905     GetTypeForName('STACKSHOT_KCTYPE_USER_STACKFRAME'),
 906     'user_stack_frames'
 907 )
 908
 909 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_USER_STACKLR')] = KCTypeDescription.FromKCTypeDescription(
 910     KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_KERN_STACKLR')],
 911     GetTypeForName('STACKSHOT_KCTYPE_USER_STACKLR'),
 912     'user_stack_frames'
 913 )
 914
 915 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_KERN_STACKFRAME64')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_KERN_STACKFRAME64'), (
 916     KCSubTypeElement.FromBasicCtype('lr', KCSUBTYPE_TYPE.KC_ST_UINT64),
 917     KCSubTypeElement.FromBasicCtype('sp', KCSUBTYPE_TYPE.KC_ST_UINT64, 8)
 918 ),
 919     'kernel_stack_frames'
 920 )
 921
 922 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_USER_STACKFRAME64')] = KCTypeDescription.FromKCTypeDescription(
 923     KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_KERN_STACKFRAME64')],
 924     GetTypeForName('STACKSHOT_KCTYPE_USER_STACKFRAME64'),
 925     'user_stack_frames'
 926 )
 927
 928
 929 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_KERN_STACKLR64')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_KERN_STACKLR64'), (
 930     KCSubTypeElement.FromBasicCtype('lr', KCSUBTYPE_TYPE.KC_ST_UINT64),
 931 ),
 932     'kernel_stack_frames'
 933 )
 934
 935 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_USER_STACKLR64')] = KCTypeDescription.FromKCTypeDescription(
 936     KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_KERN_STACKLR64')],
 937     GetTypeForName('STACKSHOT_KCTYPE_USER_STACKLR64'),
 938     'user_stack_frames'
 939 )
 940
 941
 942 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_NONRUNNABLE_TIDS')] = KCSubTypeElement.FromBasicCtype('nonrunnable_threads', KCSUBTYPE_TYPE.KC_ST_INT64)
 943
 944 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_NONRUNNABLE_TASKS')] = KCSubTypeElement.FromBasicCtype('nonrunnable_tasks', KCSUBTYPE_TYPE.KC_ST_INT64)
 945
 946 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_OSVERSION')] = KCSubTypeElement('osversion', KCSUBTYPE_TYPE.KC_ST_CHAR,
 947                           KCSubTypeElement.GetSizeForArray(256, 1), 0, 1)
 948
 949 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_BOOTARGS')] = KCSubTypeElement('boot_args', KCSUBTYPE_TYPE.KC_ST_CHAR,
 950                            KCSubTypeElement.GetSizeForArray(256, 1), 0, 1)
 951
 952 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_KERN_PAGE_SIZE')] = KCSubTypeElement('kernel_page_size', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 0, 0, KCSubTypeElement._get_naked_element_value)
 953
 954 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_THREAD_POLICY_VERSION')] = KCSubTypeElement('thread_policy_version', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 0, 0, KCSubTypeElement._get_naked_element_value)
 955
 956 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_JETSAM_LEVEL')] = KCSubTypeElement('jetsam_level', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 0, 0, KCSubTypeElement._get_naked_element_value)
 957
 958 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_DELTA_SINCE_TIMESTAMP')] = KCSubTypeElement("stackshot_delta_since_timestamp", KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0, KCSubTypeElement._get_naked_element_value)
 959
 960 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS'),
 961             (
 962                         KCSubTypeElement.FromBasicCtype('sfs_pages_faulted_in', KCSUBTYPE_TYPE.KC_ST_UINT32, 0),
 963                         KCSubTypeElement.FromBasicCtype('sfs_time_spent_faulting', KCSUBTYPE_TYPE.KC_ST_UINT64, 4),
 964                         KCSubTypeElement.FromBasicCtype('sfs_system_max_fault_time', KCSUBTYPE_TYPE.KC_ST_UINT64, 12),
 965                         KCSubTypeElement.FromBasicCtype('sfs_stopped_faulting', KCSUBTYPE_TYPE.KC_ST_UINT8, 20)
 966             ),
 967             'stackshot_fault_stats')
 968
 969 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_THREAD_WAITINFO')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_THREAD_WAITINFO'),
 970             (
 971                         KCSubTypeElement.FromBasicCtype('owner', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),
 972                         KCSubTypeElement.FromBasicCtype('waiter', KCSUBTYPE_TYPE.KC_ST_UINT64, 8),
 973                         KCSubTypeElement.FromBasicCtype('context', KCSUBTYPE_TYPE.KC_ST_UINT64, 16),
 974                         KCSubTypeElement.FromBasicCtype('wait_type', KCSUBTYPE_TYPE.KC_ST_UINT8, 24)
 975             ),
 976             'thread_waitinfo')
 977
 978 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_THREAD_GROUP'),
 979             (
 980                         KCSubTypeElement.FromBasicCtype('tgs_id', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),
 981                         KCSubTypeElement('tgs_name', KCSUBTYPE_TYPE.KC_ST_CHAR, KCSubTypeElement.GetSizeForArray(16, 1),
 982                             8, 1),
 983                         KCSubTypeElement.FromBasicCtype('tgs_flags', KCSUBTYPE_TYPE.KC_ST_UINT64, 24),
 984             ),
 985             'thread_group_snapshot')
 986
 987
 988 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_THREAD_GROUP')] = KCSubTypeElement('thread_group', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0, KCSubTypeElement._get_naked_element_value)
 989
 990 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT'),
 991             (
 992                         KCSubTypeElement.FromBasicCtype('jcs_id', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),
 993                         KCSubTypeElement.FromBasicCtype('jcs_flags', KCSUBTYPE_TYPE.KC_ST_UINT64, 8),
 994                         KCSubTypeElement.FromBasicCtype('jcs_thread_group', KCSUBTYPE_TYPE.KC_ST_UINT64, 16),
 995                         KCSubTypeElement.FromBasicCtype('jcs_leader_task_uniqueid', KCSUBTYPE_TYPE.KC_ST_UINT64, 24)
 996             ),
 997             'jetsam_coalition_snapshot')
 998
 999 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_JETSAM_COALITION')] = KCSubTypeElement('jetsam_coalition', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0, KCSubTypeElement._get_naked_element_value)
1000
1001 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_INSTRS_CYCLES')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_INSTRS_CYCLES'),
1002             (
1003                         KCSubTypeElement.FromBasicCtype('ics_instructions', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),
1004                         KCSubTypeElement.FromBasicCtype('ics_cycles', KCSUBTYPE_TYPE.KC_ST_UINT64, 8)
1005             ),
1006             'instrs_cycles_snapshot')
1007
1008 #KNOWN_TYPES_COLLECTION[0x907] = KCSubTypeElement('donating_pids', KCSUBTYPE_TYPE.KC_ST_UINT32, 4, 0, 0, KCSubTypeElement._get_naked_element_value)
1009 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_PID')] = KCSubTypeElement('pid', KCSUBTYPE_TYPE.KC_ST_INT32, 4, 0, 0)
1010 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_PPID')] = KCSubTypeElement('ppid', KCSUBTYPE_TYPE.KC_ST_INT32, 4, 0, 0)
1011 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_PROC_NAME')] = KCSubTypeElement('p_comm', KCSUBTYPE_TYPE.KC_ST_CHAR,
1012                            KCSubTypeElement.GetSizeForArray(32, 1), 0, 1)
1013 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_USERSTACK')] = KCSubTypeElement('userstack_ptr', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0)
1014 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_ARGSLEN')] = KCSubTypeElement('p_argslen', KCSUBTYPE_TYPE.KC_ST_INT32, 4, 0, 0)
1015
1016 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_PROC_PATH')] = KCSubTypeElement('p_path', KCSUBTYPE_TYPE.KC_ST_CHAR,
1017                            KCSubTypeElement.GetSizeForArray(1024, 1), 0, 1)
1018 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_PROC_CSFLAGS')] = KCSubTypeElement('p_csflags', KCSUBTYPE_TYPE.KC_ST_INT32, 4, 0, 0)
1019 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_UID')] = KCSubTypeElement('uid', KCSUBTYPE_TYPE.KC_ST_INT32, 4, 0, 0)
1020 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_GID')] = KCSubTypeElement('gid', KCSUBTYPE_TYPE.KC_ST_INT32, 4, 0, 0)
1021 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_PROC_ARGC')] = KCSubTypeElement('argc', KCSUBTYPE_TYPE.KC_ST_INT32, 4, 0, 0)
1022 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_PROC_FLAGS')] = KCSubTypeElement('p_flags', KCSUBTYPE_TYPE.KC_ST_INT32, 4, 0, 0)
1023 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_CPUTYPE')] = KCSubTypeElement('cputype', KCSUBTYPE_TYPE.KC_ST_INT32, 4, 0, 0)
1024 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_RESPONSIBLE_PID')] = KCSubTypeElement('responsible_pid', KCSUBTYPE_TYPE.KC_ST_INT32, 4, 0, 0)
1025 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_DIRTY_FLAGS')] = KCSubTypeElement('dirty_flags', KCSUBTYPE_TYPE.KC_ST_INT32, 4, 0, 0)
1026 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_CRASHED_THREADID')] = KCSubTypeElement('crashed_threadid', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0)
1027 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_COALITION_ID')] = KCSubTypeElement('coalition_id', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0)
1028
1029 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_PROC_STATUS')] = KCSubTypeElement('p_status', KCSUBTYPE_TYPE.KC_ST_UINT8, 1, 0, 0)
1030
1031 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_BSDINFOWITHUNIQID')] = KCTypeDescription(GetTypeForName('TASK_CRASHINFO_BSDINFOWITHUNIQID'),
1032     (   KCSubTypeElement('p_uuid', KCSUBTYPE_TYPE.KC_ST_UINT8, KCSubTypeElement.GetSizeForArray(16, 1), 0, 1),
1033         KCSubTypeElement.FromBasicCtype('p_uniqueid', KCSUBTYPE_TYPE.KC_ST_UINT64, 16),
1034         KCSubTypeElement.FromBasicCtype('p_puniqueid', KCSUBTYPE_TYPE.KC_ST_UINT64, 24)
1035     ),
1036     'proc_uniqidentifierinfo')
1037
1038 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_EXCEPTION_CODES')] = (
1039     KCTypeDescription(GetTypeForName('TASK_CRASHINFO_EXCEPTION_CODES'),
1040                       (KCSubTypeElement.FromBasicCtype('code_0', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),
1041                        KCSubTypeElement.FromBasicCtype('code_1', KCSUBTYPE_TYPE.KC_ST_UINT64, 8)),
1042                       'mach_exception_data_t'))
1043
1044
1045 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_PROC_STARTTIME')] = (
1046     KCTypeDescription(GetTypeForName('TASK_CRASHINFO_PROC_STARTTIME'),
1047                       (KCSubTypeElement.FromBasicCtype('tv_sec', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),
1048                        KCSubTypeElement.FromBasicCtype('tv_usec', KCSUBTYPE_TYPE.KC_ST_UINT64, 8)),
1049                       'proc_starttime'))
1050
1051
1052 KNOWN_TYPES_COLLECTION[GetTypeForName('TASK_CRASHINFO_RUSAGE_INFO')] = KCTypeDescription(GetTypeForName('TASK_CRASHINFO_RUSAGE_INFO'),
1053     (
1054         KCSubTypeElement('ri_uuid', KCSUBTYPE_TYPE.KC_ST_UINT8, KCSubTypeElement.GetSizeForArray(16, 1), 0, 1),
1055             KCSubTypeElement.FromBasicCtype('ri_user_time', KCSUBTYPE_TYPE.KC_ST_UINT64, 16),
1056             KCSubTypeElement.FromBasicCtype('ri_system_time', KCSUBTYPE_TYPE.KC_ST_UINT64, 24),
1057             KCSubTypeElement.FromBasicCtype('ri_pkg_idle_wkups', KCSUBTYPE_TYPE.KC_ST_UINT64, 32),
1058             KCSubTypeElement.FromBasicCtype('ri_interrupt_wkups', KCSUBTYPE_TYPE.KC_ST_UINT64, 40),
1059             KCSubTypeElement.FromBasicCtype('ri_pageins', KCSUBTYPE_TYPE.KC_ST_UINT64, 48),
1060             KCSubTypeElement.FromBasicCtype('ri_wired_size', KCSUBTYPE_TYPE.KC_ST_UINT64, 56),
1061             KCSubTypeElement.FromBasicCtype('ri_resident_size', KCSUBTYPE_TYPE.KC_ST_UINT64, 64),
1062             KCSubTypeElement.FromBasicCtype('ri_phys_footprint', KCSUBTYPE_TYPE.KC_ST_UINT64, 72),
1063             KCSubTypeElement.FromBasicCtype('ri_proc_start_abstime', KCSUBTYPE_TYPE.KC_ST_UINT64, 80),
1064             KCSubTypeElement.FromBasicCtype('ri_proc_exit_abstime', KCSUBTYPE_TYPE.KC_ST_UINT64, 88),
1065             KCSubTypeElement.FromBasicCtype('ri_child_user_time', KCSUBTYPE_TYPE.KC_ST_UINT64, 96),
1066             KCSubTypeElement.FromBasicCtype('ri_child_system_time', KCSUBTYPE_TYPE.KC_ST_UINT64, 104),
1067             KCSubTypeElement.FromBasicCtype('ri_child_pkg_idle_wkups', KCSUBTYPE_TYPE.KC_ST_UINT64, 112),
1068             KCSubTypeElement.FromBasicCtype('ri_child_interrupt_wkups', KCSUBTYPE_TYPE.KC_ST_UINT64, 120),
1069             KCSubTypeElement.FromBasicCtype('ri_child_pageins', KCSUBTYPE_TYPE.KC_ST_UINT64, 128),
1070             KCSubTypeElement.FromBasicCtype('ri_child_elapsed_abstime', KCSUBTYPE_TYPE.KC_ST_UINT64, 136),
1071             KCSubTypeElement.FromBasicCtype('ri_diskio_bytesread', KCSUBTYPE_TYPE.KC_ST_UINT64, 144),
1072             KCSubTypeElement.FromBasicCtype('ri_diskio_byteswritten', KCSUBTYPE_TYPE.KC_ST_UINT64, 152),
1073             KCSubTypeElement.FromBasicCtype('ri_cpu_time_qos_default', KCSUBTYPE_TYPE.KC_ST_UINT64, 160),
1074             KCSubTypeElement.FromBasicCtype('ri_cpu_time_qos_maintenance', KCSUBTYPE_TYPE.KC_ST_UINT64, 168),
1075             KCSubTypeElement.FromBasicCtype('ri_cpu_time_qos_background', KCSUBTYPE_TYPE.KC_ST_UINT64, 176),
1076             KCSubTypeElement.FromBasicCtype('ri_cpu_time_qos_utility', KCSUBTYPE_TYPE.KC_ST_UINT64, 184),
1077             KCSubTypeElement.FromBasicCtype('ri_cpu_time_qos_legacy', KCSUBTYPE_TYPE.KC_ST_UINT64, 192),
1078             KCSubTypeElement.FromBasicCtype('ri_cpu_time_qos_user_initiated', KCSUBTYPE_TYPE.KC_ST_UINT64, 200),
1079             KCSubTypeElement.FromBasicCtype('ri_cpu_time_qos_user_interactiv', KCSUBTYPE_TYPE.KC_ST_UINT64, 208),
1080             KCSubTypeElement.FromBasicCtype('ri_billed_system_time', KCSUBTYPE_TYPE.KC_ST_UINT64, 216),
1081             KCSubTypeElement.FromBasicCtype('ri_serviced_system_time', KCSUBTYPE_TYPE.KC_ST_UINT64, 224)
1082     ),
1083     'rusage_info')
1084
1085 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_CPU_TIMES')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_CPU_TIMES'),
1086     (
1087         KCSubTypeElement.FromBasicCtype('user_usec', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),
1088         KCSubTypeElement.FromBasicCtype('system_usec', KCSUBTYPE_TYPE.KC_ST_UINT64, 8),
1089     ), 'cpu_times')
1090
1091 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_STACKSHOT_DURATION')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_STACKSHOT_DURATION'),
1092     (
1093         KCSubTypeElement.FromBasicCtype('stackshot_duration', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),
1094         KCSubTypeElement.FromBasicCtype('stackshot_duration_outer', KCSUBTYPE_TYPE.KC_ST_UINT64, 8),
1095     ), 'stackshot_duration', merge=True
1096 )
1097
1098 KNOWN_TYPES_COLLECTION[GetTypeForName('KCDATA_TYPE_PROCNAME')] = (
1099     KCSubTypeElement("proc_name", KCSUBTYPE_TYPE.KC_ST_CHAR, KCSubTypeElement.GetSizeForArray(-1, 1), 0, 1))
1100
1101 KNOWN_TYPES_COLLECTION[GetTypeForName('KCDATA_TYPE_PID')] = (
1102     KCSubTypeElement('pid', KCSUBTYPE_TYPE.KC_ST_INT32, 4, 0, 0))
1103
1104 KNOWN_TYPES_COLLECTION[GetTypeForName('EXIT_REASON_SNAPSHOT')] = KCTypeDescription(GetTypeForName('EXIT_REASON_SNAPSHOT'),
1105     (
1106         KCSubTypeElement.FromBasicCtype('ers_namespace', KCSUBTYPE_TYPE.KC_ST_UINT32, 0),
1107         KCSubTypeElement.FromBasicCtype('ers_code', KCSUBTYPE_TYPE.KC_ST_UINT64, 4),
1108         KCSubTypeElement.FromBasicCtype('ers_flags', KCSUBTYPE_TYPE.KC_ST_UINT64, 12),
1109     ), 'exit_reason_basic_info')
1110
1111 KNOWN_TYPES_COLLECTION[GetTypeForName('EXIT_REASON_USER_DESC')] = (
1112     KCSubTypeElement("exit_reason_user_description", KCSUBTYPE_TYPE.KC_ST_CHAR, KCSubTypeElement.GetSizeForArray(-1, 1), 0, 1))
1113
1114 KNOWN_TYPES_COLLECTION[GetTypeForName('EXIT_REASON_USER_PAYLOAD')] = KCSubTypeElement('exit_reason_user_payload',
1115         KCSUBTYPE_TYPE.KC_ST_UINT8, KCSubTypeElement.GetSizeForArray(-1, 1), 0, 1)
1116
1117 KNOWN_TYPES_COLLECTION[GetTypeForName('EXIT_REASON_CODESIGNING_INFO')] = KCTypeDescription(GetTypeForName('EXIT_REASON_CODESIGNING_INFO'),
1118     (
1119         KCSubTypeElement.FromBasicCtype('ceri_virt_addr', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),
1120         KCSubTypeElement.FromBasicCtype('ceri_file_offset', KCSUBTYPE_TYPE.KC_ST_UINT64, 8),
1121         KCSubTypeElement("ceri_pathname", KCSUBTYPE_TYPE.KC_ST_CHAR, KCSubTypeElement.GetSizeForArray(1024, 1), 16, 1),
1122         KCSubTypeElement("ceri_filename", KCSUBTYPE_TYPE.KC_ST_CHAR, KCSubTypeElement.GetSizeForArray(1024, 1), 1040, 1),
1123         KCSubTypeElement.FromBasicCtype('ceri_codesig_modtime_secs', KCSUBTYPE_TYPE.KC_ST_UINT64, 2064),
1124         KCSubTypeElement.FromBasicCtype('ceri_codesig_modtime_nsecs', KCSUBTYPE_TYPE.KC_ST_UINT64, 2072),
1125         KCSubTypeElement.FromBasicCtype('ceri_page_modtime_secs', KCSUBTYPE_TYPE.KC_ST_UINT64, 2080),
1126         KCSubTypeElement.FromBasicCtype('ceri_page_modtime_nsecs', KCSUBTYPE_TYPE.KC_ST_UINT64, 2088),
1127         KCSubTypeElement.FromBasicCtype('ceri_path_truncated', KCSUBTYPE_TYPE.KC_ST_UINT8, 2096),
1128         KCSubTypeElement.FromBasicCtype('ceri_object_codesigned', KCSUBTYPE_TYPE.KC_ST_UINT8, 2097),
1129         KCSubTypeElement.FromBasicCtype('ceri_page_codesig_validated', KCSUBTYPE_TYPE.KC_ST_UINT8, 2098),
1130         KCSubTypeElement.FromBasicCtype('ceri_page_codesig_tainted', KCSUBTYPE_TYPE.KC_ST_UINT8, 2099),
1131         KCSubTypeElement.FromBasicCtype('ceri_page_codesig_nx', KCSUBTYPE_TYPE.KC_ST_UINT8, 2100),
1132         KCSubTypeElement.FromBasicCtype('ceri_page_wpmapped', KCSUBTYPE_TYPE.KC_ST_UINT8, 2101),
1133         KCSubTypeElement.FromBasicCtype('ceri_page_slid', KCSUBTYPE_TYPE.KC_ST_UINT8, 2102),
1134         KCSubTypeElement.FromBasicCtype('ceri_page_dirty', KCSUBTYPE_TYPE.KC_ST_UINT8, 2103),
1135         KCSubTypeElement.FromBasicCtype('ceri_page_shadow_depth', KCSUBTYPE_TYPE.KC_ST_UINT32, 2104),
1136     ), 'exit_reason_codesigning_info')
1137
1138 KNOWN_TYPES_COLLECTION[GetTypeForName('EXIT_REASON_WORKLOOP_ID')] = (
1139         KCSubTypeElement('exit_reason_workloop_id', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0, KCSubTypeElement._get_naked_element_value))
1140
1141 KNOWN_TYPES_COLLECTION[GetTypeForName('EXIT_REASON_DISPATCH_QUEUE_NO')] = (
1142         KCSubTypeElement('exit_reason_dispatch_queue_no', KCSUBTYPE_TYPE.KC_ST_UINT64, 8, 0, 0, KCSubTypeElement._get_naked_element_value))
1143
1144 def GetSecondsFromMATime(mat, tb):
1145     return (float(mat) * tb['numer']) / tb['denom']
1146
1147 def FindLibraryForAddress(liblist, address):
1148     current_lib = None
1149     for l in liblist:
1150         if address >= l[1]:
1151             current_lib = l
1152     return current_lib
1153
1154 def FindIndexOfLibInCatalog(catalog, lib):
1155     index = None
1156     i = 0
1157     for l in catalog:
1158         if l[0] == lib[0] and l[1] == lib[1]:
1159             index = i
1160             break
1161         i += 1
1162
1163     if index is None:
1164         catalog.append(lib)
1165         index = len(catalog) - 1
1166
1167     return index
1168
1169 def GetOffsetOfAddressForLib(lib, address):
1170     return (address - lib[1])
1171
1172 def GetSymbolInfoForFrame(catalog, liblist, address):
1173     lib = FindLibraryForAddress(liblist, address)
1174     if not lib:
1175         lib = ["00000000000000000000000000000000",0,"A"]
1176     offset = GetOffsetOfAddressForLib(lib, address)
1177     index = FindIndexOfLibInCatalog(catalog, lib)
1178     return [index, offset]
1179
1180 def GetStateDescription(s):
1181     retval = []
1182     TH_WAIT = 0x01
1183     TH_SUSP = 0x02
1184     TH_RUN = 0x04
1185     TH_UNINT = 0x08
1186     TH_TERMINATE = 0x10
1187     TH_TERMINATE2 = 0x20
1188     TH_IDLE = 0x80
1189     if (s & TH_WAIT):
1190         retval.append("TH_WAIT")
1191     if (s & TH_SUSP):
1192         retval.append("TH_SUSP")
1193     if (s & TH_RUN):
1194         retval.append("TH_RUN")
1195     if (s & TH_UNINT):
1196         retval.append("TH_UNINT")
1197     if (s & TH_TERMINATE):
1198         retval.append("TH_TERMINATE")
1199     if (s & TH_TERMINATE2):
1200         retval.append("TH_TERMINATE2")
1201     if (s & TH_IDLE):
1202         retval.append("TH_IDLE")
1203     return retval
1204
1205
1206 def format_uuid(elementValues):
1207     return ''.join("%02x" % i for i in elementValues)
1208
1209 kThreadWaitNone                 = 0x00
1210 kThreadWaitKernelMutex          = 0x01
1211 kThreadWaitPortReceive          = 0x02
1212 kThreadWaitPortSetReceive       = 0x03
1213 kThreadWaitPortSend             = 0x04
1214 kThreadWaitPortSendInTransit    = 0x05
1215 kThreadWaitSemaphore            = 0x06
1216 kThreadWaitKernelRWLockRead     = 0x07
1217 kThreadWaitKernelRWLockWrite    = 0x08
1218 kThreadWaitKernelRWLockUpgrade  = 0x09
1219 kThreadWaitUserLock             = 0x0a
1220 kThreadWaitPThreadMutex         = 0x0b
1221 kThreadWaitPThreadRWLockRead    = 0x0c
1222 kThreadWaitPThreadRWLockWrite   = 0x0d
1223 kThreadWaitPThreadCondVar       = 0x0e
1224 kThreadWaitParkedWorkQueue      = 0x0f
1225 kThreadWaitWorkloopSyncWait     = 0x10
1226
1227
1228 UINT64_MAX = 0xffffffffffffffff
1229 STACKSHOT_WAITOWNER_KERNEL      = (UINT64_MAX - 1)
1230 STACKSHOT_WAITOWNER_PORT_LOCKED = (UINT64_MAX - 2)
1231 STACKSHOT_WAITOWNER_PSET_LOCKED = (UINT64_MAX - 3)
1232 STACKSHOT_WAITOWNER_INTRANSIT   = (UINT64_MAX - 4)
1233 STACKSHOT_WAITOWNER_MTXSPIN     = (UINT64_MAX - 5)
1234 STACKSHOT_WAITOWNER_THREQUESTED = (UINT64_MAX - 6)
1235 STACKSHOT_WAITOWNER_SUSPENDED   = (UINT64_MAX - 7)
1236
1237 def formatWaitInfo(info):
1238     s = 'thread %d: ' % info['waiter'];
1239     type = info['wait_type']
1240     context = info['context']
1241     owner = info['owner']
1242     if type == kThreadWaitKernelMutex:
1243         s += 'kernel mutex %x' % context
1244         if owner == STACKSHOT_WAITOWNER_MTXSPIN:
1245             s += " in spin mode"
1246         elif owner:
1247             s += " owned by thread %u" % owner
1248         else:
1249             s += "with unknown owner"
1250     elif type == kThreadWaitPortReceive:
1251         s += "mach_msg receive on "
1252         if owner == STACKSHOT_WAITOWNER_PORT_LOCKED:
1253             s += "locked port %x" % context
1254         elif owner == STACKSHOT_WAITOWNER_INTRANSIT:
1255             s += "intransit port %x" % context
1256         elif owner:
1257             s += "port %x name %x" % (context, owner)
1258         else:
1259             s += "port %x" % context
1260     elif type == kThreadWaitPortSetReceive:
1261         if owner == STACKSHOT_WAITOWNER_PSET_LOCKED:
1262             s += "mach_msg receive on locked port set %x" % context
1263         else:
1264             s += "mach_msg receive on port set %x" % context
1265     elif type == kThreadWaitPortSend:
1266         s += "mach_msg send on "
1267         if owner == STACKSHOT_WAITOWNER_PORT_LOCKED:
1268             s += "locked port %x" % context
1269         elif owner == STACKSHOT_WAITOWNER_INTRANSIT:
1270             s += "intransit port %x" % context
1271         elif owner == STACKSHOT_WAITOWNER_KERNEL:
1272             s += "port %x owned by kernel" % context
1273         elif owner:
1274             s += "port %x owned by pid %d" % (context, owner)
1275         else:
1276             s += "port %x with unknown owner" % context
1277     elif type == kThreadWaitPortSendInTransit:
1278         s += "mach_msg send on port %x in transit to " % context
1279         if owner:
1280             s += "port %x" % owner
1281         else:
1282             s += "unknown port"
1283     elif type == kThreadWaitSemaphore:
1284         s += "semaphore port %x" % context
1285         if owner:
1286             s += "owned by pid %d" % owner
1287         else:
1288             s += "with unknown owner"
1289     elif type == kThreadWaitKernelRWLockRead:
1290         s += "krwlock %x for reading" % context
1291     elif type == kThreadWaitKernelRWLockWrite:
1292         s += "krwlock %x for writing" % context
1293     elif type == kThreadWaitKernelRWLockUpgrade:
1294         s += "krwlock %x for upgrading" % context
1295     elif type == kThreadWaitUserLock:
1296         if owner:
1297             s += "unfair lock %x owned by pid %d" % (context, owner)
1298         else:
1299             s += "spin lock %x" % context
1300     elif type == kThreadWaitPThreadMutex:
1301         s += "pthread mutex %x" % context
1302         if owner:
1303             s += " owned by pid %d" % owner
1304         else:
1305             s += " with unknown owner"
1306     elif type == kThreadWaitPThreadRWLockRead:
1307         s += "pthread rwlock %x for reading" % context
1308     elif type == kThreadWaitPThreadRWLockWrite:
1309         s += "pthread rwlock %x for writing" % context
1310     elif type == kThreadWaitPThreadCondVar:
1311         s += "pthread condvar %x" % context
1312     elif type == kThreadWaitWorkloopSyncWait:
1313         s += "workloop sync wait"
1314         if owner == STACKSHOT_WAITOWNER_SUSPENDED:
1315             s += ", suspended"
1316         elif owner == STACKSHOT_WAITOWNER_THREQUESTED:
1317             s += ", thread requested"
1318         elif owner != 0:
1319             s += ", owned by thread %u" % owner
1320         else:
1321             s += ", unknown owner"
1322         s += ", workloop id %x" % context
1323     else:
1324         s += "unknown type %d (owner %d, context %x)" % (type, owner, context)
1325
1326     return s
1327
1328
1329 def SaveStackshotReport(j, outfile_name, dsc_uuid, dsc_libs_arr, incomplete):
1330     import time
1331     from operator import itemgetter, attrgetter
1332     ss = j.get('kcdata_stackshot')
1333     if not ss:
1334         print "No KCDATA_BUFFER_BEGIN_STACKSHOT object found. Skipping writing report."
1335         return
1336
1337     timestamp = ss.get('usecs_since_epoch', int(time.time()))
1338     try:
1339         timestamp = time.strftime("%Y-%m-%d %H:%M:%S %z",time.gmtime(timestamp))
1340     except ValueError, e:
1341         print "couldn't convert timestamp:", str(e)
1342         timestamp = None
1343
1344     os_version = ss.get('osversion', 'Unknown')
1345     timebase = ss.get('mach_timebase_info', {"denom": 1, "numer": 1})
1346     if not dsc_uuid and 'imageSlidBaseAddress' not in ss.get('shared_cache_dyld_load_info'):
1347         print "Stackshot format does not include slid shared cache base address and no UUID provided. Skipping writing report."
1348         return
1349
1350     # If a shared cache UUID is provided, treat the slide as the base address
1351     # for compatibility with existing tools that operate based on this logic
1352     if dsc_uuid:
1353         shared_cache_base_addr = ss.get('shared_cache_dyld_load_info')['imageLoadAddress']
1354     elif 'imageSlidBaseAddress' in ss.get('shared_cache_dyld_load_info'):
1355         shared_cache_base_addr = ss.get('shared_cache_dyld_load_info')['imageSlidBaseAddress']
1356     else:
1357         print "No shared cache UUID provided and data doesn't include imageSlidBaseAddress. Skipping writing report."
1358         return
1359
1360     dsc_common = [format_uuid(ss.get('shared_cache_dyld_load_info')['imageUUID']),
1361                   shared_cache_base_addr,
1362                   "S"
1363                  ]
1364
1365     dsc_libs = []
1366     print "Shared cache UUID found from the binary data is <%s> " % str(dsc_common[0])
1367     if dsc_common[0].replace('-', '').lower() == dsc_uuid:
1368         print "SUCCESS: Found Matching dyld shared cache uuid. Loading library load addresses from layout provided."
1369         _load_addr = dsc_common[1]
1370         #print _load_addr
1371         #print dsc_libs_arr
1372         for i in dsc_libs_arr:
1373             _uuid = i[2].lower().replace('-','').strip()
1374             _addr = int(i[0], 16) + _load_addr
1375             dsc_libs.append([_uuid, _addr, "P"])
1376             #print "adding ", [_uuid, _addr, "C"]
1377     elif dsc_uuid:
1378         print "Provided shared cache UUID does not match. Skipping writing report."
1379         return
1380
1381     AllImageCatalog = []
1382     obj = {}
1383     obj["kernel"] = os_version
1384     if timestamp is not None:
1385         obj["date"] = timestamp
1386     obj["reason"] = "kernel panic stackshot"
1387     obj["incident"] = "ABCDEFGH-1234-56IJ-789K-0LMNOPQRSTUV"
1388     obj["crashReporterKey"] = "12ab34cd45aabbccdd6712ab34cd45aabbccdd67"
1389     obj["bootArgs"] = ss.get('boot_args','')
1390     obj["frontmostPids"] = [0]
1391     obj["exception"] = "0xDEADF157"
1392     obj["processByPid"] = {}
1393
1394     if incomplete:
1395         obj["reason"] = "!!!INCOMPLETE!!! kernel panic stackshot"
1396         obj["notes"] = "This stackshot report generated from incomplete data!   Some information is missing! "
1397
1398     processByPid = obj["processByPid"]
1399     ssplist = ss.get('task_snapshots', {})
1400     kern_load_info = []
1401     if "0" in ssplist:
1402         kc_uuid = ssplist["0"].get('kernelcache_load_info', None)
1403         if kc_uuid:
1404             kernelcache_uuid = [format_uuid(kc_uuid['imageUUID']), kc_uuid['imageLoadAddress'], "U" ]
1405             kern_load_info.append(kernelcache_uuid)
1406
1407         kl_infos = ssplist["0"].get("dyld_load_info", [])
1408         for dlinfo in kl_infos:
1409             kern_load_info.append([format_uuid(dlinfo['imageUUID']), dlinfo['imageLoadAddress'], "K"])
1410     for pid,piddata in ssplist.iteritems():
1411         processByPid[str(pid)] = {}
1412         tsnap = processByPid[str(pid)]
1413         pr_lib_dsc = dsc_common
1414         if 'shared_cache_dyld_load_info' in tsnap:
1415             if 'imageSlidBaseAddress' in tsnap.get('shared_cache_dyld_load_info'):
1416                 shared_cache_base_addr = tsnap.get('shared_cache_dyld_load_info')['imageSlidBaseAddress']
1417             else:
1418                 print "Specific task shared cache format does not include slid shared cache base address. Skipping writing report."
1419                 return
1420
1421             pr_lib_dsc = [format_uuid(tsnap.get('shared_cache_dyld_load_info')['imageUUID']),
1422                           tsnap.get('shared_cache_dyld_load_info')['imageSlidBaseAddress'],
1423                           "S"
1424                          ]
1425
1426         pr_libs = []
1427         if len(dsc_libs) == 0:
1428             pr_libs.append(pr_lib_dsc)
1429         _lib_type = "P"
1430         if int(pid) == 0:
1431             _lib_type = "K"
1432             pr_libs = []
1433         else:
1434             for dlinfo in piddata.get('dyld_load_info',[]):
1435                 pr_libs.append([format_uuid(dlinfo['imageUUID']), dlinfo['imageLoadAddress'], _lib_type])
1436
1437         pr_libs.extend(kern_load_info)
1438         pr_libs.extend(dsc_libs)
1439
1440         pr_libs.sort(key=itemgetter(1))
1441
1442         if 'task_snapshot' not in piddata:
1443             continue
1444         tasksnap = piddata['task_snapshot']
1445         tsnap["pid"] = tasksnap["ts_pid"]
1446         tsnap["residentMemoryBytes"] = tasksnap["ts_task_size"]
1447         tsnap["timesDidThrottle"] = tasksnap["ts_did_throttle"]
1448         tsnap["systemTimeTask"] = GetSecondsFromMATime(tasksnap["ts_system_time_in_terminated_th"], timebase)
1449         tsnap["pageIns"] = tasksnap["ts_pageins"]
1450         tsnap["pageFaults"] = tasksnap["ts_faults"]
1451         tsnap["userTimeTask"] = GetSecondsFromMATime(tasksnap[  "ts_user_time_in_terminated_thre"], timebase)
1452         tsnap["procname"] = tasksnap["ts_p_comm"]
1453         tsnap["copyOnWriteFaults"] = tasksnap["ts_cow_faults"]
1454         tsnap["timesThrottled"] = tasksnap["ts_was_throttled"]
1455         tsnap["threadById"] = {}
1456         threadByID = tsnap["threadById"]
1457         thlist = piddata.get('thread_snapshots', {})
1458         for tid,thdata in thlist.iteritems():
1459             threadByID[str(tid)] = {}
1460             thsnap = threadByID[str(tid)]
1461             if "thread_snapshot" not in thdata:
1462                 print "Found broken thread state for thread ID: %s." % tid
1463                 break
1464             threadsnap = thdata["thread_snapshot"]
1465             thsnap["userTime"] = GetSecondsFromMATime(threadsnap["ths_user_time"], timebase)
1466             thsnap["id"] = threadsnap["ths_thread_id"]
1467             thsnap["basePriority"] = threadsnap["ths_base_priority"]
1468             thsnap["systemTime"] = threadsnap["ths_sys_time"]
1469             thsnap["schedPriority"] = threadsnap["ths_sched_priority"]
1470             thsnap["state"] = GetStateDescription(threadsnap['ths_state'])
1471             thsnap["qosEffective"] = threadsnap["ths_eqos"]
1472             thsnap["qosRequested"] = threadsnap["ths_rqos"]
1473
1474             if threadsnap['ths_continuation']:
1475                 thsnap["continuation"] = GetSymbolInfoForFrame(AllImageCatalog, pr_libs, threadsnap['ths_continuation'])
1476             if "kernel_stack_frames" in thdata:
1477                 kuserframes = []
1478                 for f in thdata["kernel_stack_frames"]:
1479                     kuserframes.append(GetSymbolInfoForFrame(AllImageCatalog, pr_libs, f['lr']))
1480                 thsnap["kernelFrames"] = kuserframes
1481
1482             if "user_stack_frames" in thdata:
1483                 uframes = []
1484                 for f in thdata["user_stack_frames"]:
1485                     uframes.append(GetSymbolInfoForFrame(AllImageCatalog, pr_libs, f['lr']))
1486                 thsnap["userFrames"] = uframes
1487             if threadsnap['ths_wait_event']:
1488                 thsnap["waitEvent"] = GetSymbolInfoForFrame(AllImageCatalog, pr_libs, threadsnap['ths_wait_event'])
1489
1490         if 'thread_waitinfo' in piddata:
1491             tsnap['waitInfo'] = map(formatWaitInfo, piddata['thread_waitinfo'])
1492
1493     obj['binaryImages'] = AllImageCatalog
1494     if outfile_name == '-':
1495         fh = sys.stdout
1496     else:
1497         fh = open(outfile_name, "w")
1498
1499     header = {}
1500     header['bug_type'] = 288
1501     if timestamp is not None:
1502         header['timestamp'] = timestamp
1503     header['os_version'] = os_version
1504     fh.write(json.dumps(header))
1505     fh.write("\n")
1506
1507     fh.write(json.dumps(obj, sort_keys=False, indent=2, separators=(',', ': ')))
1508     fh.close()
1509
1510 ## Base utils for interacting with shell ##
1511 def RunCommand(bash_cmd_string, get_stderr = True):
1512     """
1513         returns: (int,str) : exit_code and output_str
1514     """
1515     print "RUNNING: %s" % bash_cmd_string
1516     cmd_args = shlex.split(bash_cmd_string)
1517     output_str = ""
1518     exit_code = 0
1519     try:
1520         if get_stderr:
1521             output_str = subprocess.check_output(cmd_args, stderr=subprocess.STDOUT)
1522         else:
1523             output_str = subprocess.check_output(cmd_args, stderr=None)
1524     except subprocess.CalledProcessError, e:
1525         exit_code = e.returncode
1526     finally:
1527         return (exit_code, output_str)
1528
1529 def ProcessDyldSharedCacheFile(shared_cache_file_path, sdk_str=""):
1530     """ returns (uuid, text_info) output from shared_cache_util.
1531                 In case of error None is returned and err message is printed to stdout.
1532     """
1533     if not os.path.exists(shared_cache_file_path):
1534         print "File path: %s does not exists" % shared_cache_file_path
1535         return None
1536     if sdk_str:
1537         sdk_str = ' -sdk "%s" ' % sdk_str
1538     (c, so) = RunCommand("xcrun {} -find dyld_shared_cache_util".format(sdk_str))
1539     if c:
1540         print "Failed to find path to dyld_shared_cache_util. Exit code: %d , message: %s" % (c,so)
1541         return None
1542     dyld_shared_cache_util = so.strip()
1543     (c, so) = RunCommand("{} -info {}".format(dyld_shared_cache_util, shared_cache_file_path))
1544     if c:
1545         print "Failed to get uuid info from %s" % shared_cache_file_path
1546         print so
1547         return None
1548
1549     uuid = so.splitlines()[0].split(": ")[-1].strip().replace("-","").lower()
1550
1551     (c, so) = RunCommand("{} -text_info {}".format(dyld_shared_cache_util, shared_cache_file_path))
1552     if c:
1553         print "Failed to get text_info from %s" % shared_cache_file_path
1554         print so
1555         return None
1556
1557     print "Found %s uuid: %s" % (shared_cache_file_path, uuid)
1558     text_info = so
1559
1560     return (uuid, so)
1561
1562 parser = argparse.ArgumentParser(description="Decode a kcdata binary file.")
1563 parser.add_argument("-l", "--listtypes", action="store_true", required=False, default=False,
1564                     help="List all known types",
1565                     dest="list_known_types")
1566
1567 parser.add_argument("-s", "--stackshot", required=False, default=False,
1568                     help="Generate a stackshot report file",
1569                     dest="stackshot_file")
1570
1571 parser.add_argument("--multiple", help="look for multiple stackshots in a single file", action='store_true')
1572
1573 parser.add_argument("-p", "--plist", required=False, default=False,
1574                     help="output as plist", action="store_true")
1575
1576 parser.add_argument("-U", "--uuid", required=False, default="", help="UUID of dyld shared cache to be analysed and filled in libs of stackshot report", dest="uuid")
1577 parser.add_argument("-L", "--layout", required=False, type=argparse.FileType("r"), help="Path to layout file for DyldSharedCache. You can generate one by doing \n\tbash$xcrun -sdk <sdk> dyld_shared_cache_util -text_info </path/to/dyld_shared_cache> ", dest="layout")
1578 parser.add_argument("-S", "--sdk", required=False, default="", help="sdk property passed to xcrun command to find the required tools. Default is empty string.", dest="sdk")
1579 parser.add_argument("-D", "--dyld_shared_cache", required=False, default="", help="Path to dyld_shared_cache built by B&I", dest="dsc")
1580 parser.add_argument("--pretty", default=False, action='store_true', help="make the output a little more human readable")
1581 parser.add_argument("--incomplete", action='store_true', help="accept incomplete data")
1582 parser.add_argument("kcdata_file", type=argparse.FileType('r'), help="Path to a kcdata binary file.")
1583
1584 class VerboseAction(argparse.Action):
1585     def __call__(self, parser, namespace, values, option_string=None):
1586         logging.basicConfig(level=logging.INFO, stream=sys.stderr, format='%(message)s')
1587 parser.add_argument('-v', "--verbose", action=VerboseAction, nargs=0)
1588
1589 @contextlib.contextmanager
1590 def data_from_stream(stream):
1591     try:
1592         fmap = mmap.mmap(stream.fileno(), 0, mmap.MAP_SHARED, mmap.PROT_READ)
1593     except:
1594         yield stream.read()
1595     else:
1596         try:
1597             yield fmap
1598         finally:
1599             fmap.close()
1600
1601 def iterate_kcdatas(kcdata_file):
1602     with data_from_stream(kcdata_file) as data:
1603         iterator = kcdata_item_iterator(data)
1604         kcdata_buffer = KCObject.FromKCItem(iterator.next())
1605         if not isinstance(kcdata_buffer, KCBufferObject):
1606             try:
1607                 decoded = base64.b64decode(data)
1608             except:
1609                 pass
1610             else:
1611                 iterator = kcdata_item_iterator(decoded)
1612                 kcdata_buffer = KCObject.FromKCItem(iterator.next())
1613         if not isinstance(kcdata_buffer, KCBufferObject):
1614             import gzip
1615             from io import BytesIO
1616             try:
1617                 decompressed = gzip.GzipFile(fileobj=BytesIO(data[:])).read()
1618             except:
1619                 pass
1620             else:
1621                 iterator = kcdata_item_iterator(decompressed)
1622                 kcdata_buffer = KCObject.FromKCItem(iterator.next())
1623         if not isinstance(kcdata_buffer, KCBufferObject):
1624             raise Exception, "unknown file type"
1625
1626         kcdata_buffer.ReadItems(iterator)
1627         yield kcdata_buffer
1628
1629         for magic in iterator:
1630             kcdata_buffer = KCObject.FromKCItem(magic)
1631             if not isinstance(kcdata_buffer, KCBufferObject):
1632                 raise Exception, "unknown file type"
1633             kcdata_buffer.ReadItems(iterator)
1634             yield kcdata_buffer
1635
1636
1637 def prettify(data):
1638     if isinstance(data, list):
1639         return map(prettify, data);
1640
1641     elif isinstance(data, dict):
1642         newdata = dict()
1643         for key, value in data.items():
1644             if 'uuid' in key.lower() and isinstance(value, list) and len(value) == 16:
1645                 value = '%02X%02X%02X%02X-%02X%02X-%02X%02X-%02X%02X-%02X%02X%02X%02X%02X%02X' % tuple(value)
1646             elif 'address' in key.lower() and isinstance(value, (int, long)):
1647                 value = '0x%X' % value
1648             else:
1649                 value = prettify(value);
1650             newdata[key] = value
1651
1652         return newdata
1653
1654     else:
1655         return data
1656
1657
1658 if __name__ == '__main__':
1659     args = parser.parse_args()
1660
1661     if args.multiple and args.stackshot_file:
1662         raise NotImplementedError
1663
1664     if args.list_known_types:
1665         for (n, t) in KNOWN_TYPES_COLLECTION.items():
1666             print "%d : %s " % (n, str(t))
1667         sys.exit(1)
1668
1669     if args.incomplete or args.stackshot_file:
1670         G.accept_incomplete_data = True
1671
1672     for i,kcdata_buffer in enumerate(iterate_kcdatas(args.kcdata_file)):
1673         if i > 0 and not args.multiple:
1674             break
1675
1676         str_data = "{" + kcdata_buffer.GetJsonRepr() + "}"
1677         str_data = str_data.replace("\t", "    ")
1678
1679         try:
1680             json_obj = json.loads(str_data)
1681         except:
1682             print >>sys.stderr, "JSON reparsing failed!  Printing string data!\n"
1683             import textwrap
1684             print textwrap.fill(str_data, 100)
1685             raise
1686
1687         if args.pretty:
1688             json_obj = prettify(json_obj)
1689
1690         dsc_uuid = None
1691         dsc_libs_arr = []
1692         libs_re = re.compile("^\s*(0x[a-fA-F0-9]+)\s->\s(0x[a-fA-F0-9]+)\s+<([a-fA-F0-9\-]+)>\s+.*$", re.MULTILINE)
1693         if args.uuid and args.layout:
1694             dsc_uuid = args.uuid.strip().replace("-",'').lower()
1695             dsc_libs_arr = libs_re.findall(args.layout.read())
1696
1697         if args.dsc:
1698             _ret = ProcessDyldSharedCacheFile(args.dsc, args.sdk)
1699             if _ret:
1700                 dsc_uuid = _ret[0]
1701                 dsc_libs_arr = libs_re.findall(_ret[1])
1702
1703         if args.stackshot_file:
1704             SaveStackshotReport(json_obj, args.stackshot_file, dsc_uuid, dsc_libs_arr, G.data_was_incomplete)
1705         elif args.plist:
1706             import Foundation
1707             plist = Foundation.NSPropertyListSerialization.dataWithPropertyList_format_options_error_(
1708                 json_obj, Foundation.NSPropertyListXMLFormat_v1_0, 0, None)[0].bytes().tobytes()
1709             #sigh.  on some pythons long integers are getting output with L's in the plist.
1710             plist = re.sub(r'^(\s*<integer>\d+)L(</integer>\s*)$', r"\1\2", plist, flags=re.MULTILINE)
1711             print plist,
1712         else:
1713             print json.dumps(json_obj, sort_keys=True, indent=4, separators=(',', ': '))