osfmk/arm64/pgtrace.c

   1 /*
   2  * Copyright (c) 2015 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #if CONFIG_PGTRACE
  30 #include <mach/mach_types.h>
  31 #include <IOKit/IOLib.h>
  32 #include <sys/msgbuf.h>
  33 #include <sys/errno.h>
  34 #include <arm64/pgtrace.h>
  35 #include <libkern/OSDebug.h>
  36
  37 typedef struct {
  38     queue_chain_t chain;
  39
  40     pmap_t      pmap;
  41     vm_offset_t start;
  42     vm_offset_t end;
  43 } probe_t;
  44
  45 #if CONFIG_PGTRACE_NONKEXT
  46 #include "pgtrace_decoder.h"
  47
  48 //--------------------------------------------
  49 // Macros
  50 //
  51 #define RBUF_DEFAULT_SIZE   1024
  52 #define RBUF_IDX(idx, mask) ((idx) & (mask))
  53 #define MSG_MAX             130
  54
  55 //--------------------------------------------
  56 // Types
  57 //
  58 typedef uint8_t RWLOCK;
  59
  60 typedef struct {
  61     uint64_t                id;
  62     pgtrace_run_result_t    res;
  63     void                    *stack[PGTRACE_STACK_DEPTH];
  64 } log_t;
  65
  66 //--------------------------------------------
  67 // Statics
  68 //
  69 static struct {
  70     log_t           *logs;              // Protect
  71     uint32_t        size;               // Protect
  72     uint64_t        rdidx, wridx;       // Protect
  73     decl_simple_lock_data(, loglock);
  74
  75     uint64_t id;
  76     uint32_t option;
  77     uint32_t enabled;
  78     uint32_t bytes;
  79
  80     queue_head_t    probes;             // Protect
  81
  82     lck_grp_t       *lock_grp;
  83     lck_grp_attr_t  *lock_grp_attr;
  84     lck_attr_t      *lock_attr;
  85     lck_mtx_t       probelock;
  86 } pgtrace = {};
  87
  88 //--------------------------------------------
  89 // Globals
  90 //
  91 void pgtrace_init(void)
  92 {
  93     simple_lock_init(&pgtrace.loglock, 0);
  94
  95     pgtrace.lock_attr = lck_attr_alloc_init();
  96     pgtrace.lock_grp_attr = lck_grp_attr_alloc_init();
  97     pgtrace.lock_grp = lck_grp_alloc_init("pgtrace_lock", pgtrace.lock_grp_attr);
  98
  99     lck_mtx_init(&pgtrace.probelock, pgtrace.lock_grp, pgtrace.lock_attr);
 100
 101     queue_init(&pgtrace.probes);
 102
 103     pgtrace.size = RBUF_DEFAULT_SIZE;
 104     pgtrace.logs = kalloc(RBUF_DEFAULT_SIZE * sizeof(log_t));
 105 }
 106
 107 void pgtrace_clear_probe(void)
 108 {
 109     probe_t *p, *next;
 110     queue_head_t *q = &pgtrace.probes;
 111
 112     lck_mtx_lock(&pgtrace.probelock);
 113
 114     p = (probe_t *)queue_first(q);
 115     while (!queue_end(q, (queue_entry_t)p)) {
 116         next = (probe_t *)queue_next(&(p->chain));
 117
 118         queue_remove(q, p, probe_t *, chain);
 119         kfree(p, sizeof(probe_t));
 120
 121         p = next;
 122     }
 123
 124     lck_mtx_unlock(&pgtrace.probelock);
 125
 126     return;
 127 }
 128
 129 int pgtrace_add_probe(thread_t thread, vm_offset_t start, vm_offset_t end)
 130 {
 131     probe_t *p;
 132     queue_head_t *q = &pgtrace.probes;
 133
 134     if (start > end) {
 135         kprintf("%s Invalid start=%lx end=%lx\n", __func__, start, end);
 136         return -1;
 137     }
 138
 139     p = kalloc(sizeof(probe_t));
 140     p->start = start;
 141     p->end = end;
 142     if (thread == NULL) {
 143         p->pmap = NULL;
 144     } else {
 145         p->pmap = vm_map_pmap(thread->map);
 146     }
 147
 148     lck_mtx_lock(&pgtrace.probelock);
 149     queue_enter(q, p, probe_t *, chain);
 150     lck_mtx_unlock(&pgtrace.probelock);
 151
 152     return 0;
 153 }
 154
 155 void pgtrace_start(void)
 156 {
 157     probe_t *p;
 158     queue_head_t *q = &pgtrace.probes;
 159
 160     kprintf("%s\n", __func__);
 161
 162     if (pgtrace.enabled) {
 163         return;
 164     }
 165
 166     pgtrace.enabled = 1;
 167
 168     lck_mtx_lock(&pgtrace.probelock);
 169
 170     queue_iterate(q, p, probe_t *, chain) {
 171         pmap_pgtrace_add_page(p->pmap, p->start, p->end);
 172     }
 173
 174     lck_mtx_unlock(&pgtrace.probelock);
 175
 176     return;
 177 }
 178
 179 void pgtrace_stop(void)
 180 {
 181     probe_t *p;
 182     queue_head_t *q = &pgtrace.probes;
 183
 184     kprintf("%s\n", __func__);
 185
 186     lck_mtx_lock(&pgtrace.probelock);
 187
 188     queue_iterate(q, p, probe_t *, chain) {
 189         pmap_pgtrace_delete_page(p->pmap, p->start, p->end);
 190     }
 191
 192     lck_mtx_unlock(&pgtrace.probelock);
 193
 194     pgtrace.enabled = 0;
 195 }
 196
 197 uint32_t pgtrace_get_size(void)
 198 {
 199     return pgtrace.size;
 200 }
 201
 202 bool pgtrace_set_size(uint32_t size)
 203 {
 204     log_t *old_buf, *new_buf;
 205     uint32_t old_size, new_size = 1;
 206
 207     // round up to next power of 2
 208     while (size > new_size) {
 209         new_size <<= 1;
 210         if (new_size > 0x100000) {
 211             // over million entries
 212             kprintf("%s: size=%x new_size=%x is too big\n", __func__, size, new_size);
 213             return false;
 214         }
 215     }
 216
 217     new_buf = kalloc(new_size * sizeof(log_t));
 218     if (new_buf == NULL) {
 219         kprintf("%s: can't allocate new_size=%x\n entries", __func__, new_size);
 220         return false;
 221     }
 222
 223     pgtrace_stop();
 224
 225     simple_lock(&pgtrace.loglock);
 226     old_buf = pgtrace.logs;
 227     old_size = pgtrace.size;
 228     pgtrace.logs = new_buf;
 229     pgtrace.size = new_size;
 230     pgtrace.rdidx = pgtrace.wridx = 0;
 231     simple_unlock(&pgtrace.loglock);
 232
 233     if (old_buf) {
 234         kfree(old_buf, old_size * sizeof(log_t));
 235     }
 236
 237     return true;
 238 }
 239
 240 void pgtrace_clear_trace(void)
 241 {
 242     simple_lock(&pgtrace.loglock);
 243     pgtrace.rdidx = pgtrace.wridx = 0;
 244     simple_unlock(&pgtrace.loglock);
 245 }
 246
 247 boolean_t pgtrace_active(void)
 248 {
 249     return (pgtrace.enabled > 0);
 250 }
 251
 252 uint32_t pgtrace_get_option(void)
 253 {
 254     return pgtrace.option;
 255 }
 256
 257 void pgtrace_set_option(uint32_t option)
 258 {
 259     pgtrace.option = option;
 260 }
 261
 262 // pgtrace_write_log() is in interrupt disabled context
 263 void pgtrace_write_log(pgtrace_run_result_t res)
 264 {
 265     uint8_t i;
 266     log_t log = {};
 267     const char *rwmap[] = { "R", "W", "PREFETCH" };
 268
 269     log.id = pgtrace.id++;
 270     log.res = res;
 271
 272     if (pgtrace.option & PGTRACE_OPTION_KPRINTF) {
 273         char msg[MSG_MAX];
 274         char *p;
 275
 276         p = msg;
 277
 278         snprintf(p, MSG_MAX, "%llu %s ", res.rr_time, rwmap[res.rr_rw]);
 279         p += strlen(p);
 280
 281         for (i = 0; i < res.rr_num; i++) {
 282             snprintf(p, MSG_MAX-(p-msg), "%lx=%llx ", res.rr_addrdata[i].ad_addr, res.rr_addrdata[i].ad_data);
 283             p += strlen(p);
 284         }
 285
 286         kprintf("%s %s\n", __func__, msg);
 287     }
 288
 289     if (pgtrace.option & PGTRACE_OPTION_STACK) {
 290         OSBacktrace(log.stack, PGTRACE_STACK_DEPTH);
 291     }
 292
 293     pgtrace.bytes += sizeof(log);
 294
 295     simple_lock(&pgtrace.loglock);
 296
 297     pgtrace.logs[RBUF_IDX(pgtrace.wridx, pgtrace.size-1)] = log;
 298
 299     // Advance rdidx if ring is full
 300     if (RBUF_IDX(pgtrace.wridx, pgtrace.size-1) == RBUF_IDX(pgtrace.rdidx, pgtrace.size-1) &&
 301         (pgtrace.wridx != pgtrace.rdidx)) {
 302         pgtrace.rdidx++;
 303     }
 304     pgtrace.wridx++;
 305
 306     // Signal if ring was empty
 307     if (pgtrace.wridx == (pgtrace.rdidx + 1)) {
 308         thread_wakeup(pgtrace.logs);
 309     }
 310
 311     simple_unlock(&pgtrace.loglock);
 312
 313     return;
 314 }
 315
 316 // pgtrace_read_log() is in user thread
 317 int64_t pgtrace_read_log(uint8_t *buf, uint32_t size)
 318 {
 319     int total, front, back;
 320     boolean_t ints;
 321     wait_result_t wr;
 322
 323     if (pgtrace.enabled == FALSE) {
 324         return -EINVAL;
 325     }
 326
 327     total = size / sizeof(log_t);
 328
 329     // Check if buf is too small
 330     if (buf && total == 0) {
 331         return -EINVAL;
 332     }
 333
 334     ints = ml_set_interrupts_enabled(FALSE);
 335     simple_lock(&pgtrace.loglock);
 336
 337     // Wait if ring is empty
 338     if (pgtrace.rdidx == pgtrace.wridx) {
 339         assert_wait(pgtrace.logs, THREAD_ABORTSAFE);
 340
 341         simple_unlock(&pgtrace.loglock);
 342         ml_set_interrupts_enabled(ints);
 343
 344         wr = thread_block(NULL);
 345         if (wr != THREAD_AWAKENED) {
 346             return -EINTR;
 347         }
 348
 349         ints = ml_set_interrupts_enabled(FALSE);
 350         simple_lock(&pgtrace.loglock);
 351     }
 352
 353     // Trim the size
 354     if ((pgtrace.rdidx + total) > pgtrace.wridx) {
 355         total = (int)(pgtrace.wridx - pgtrace.rdidx);
 356     }
 357
 358     // Copy front
 359     if ((RBUF_IDX(pgtrace.rdidx, pgtrace.size-1) + total) >= pgtrace.size) {
 360         front = pgtrace.size - RBUF_IDX(pgtrace.rdidx, pgtrace.size-1);
 361     } else {
 362         front = total;
 363     }
 364
 365     memcpy(buf, &(pgtrace.logs[RBUF_IDX(pgtrace.rdidx, pgtrace.size-1)]), front*sizeof(log_t));
 366
 367     // Copy back if any
 368     back = total-front;
 369     if (back) {
 370         buf += front * sizeof(log_t);
 371         memcpy(buf, pgtrace.logs, back*sizeof(log_t));
 372     }
 373
 374     pgtrace.rdidx += total;
 375
 376     simple_unlock(&pgtrace.loglock);
 377     ml_set_interrupts_enabled(ints);
 378
 379     return total*sizeof(log_t);
 380 }
 381
 382 int pgtrace_get_stats(pgtrace_stats_t *stats)
 383 {
 384     if (!stats) {
 385         return -1;
 386     }
 387
 388     stats->stat_logger.sl_bytes = pgtrace.bytes;
 389     pgtrace_decoder_get_stats(stats);
 390
 391     return 0;
 392 }
 393
 394 #else // CONFIG_PGTRACE_NONKEXT
 395
 396 static struct {
 397     bool            active;
 398     decoder_t       *decoder;
 399     logger_t        *logger;
 400     queue_head_t    probes;
 401
 402     lck_grp_t       *lock_grp;
 403     lck_grp_attr_t  *lock_grp_attr;
 404     lck_attr_t      *lock_attr;
 405     lck_mtx_t       probelock;
 406 } pgtrace = {};
 407
 408 //------------------------------------
 409 // functions for pmap fault handler
 410 // - pgtrace_decode_and_run
 411 // - pgtrace_write_log
 412 //------------------------------------
 413 int pgtrace_decode_and_run(uint32_t inst, vm_offset_t fva, vm_map_offset_t *cva_page, arm_saved_state_t *ss, pgtrace_run_result_t *res)
 414 {
 415     vm_offset_t pa, cva;
 416     pgtrace_instruction_info_t info;
 417     vm_offset_t cva_front_page = cva_page[0];
 418     vm_offset_t cva_cur_page = cva_page[1];
 419
 420     pgtrace.decoder->decode(inst, ss, &info);
 421
 422     if (info.addr == fva) {
 423         cva = cva_cur_page + (fva & ARM_PGMASK);
 424     } else {
 425         // which means a front page is not a tracing page
 426         cva = cva_front_page + (fva & ARM_PGMASK);
 427     }
 428
 429     pa = mmu_kvtop(cva);
 430     if (!pa) {
 431         panic("%s: invalid address cva=%lx fva=%lx info.addr=%lx inst=%x", __func__, cva, fva, info.addr, inst);
 432     }
 433
 434     absolutetime_to_nanoseconds(mach_absolute_time(), &res->rr_time);
 435
 436     pgtrace.decoder->run(inst, pa, cva, ss, res);
 437
 438     return 0;
 439 }
 440
 441 int pgtrace_write_log(pgtrace_run_result_t res)
 442 {
 443     pgtrace.logger->write(res);
 444     return 0;
 445 }
 446
 447 //------------------------------------
 448 // functions for kext
 449 //  - pgtrace_init
 450 //  - pgtrace_add_probe
 451 //  - pgtrace_clear_probe
 452 //  - pgtrace_start
 453 //  - pgtrace_stop
 454 //  - pgtrace_active
 455 //------------------------------------
 456 int pgtrace_init(decoder_t *decoder, logger_t *logger)
 457 {
 458     kprintf("%s decoder=%p logger=%p\n", __func__, decoder, logger);
 459
 460     assert(decoder && logger);
 461
 462     if (decoder->magic != 0xfeedface || logger->magic != 0xfeedface ||
 463         strcmp(decoder->arch, "arm64") != 0 || strcmp(logger->arch, "arm64") != 0) {
 464         kprintf("%s:wrong decoder/logger magic=%llx/%llx arch=%s/%s", __func__, decoder->magic, logger->magic, decoder->arch, logger->arch);
 465         return EINVAL;
 466     }
 467
 468     pgtrace.lock_attr = lck_attr_alloc_init();
 469     pgtrace.lock_grp_attr = lck_grp_attr_alloc_init();
 470     pgtrace.lock_grp = lck_grp_alloc_init("pgtrace_lock", pgtrace.lock_grp_attr);
 471
 472     lck_mtx_init(&pgtrace.probelock, pgtrace.lock_grp, pgtrace.lock_attr);
 473
 474     queue_init(&pgtrace.probes);
 475     pgtrace.decoder = decoder;
 476     pgtrace.logger = logger;
 477
 478     return 0;
 479 }
 480
 481 int pgtrace_add_probe(thread_t thread, vm_offset_t start, vm_offset_t end)
 482 {
 483     probe_t *p;
 484     queue_head_t *q = &pgtrace.probes;
 485
 486     kprintf("%s start=%lx end=%lx\n", __func__, start, end);
 487
 488     if (start > end) {
 489         kprintf("%s Invalid start=%lx end=%lx\n", __func__, start, end);
 490         return -1;
 491     }
 492
 493     p = kalloc(sizeof(probe_t));
 494     p->start = start;
 495     p->end = end;
 496     if (thread == NULL) {
 497         p->pmap = NULL;
 498     } else {
 499         p->pmap = vm_map_pmap(thread->map);
 500     }
 501
 502     lck_mtx_lock(&pgtrace.probelock);
 503     queue_enter(q, p, probe_t *, chain);
 504     lck_mtx_unlock(&pgtrace.probelock);
 505
 506     return 0;
 507 }
 508
 509 void pgtrace_clear_probe(void)
 510 {
 511     probe_t *p, *next;
 512     queue_head_t *q = &pgtrace.probes;
 513
 514     kprintf("%s\n", __func__);
 515
 516     lck_mtx_lock(&pgtrace.probelock);
 517
 518     p = (probe_t *)queue_first(q);
 519     while (!queue_end(q, (queue_entry_t)p)) {
 520         next = (probe_t *)queue_next(&(p->chain));
 521
 522         queue_remove(q, p, probe_t *, chain);
 523         kfree(p, sizeof(probe_t));
 524
 525         p = next;
 526     }
 527
 528     lck_mtx_unlock(&pgtrace.probelock);
 529
 530     return;
 531 }
 532
 533 void pgtrace_start(void)
 534 {
 535     probe_t *p;
 536     queue_head_t *q = &pgtrace.probes;
 537
 538     kprintf("%s\n", __func__);
 539
 540     if (pgtrace.active == true) {
 541         return;
 542     }
 543
 544     pgtrace.active = true;
 545
 546     lck_mtx_lock(&pgtrace.probelock);
 547
 548     queue_iterate(q, p, probe_t *, chain) {
 549         pmap_pgtrace_add_page(p->pmap, p->start, p->end);
 550     }
 551
 552     lck_mtx_unlock(&pgtrace.probelock);
 553
 554     return;
 555 }
 556
 557 void pgtrace_stop(void)
 558 {
 559     probe_t *p;
 560     queue_head_t *q = &pgtrace.probes;
 561
 562     kprintf("%s\n", __func__);
 563
 564     lck_mtx_lock(&pgtrace.probelock);
 565
 566     queue_iterate(q, p, probe_t *, chain) {
 567         pmap_pgtrace_delete_page(p->pmap, p->start, p->end);
 568     }
 569
 570     lck_mtx_unlock(&pgtrace.probelock);
 571
 572     pgtrace.active = false;
 573 }
 574
 575 bool pgtrace_active(void)
 576 {
 577     return pgtrace.active;
 578 }
 579 #endif // CONFIG_PGTRACE_NONKEXT
 580 #else
 581 // empty funcs for release kernel
 582 extern void pgtrace_stop(void);
 583 extern void pgtrace_start(void);
 584 extern void pgtrace_clear_probe(void);
 585 extern void pgtrace_add_probe(void);
 586 extern void pgtrace_init(void);
 587 extern void pgtrace_active(void);
 588 void pgtrace_stop(void) {}
 589 void pgtrace_start(void) {}
 590 void pgtrace_clear_probe(void) {}
 591 void pgtrace_add_probe(void) {}
 592 void pgtrace_init(void) {}
 593 void pgtrace_active(void) {}
 594 #endif