bsd/dev/dtrace/fbt.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25
  26 #include <mach-o/loader.h>
  27 #include <libkern/kernel_mach_header.h>
  28
  29 #include <sys/param.h>
  30 #include <sys/systm.h>
  31 #include <sys/sysctl.h>
  32 #include <sys/errno.h>
  33 #include <sys/stat.h>
  34 #include <sys/ioctl.h>
  35 #include <sys/conf.h>
  36 #include <sys/fcntl.h>
  37 #include <miscfs/devfs/devfs.h>
  38 #include <pexpert/pexpert.h>
  39
  40 #include <sys/dtrace.h>
  41 #include <sys/dtrace_impl.h>
  42 #include <sys/fbt.h>
  43
  44 #include <sys/dtrace_glue.h>
  45 #include <san/kasan.h>
  46
  47 /* #include <machine/trap.h> */
  48 struct savearea_t; /* Used anonymously */
  49
  50 #if defined(__arm__) || defined(__arm64__)
  51 typedef kern_return_t (*perfCallback)(int, struct savearea_t *, __unused int, __unused int);
  52 extern perfCallback tempDTraceTrapHook;
  53 extern kern_return_t fbt_perfCallback(int, struct savearea_t *, __unused int, __unused int);
  54 #elif defined(__x86_64__)
  55 typedef kern_return_t (*perfCallback)(int, struct savearea_t *, uintptr_t *, __unused int);
  56 extern perfCallback tempDTraceTrapHook;
  57 extern kern_return_t fbt_perfCallback(int, struct savearea_t *, uintptr_t *, __unused int);
  58 #else
  59 #error Unknown architecture
  60 #endif
  61
  62 __private_extern__
  63 void
  64 qsort(void *a, size_t n, size_t es, int (*cmp)(const void *, const void *));
  65
  66 #define FBT_ADDR2NDX(addr)      ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask)
  67 #define FBT_PROBETAB_SIZE       0x8000          /* 32k entries -- 128K total */
  68
  69 static int                              fbt_probetab_size;
  70 dtrace_provider_id_t    fbt_id;
  71 fbt_probe_t                             **fbt_probetab;
  72 int                                             fbt_probetab_mask;
  73 static int                              fbt_verbose = 0;
  74
  75 extern int ignore_fbt_blacklist;
  76
  77 extern int dtrace_kernel_symbol_mode;
  78
  79
  80 void fbt_init( void );
  81
  82 /*ARGSUSED*/
  83 static void
  84 fbt_destroy(void *arg, dtrace_id_t id, void *parg)
  85 {
  86 #pragma unused(arg,id)
  87         fbt_probe_t *fbt = parg, *next, *hash, *last;
  88         int ndx;
  89
  90         do {
  91                 /*
  92                  * Now we need to remove this probe from the fbt_probetab.
  93                  */
  94                 ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint);
  95                 last = NULL;
  96                 hash = fbt_probetab[ndx];
  97
  98                 while (hash != fbt) {
  99                         ASSERT(hash != NULL);
 100                         last = hash;
 101                         hash = hash->fbtp_hashnext;
 102                 }
 103
 104                 if (last != NULL) {
 105                         last->fbtp_hashnext = fbt->fbtp_hashnext;
 106                 } else {
 107                         fbt_probetab[ndx] = fbt->fbtp_hashnext;
 108                 }
 109
 110                 next = fbt->fbtp_next;
 111                 kmem_free(fbt, sizeof(fbt_probe_t));
 112
 113                 fbt = next;
 114         } while (fbt != NULL);
 115 }
 116
 117 /*ARGSUSED*/
 118 int
 119 fbt_enable(void *arg, dtrace_id_t id, void *parg)
 120 {
 121 #pragma unused(arg,id)
 122         fbt_probe_t *fbt = parg;
 123         struct modctl *ctl = NULL;
 124
 125         for (; fbt != NULL; fbt = fbt->fbtp_next) {
 126                 ctl = fbt->fbtp_ctl;
 127
 128                 if (!ctl->mod_loaded) {
 129                         if (fbt_verbose) {
 130                                 cmn_err(CE_NOTE, "fbt is failing for probe %s "
 131                                     "(module %s unloaded)",
 132                                     fbt->fbtp_name, ctl->mod_modname);
 133                         }
 134
 135                         continue;
 136                 }
 137
 138                 /*
 139                  * Now check that our modctl has the expected load count.  If it
 140                  * doesn't, this module must have been unloaded and reloaded -- and
 141                  * we're not going to touch it.
 142                  */
 143                 if (ctl->mod_loadcnt != fbt->fbtp_loadcnt) {
 144                         if (fbt_verbose) {
 145                                 cmn_err(CE_NOTE, "fbt is failing for probe %s "
 146                                     "(module %s reloaded)",
 147                                     fbt->fbtp_name, ctl->mod_modname);
 148                         }
 149
 150                         continue;
 151                 }
 152
 153                 dtrace_casptr(&tempDTraceTrapHook, NULL, fbt_perfCallback);
 154                 if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) {
 155                         if (fbt_verbose) {
 156                                 cmn_err(CE_NOTE, "fbt_enable is failing for probe %s "
 157                                     "in module %s: tempDTraceTrapHook already occupied.",
 158                                     fbt->fbtp_name, ctl->mod_modname);
 159                         }
 160                         continue;
 161                 }
 162
 163                 if (fbt->fbtp_currentval != fbt->fbtp_patchval) {
 164 #if KASAN
 165                         /* Since dtrace probes can call into KASan and vice versa, things can get
 166                          * very slow if we have a lot of probes. This call will disable the KASan
 167                          * fakestack after a threshold of probes is reached. */
 168                         kasan_fakestack_suspend();
 169 #endif
 170
 171                         (void)ml_nofault_copy((vm_offset_t)&fbt->fbtp_patchval, (vm_offset_t)fbt->fbtp_patchpoint,
 172                             sizeof(fbt->fbtp_patchval));
 173                         /*
 174                          * Make the patched instruction visible via a data + instruction
 175                          * cache flush for the platforms that need it
 176                          */
 177                         flush_dcache((vm_offset_t)fbt->fbtp_patchpoint, (vm_size_t)sizeof(fbt->fbtp_patchval), 0);
 178                         invalidate_icache((vm_offset_t)fbt->fbtp_patchpoint, (vm_size_t)sizeof(fbt->fbtp_patchval), 0);
 179                         fbt->fbtp_currentval = fbt->fbtp_patchval;
 180
 181                         ctl->mod_nenabled++;
 182                 }
 183         }
 184
 185         dtrace_membar_consumer();
 186
 187         return 0;
 188 }
 189
 190 /*ARGSUSED*/
 191 static void
 192 fbt_disable(void *arg, dtrace_id_t id, void *parg)
 193 {
 194 #pragma unused(arg,id)
 195         fbt_probe_t *fbt = parg;
 196         struct modctl *ctl = NULL;
 197
 198         for (; fbt != NULL; fbt = fbt->fbtp_next) {
 199                 ctl = fbt->fbtp_ctl;
 200
 201                 if (!ctl->mod_loaded || (ctl->mod_loadcnt != fbt->fbtp_loadcnt)) {
 202                         continue;
 203                 }
 204
 205                 if (fbt->fbtp_currentval != fbt->fbtp_savedval) {
 206                         (void)ml_nofault_copy((vm_offset_t)&fbt->fbtp_savedval, (vm_offset_t)fbt->fbtp_patchpoint,
 207                             sizeof(fbt->fbtp_savedval));
 208                         /*
 209                          * Make the patched instruction visible via a data + instruction
 210                          * cache flush for the platforms that need it
 211                          */
 212                         flush_dcache((vm_offset_t)fbt->fbtp_patchpoint, (vm_size_t)sizeof(fbt->fbtp_patchval), 0);
 213                         invalidate_icache((vm_offset_t)fbt->fbtp_patchpoint, (vm_size_t)sizeof(fbt->fbtp_patchval), 0);
 214
 215                         fbt->fbtp_currentval = fbt->fbtp_savedval;
 216                         ASSERT(ctl->mod_nenabled > 0);
 217                         ctl->mod_nenabled--;
 218
 219 #if KASAN
 220                         kasan_fakestack_resume();
 221 #endif
 222                 }
 223         }
 224         dtrace_membar_consumer();
 225 }
 226
 227 /*ARGSUSED*/
 228 static void
 229 fbt_suspend(void *arg, dtrace_id_t id, void *parg)
 230 {
 231 #pragma unused(arg,id)
 232         fbt_probe_t *fbt = parg;
 233         struct modctl *ctl = NULL;
 234
 235         for (; fbt != NULL; fbt = fbt->fbtp_next) {
 236                 ctl = fbt->fbtp_ctl;
 237
 238                 ASSERT(ctl->mod_nenabled > 0);
 239                 if (!ctl->mod_loaded || (ctl->mod_loadcnt != fbt->fbtp_loadcnt)) {
 240                         continue;
 241                 }
 242
 243                 (void)ml_nofault_copy((vm_offset_t)&fbt->fbtp_savedval, (vm_offset_t)fbt->fbtp_patchpoint,
 244                     sizeof(fbt->fbtp_savedval));
 245
 246                 /*
 247                  * Make the patched instruction visible via a data + instruction
 248                  * cache flush for the platforms that need it
 249                  */
 250                 flush_dcache((vm_offset_t)fbt->fbtp_patchpoint, (vm_size_t)sizeof(fbt->fbtp_savedval), 0);
 251                 invalidate_icache((vm_offset_t)fbt->fbtp_patchpoint, (vm_size_t)sizeof(fbt->fbtp_savedval), 0);
 252
 253                 fbt->fbtp_currentval = fbt->fbtp_savedval;
 254         }
 255
 256         dtrace_membar_consumer();
 257 }
 258
 259 /*ARGSUSED*/
 260 static void
 261 fbt_resume(void *arg, dtrace_id_t id, void *parg)
 262 {
 263 #pragma unused(arg,id)
 264         fbt_probe_t *fbt = parg;
 265         struct modctl *ctl = NULL;
 266
 267         for (; fbt != NULL; fbt = fbt->fbtp_next) {
 268                 ctl = fbt->fbtp_ctl;
 269
 270                 ASSERT(ctl->mod_nenabled > 0);
 271                 if (!ctl->mod_loaded || (ctl->mod_loadcnt != fbt->fbtp_loadcnt)) {
 272                         continue;
 273                 }
 274
 275                 dtrace_casptr(&tempDTraceTrapHook, NULL, fbt_perfCallback);
 276                 if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) {
 277                         if (fbt_verbose) {
 278                                 cmn_err(CE_NOTE, "fbt_resume is failing for probe %s "
 279                                     "in module %s: tempDTraceTrapHook already occupied.",
 280                                     fbt->fbtp_name, ctl->mod_modname);
 281                         }
 282                         return;
 283                 }
 284
 285                 (void)ml_nofault_copy((vm_offset_t)&fbt->fbtp_patchval, (vm_offset_t)fbt->fbtp_patchpoint,
 286                     sizeof(fbt->fbtp_patchval));
 287
 288                 /*
 289                  * Make the patched instruction visible via a data + instruction cache flush.
 290                  */
 291                 flush_dcache((vm_offset_t)fbt->fbtp_patchpoint, (vm_size_t)sizeof(fbt->fbtp_patchval), 0);
 292                 invalidate_icache((vm_offset_t)fbt->fbtp_patchpoint, (vm_size_t)sizeof(fbt->fbtp_patchval), 0);
 293
 294                 fbt->fbtp_currentval = fbt->fbtp_patchval;
 295         }
 296
 297         dtrace_membar_consumer();
 298 }
 299
 300 static void
 301 fbt_provide_module_user_syms(struct modctl *ctl)
 302 {
 303         unsigned int i;
 304         char *modname = ctl->mod_modname;
 305
 306         dtrace_module_symbols_t* module_symbols = ctl->mod_user_symbols;
 307         if (module_symbols) {
 308                 for (i = 0; i < module_symbols->dtmodsyms_count; i++) {
 309                         /*
 310                          * symbol->dtsym_addr (the symbol address) passed in from
 311                          * user space, is already slid for both kexts and kernel.
 312                          */
 313                         dtrace_symbol_t* symbol = &module_symbols->dtmodsyms_symbols[i];
 314
 315                         char* name = symbol->dtsym_name;
 316
 317                         /* Lop off omnipresent leading underscore. */
 318                         if (*name == '_') {
 319                                 name += 1;
 320                         }
 321
 322                         if (fbt_excluded(name)) {
 323                                 continue;
 324                         }
 325
 326                         /*
 327                          * Ignore symbols with a null address
 328                          */
 329                         if (!symbol->dtsym_addr) {
 330                                 continue;
 331                         }
 332
 333                         /*
 334                          * Ignore symbols not part of this module
 335                          */
 336                         if (!dtrace_addr_in_module((void*)symbol->dtsym_addr, ctl)) {
 337                                 continue;
 338                         }
 339
 340                         fbt_provide_probe(ctl, modname, name, (machine_inst_t*)(uintptr_t)symbol->dtsym_addr, (machine_inst_t*)(uintptr_t)(symbol->dtsym_addr + symbol->dtsym_size));
 341                 }
 342         }
 343 }
 344 static void
 345 fbt_provide_kernel_section(struct modctl *ctl, kernel_section_t *sect, kernel_nlist_t *sym, uint32_t nsyms, const char *strings)
 346 {
 347         uintptr_t sect_start = (uintptr_t)sect->addr;
 348         uintptr_t sect_end = (uintptr_t)sect->size + sect->addr;
 349         unsigned int i;
 350
 351         if ((sect->flags & S_ATTR_PURE_INSTRUCTIONS) != S_ATTR_PURE_INSTRUCTIONS) {
 352                 return;
 353         }
 354
 355         for (i = 0; i < nsyms; i++) {
 356                 uint8_t         n_type = sym[i].n_type & (N_TYPE | N_EXT);
 357                 const char           *name = strings + sym[i].n_un.n_strx;
 358                 uint64_t limit;
 359
 360                 if (sym[i].n_value < sect_start || sym[i].n_value > sect_end) {
 361                         continue;
 362                 }
 363
 364                 /* Check that the symbol is a global and that it has a name. */
 365                 if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type)) {
 366                         continue;
 367                 }
 368
 369                 if (0 == sym[i].n_un.n_strx) {  /* iff a null, "", name. */
 370                         continue;
 371                 }
 372
 373                 /* Lop off omnipresent leading underscore. */
 374                 if (*name == '_') {
 375                         name += 1;
 376                 }
 377
 378 #if defined(__arm__)
 379                 // Skip non-thumb functions on arm32
 380                 if (sym[i].n_sect == 1 && !(sym[i].n_desc & N_ARM_THUMB_DEF)) {
 381                         continue;
 382                 }
 383 #endif /* defined(__arm__) */
 384
 385                 if (fbt_excluded(name)) {
 386                         continue;
 387                 }
 388
 389                 /*
 390                  * Find the function boundary by looking at either the
 391                  * end of the section or the beginning of the next symbol
 392                  */
 393                 if (i == nsyms - 1) {
 394                         limit = sect_end;
 395                 } else {
 396                         limit = sym[i + 1].n_value;
 397                 }
 398
 399                 fbt_provide_probe(ctl, ctl->mod_modname, name, (machine_inst_t*)sym[i].n_value, (machine_inst_t*)limit);
 400         }
 401 }
 402
 403 static int
 404 fbt_sym_cmp(const void *ap, const void *bp)
 405 {
 406         return (int)(((const kernel_nlist_t*)ap)->n_value - ((const kernel_nlist_t*)bp)->n_value);
 407 }
 408
 409 static void
 410 fbt_provide_module_kernel_syms(struct modctl *ctl)
 411 {
 412         kernel_mach_header_t *mh = (kernel_mach_header_t *)(ctl->mod_address);
 413         kernel_segment_command_t *seg;
 414         struct load_command *cmd;
 415         kernel_segment_command_t *linkedit = NULL;
 416         struct symtab_command *symtab = NULL;
 417         kernel_nlist_t *syms = NULL, *sorted_syms = NULL;
 418         const char *strings;
 419         unsigned int i;
 420         size_t symlen;
 421
 422         if (mh->magic != MH_MAGIC_KERNEL) {
 423                 return;
 424         }
 425
 426         cmd = (struct load_command *) &mh[1];
 427         for (i = 0; i < mh->ncmds; i++) {
 428                 if (cmd->cmd == LC_SEGMENT_KERNEL) {
 429                         kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd;
 430                         if (LIT_STRNEQL(orig_sg->segname, SEG_LINKEDIT)) {
 431                                 linkedit = orig_sg;
 432                         }
 433                 } else if (cmd->cmd == LC_SYMTAB) {
 434                         symtab = (struct symtab_command *) cmd;
 435                 }
 436                 if (symtab && linkedit) {
 437                         break;
 438                 }
 439                 cmd = (struct load_command *) ((caddr_t) cmd + cmd->cmdsize);
 440         }
 441
 442         if ((symtab == NULL) || (linkedit == NULL)) {
 443                 return;
 444         }
 445
 446         syms = (kernel_nlist_t *)(linkedit->vmaddr + symtab->symoff - linkedit->fileoff);
 447         strings = (const char *)(linkedit->vmaddr + symtab->stroff - linkedit->fileoff);
 448
 449         /*
 450          * Make a copy of the symbol table and sort it to not cross into the next function
 451          * when disassembling the function
 452          */
 453         symlen = sizeof(kernel_nlist_t) * symtab->nsyms;
 454         sorted_syms = kmem_alloc(symlen, KM_SLEEP);
 455         bcopy(syms, sorted_syms, symlen);
 456         qsort(sorted_syms, symtab->nsyms, sizeof(kernel_nlist_t), fbt_sym_cmp);
 457
 458         for (seg = firstsegfromheader(mh); seg != NULL; seg = nextsegfromheader(mh, seg)) {
 459                 kernel_section_t *sect = firstsect(seg);
 460
 461                 if (strcmp(seg->segname, "__KLD") == 0) {
 462                         continue;
 463                 }
 464
 465                 for (sect = firstsect(seg); sect != NULL; sect = nextsect(seg, sect)) {
 466                         fbt_provide_kernel_section(ctl, sect, sorted_syms, symtab->nsyms, strings);
 467                 }
 468         }
 469
 470         kmem_free(sorted_syms, symlen);
 471 }
 472
 473 void
 474 fbt_provide_module(void *arg, struct modctl *ctl)
 475 {
 476 #pragma unused(arg)
 477         ASSERT(ctl != NULL);
 478         ASSERT(dtrace_kernel_symbol_mode != DTRACE_KERNEL_SYMBOLS_NEVER);
 479         LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED);
 480
 481         // Update the "ignore blacklist" bit
 482         if (ignore_fbt_blacklist) {
 483                 ctl->mod_flags |= MODCTL_FBT_PROVIDE_BLACKLISTED_PROBES;
 484         }
 485
 486         if (MOD_FBT_DONE(ctl)) {
 487                 return;
 488         }
 489
 490         if (fbt_module_excluded(ctl)) {
 491                 ctl->mod_flags |= MODCTL_FBT_INVALID;
 492                 return;
 493         }
 494
 495         if (MOD_HAS_KERNEL_SYMBOLS(ctl)) {
 496                 fbt_provide_module_kernel_syms(ctl);
 497                 ctl->mod_flags |= MODCTL_FBT_PROBES_PROVIDED;
 498                 if (MOD_FBT_PROVIDE_BLACKLISTED_PROBES(ctl)) {
 499                         ctl->mod_flags |= MODCTL_FBT_BLACKLISTED_PROBES_PROVIDED;
 500                 }
 501                 return;
 502         }
 503
 504         if (MOD_HAS_USERSPACE_SYMBOLS(ctl)) {
 505                 fbt_provide_module_user_syms(ctl);
 506                 ctl->mod_flags |= MODCTL_FBT_PROBES_PROVIDED;
 507                 if (MOD_FBT_PROVIDE_BLACKLISTED_PROBES(ctl)) {
 508                         ctl->mod_flags |= MODCTL_FBT_BLACKLISTED_PROBES_PROVIDED;
 509                 }
 510                 return;
 511         }
 512 }
 513
 514 static dtrace_pattr_t fbt_attr = {
 515         { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
 516         { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
 517         { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
 518         { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
 519         { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
 520 };
 521
 522 static dtrace_pops_t fbt_pops = {
 523         .dtps_provide =         NULL,
 524         .dtps_provide_module =  fbt_provide_module,
 525         .dtps_enable =          fbt_enable,
 526         .dtps_disable =         fbt_disable,
 527         .dtps_suspend =         fbt_suspend,
 528         .dtps_resume =          fbt_resume,
 529         .dtps_getargdesc =      NULL, /* APPLE NOTE: fbt_getargdesc implemented in userspace */
 530         .dtps_getargval =       NULL,
 531         .dtps_usermode =        NULL,
 532         .dtps_destroy =         fbt_destroy
 533 };
 534
 535 static void
 536 fbt_cleanup(dev_info_t *devi)
 537 {
 538         dtrace_invop_remove(fbt_invop);
 539         ddi_remove_minor_node(devi, NULL);
 540         kmem_free(fbt_probetab, fbt_probetab_size * sizeof(fbt_probe_t *));
 541         fbt_probetab = NULL;
 542         fbt_probetab_mask = 0;
 543 }
 544
 545 static int
 546 fbt_attach(dev_info_t *devi)
 547 {
 548         if (fbt_probetab_size == 0) {
 549                 fbt_probetab_size = FBT_PROBETAB_SIZE;
 550         }
 551
 552         fbt_probetab_mask = fbt_probetab_size - 1;
 553         fbt_probetab =
 554             kmem_zalloc(fbt_probetab_size * sizeof(fbt_probe_t *), KM_SLEEP);
 555
 556         dtrace_invop_add(fbt_invop);
 557
 558         if (ddi_create_minor_node(devi, "fbt", S_IFCHR, 0,
 559             DDI_PSEUDO, 0) == DDI_FAILURE ||
 560             dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_KERNEL, NULL,
 561             &fbt_pops, NULL, &fbt_id) != 0) {
 562                 fbt_cleanup(devi);
 563                 return DDI_FAILURE;
 564         }
 565
 566         return DDI_SUCCESS;
 567 }
 568
 569 static d_open_t _fbt_open;
 570
 571 static int
 572 _fbt_open(dev_t dev, int flags, int devtype, struct proc *p)
 573 {
 574 #pragma unused(dev,flags,devtype,p)
 575         return 0;
 576 }
 577
 578 #define FBT_MAJOR  -24 /* let the kernel pick the device number */
 579
 580
 581 /*
 582  * A struct describing which functions will get invoked for certain
 583  * actions.
 584  */
 585 static struct cdevsw fbt_cdevsw =
 586 {
 587         _fbt_open,              /* open */
 588         eno_opcl,                       /* close */
 589         eno_rdwrt,                      /* read */
 590         eno_rdwrt,                      /* write */
 591         eno_ioctl,                      /* ioctl */
 592         (stop_fcn_t *)nulldev, /* stop */
 593         (reset_fcn_t *)nulldev, /* reset */
 594         NULL,                           /* tty's */
 595         eno_select,                     /* select */
 596         eno_mmap,                       /* mmap */
 597         eno_strat,                      /* strategy */
 598         eno_getc,                       /* getc */
 599         eno_putc,                       /* putc */
 600         0                                       /* type */
 601 };
 602
 603 #undef kmem_alloc /* from its binding to dt_kmem_alloc glue */
 604 #undef kmem_free /* from its binding to dt_kmem_free glue */
 605 #include <vm/vm_kern.h>
 606
 607
 608 void
 609 fbt_init( void )
 610 {
 611         int majdevno = cdevsw_add(FBT_MAJOR, &fbt_cdevsw);
 612
 613         if (majdevno < 0) {
 614                 printf("fbt_init: failed to allocate a major number!\n");
 615                 return;
 616         }
 617
 618         fbt_blacklist_init();
 619         fbt_attach((dev_info_t*)(uintptr_t)majdevno);
 620 }
 621 #undef FBT_MAJOR