bsd/dev/arm64/dtrace_isa.c

   1 /*
   2  * Copyright (c) 2005-2008 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #define MACH__POSIX_C_SOURCE_PRIVATE 1  /* pulls in suitable savearea from
  30                                          * mach/ppc/thread_status.h */
  31 #include <arm/proc_reg.h>
  32
  33 #include <kern/thread.h>
  34 #include <mach/thread_status.h>
  35
  36 #include <stdarg.h>
  37 #include <string.h>
  38 #include <sys/malloc.h>
  39 #include <sys/time.h>
  40 #include <sys/systm.h>
  41 #include <sys/proc.h>
  42 #include <sys/proc_internal.h>
  43 #include <sys/kauth.h>
  44 #include <sys/dtrace.h>
  45 #include <sys/dtrace_impl.h>
  46 #include <libkern/OSAtomic.h>
  47 #include <kern/simple_lock.h>
  48 #include <kern/sched_prim.h>            /* for thread_wakeup() */
  49 #include <kern/thread_call.h>
  50 #include <kern/task.h>
  51 #include <miscfs/devfs/devfs.h>
  52 #include <mach/vm_param.h>
  53
  54 extern struct arm_saved_state *find_kern_regs(thread_t);
  55
  56 extern dtrace_id_t      dtrace_probeid_error;   /* special ERROR probe */
  57 typedef arm_saved_state_t savearea_t;
  58
  59 extern lck_attr_t       *dtrace_lck_attr;
  60 extern lck_grp_t        *dtrace_lck_grp;
  61
  62
  63 struct frame {
  64         struct frame *backchain;
  65         uintptr_t retaddr;
  66 };
  67
  68 /*
  69  * Atomicity and synchronization
  70  */
  71 inline void
  72 dtrace_membar_producer(void)
  73 {
  74 #if __ARM_SMP__
  75         __asm__ volatile("dmb ish" : : : "memory");
  76 #else
  77         __asm__ volatile("nop" : : : "memory");
  78 #endif
  79 }
  80
  81 inline void
  82 dtrace_membar_consumer(void)
  83 {
  84 #if __ARM_SMP__
  85         __asm__ volatile("dmb ish" : : : "memory");
  86 #else
  87         __asm__ volatile("nop" : : : "memory");
  88 #endif
  89 }
  90
  91 /*
  92  * Interrupt manipulation
  93  * XXX dtrace_getipl() can be called from probe context.
  94  */
  95 int
  96 dtrace_getipl(void)
  97 {
  98         /*
  99          * XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE
 100          * in osfmk/kern/cpu_data.h
 101          */
 102         /* return get_interrupt_level(); */
 103         return (ml_at_interrupt_context() ? 1 : 0);
 104 }
 105
 106 #if __ARM_SMP__
 107 /*
 108  * MP coordination
 109  */
 110
 111 decl_lck_mtx_data(static, dt_xc_lock);
 112 static uint32_t dt_xc_sync;
 113
 114 typedef struct xcArg {
 115         processorid_t   cpu;
 116         dtrace_xcall_t  f;
 117         void           *arg;
 118 } xcArg_t;
 119
 120 static void
 121 xcRemote(void *foo)
 122 {
 123         xcArg_t *pArg = (xcArg_t *) foo;
 124
 125         if (pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL)
 126                 (pArg->f) (pArg->arg);
 127
 128         if (hw_atomic_sub(&dt_xc_sync, 1) == 0)
 129                 thread_wakeup((event_t) &dt_xc_sync);
 130 }
 131 #endif
 132
 133 /*
 134  * dtrace_xcall() is not called from probe context.
 135  */
 136 void
 137 dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg)
 138 {
 139 #if __ARM_SMP__
 140         /* Only one dtrace_xcall in flight allowed */
 141         lck_mtx_lock(&dt_xc_lock);
 142
 143         xcArg_t xcArg;
 144
 145         xcArg.cpu = cpu;
 146         xcArg.f = f;
 147         xcArg.arg = arg;
 148
 149         cpu_broadcast_xcall(&dt_xc_sync, TRUE, xcRemote, (void*) &xcArg);
 150
 151         lck_mtx_unlock(&dt_xc_lock);
 152         return;
 153 #else
 154 #pragma unused(cpu)
 155         /* On uniprocessor systems, the cpu should always be either ourselves or all */
 156         ASSERT(cpu == CPU->cpu_id || cpu == DTRACE_CPUALL);
 157
 158         (*f)(arg);
 159         return;
 160 #endif
 161 }
 162
 163 /*
 164  * Initialization
 165  */
 166 void
 167 dtrace_isa_init(void)
 168 {
 169         lck_mtx_init(&dt_xc_lock, dtrace_lck_grp, dtrace_lck_attr);
 170         return;
 171 }
 172
 173
 174 /**
 175  * Register definitions
 176  */
 177 #define ARM_FP 7
 178 #define ARM_SP 13
 179 #define ARM_LR 14
 180 #define ARM_PC 15
 181 #define ARM_CPSR 16
 182
 183 #define ARM64_FP 29
 184 #define ARM64_LR 30
 185 #define ARM64_SP 31
 186 #define ARM64_PC 32
 187 #define ARM64_CPSR 33
 188
 189 /*
 190  * Runtime and ABI
 191  */
 192 uint64_t
 193 dtrace_getreg(struct regs * savearea, uint_t reg)
 194 {
 195         struct arm_saved_state *regs = (struct arm_saved_state *) savearea;
 196
 197         if (is_saved_state32(regs)) {
 198                 // Fix special registers if user is 32 bits
 199                 switch (reg) {
 200                         case ARM64_FP:
 201                                 reg = ARM_FP;
 202                         break;
 203                         case ARM64_SP:
 204                                 reg = ARM_SP;
 205                         break;
 206                         case ARM64_LR:
 207                                 reg = ARM_LR;
 208                         break;
 209                         case ARM64_PC:
 210                                 reg = ARM_PC;
 211                         break;
 212                         case ARM64_CPSR:
 213                                 reg = ARM_CPSR;
 214                         break;
 215                 }
 216         }
 217
 218         if (!check_saved_state_reglimit(regs, reg)) {
 219                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 220                 return (0);
 221         }
 222
 223         return ((uint64_t)get_saved_state_reg(regs, reg));
 224 }
 225
 226 #define RETURN_OFFSET 4
 227 #define RETURN_OFFSET64 8
 228
 229 static int
 230 dtrace_getustack_common(uint64_t * pcstack, int pcstack_limit, user_addr_t pc,
 231                         user_addr_t sp)
 232 {
 233         int ret = 0;
 234         boolean_t is64bit = proc_is64bit(current_proc());
 235
 236         ASSERT(pcstack == NULL || pcstack_limit > 0);
 237
 238         while (pc != 0) {
 239                 ret++;
 240                 if (pcstack != NULL) {
 241                         *pcstack++ = (uint64_t) pc;
 242                         pcstack_limit--;
 243                         if (pcstack_limit <= 0)
 244                                 break;
 245                 }
 246
 247                 if (sp == 0)
 248                         break;
 249
 250                 if (is64bit) {
 251                         pc = dtrace_fuword64((sp + RETURN_OFFSET64));
 252                         sp = dtrace_fuword64(sp);
 253                 } else {
 254                         pc = dtrace_fuword32((sp + RETURN_OFFSET));
 255                         sp = dtrace_fuword32(sp);
 256                 }
 257         }
 258
 259         return (ret);
 260 }
 261
 262 void
 263 dtrace_getupcstack(uint64_t * pcstack, int pcstack_limit)
 264 {
 265         thread_t        thread = current_thread();
 266         savearea_t     *regs;
 267         user_addr_t     pc, sp, fp;
 268         volatile uint16_t *flags = (volatile uint16_t *) & cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 269         int n;
 270
 271         if (*flags & CPU_DTRACE_FAULT)
 272                 return;
 273
 274         if (pcstack_limit <= 0)
 275                 return;
 276
 277         /*
 278          * If there's no user context we still need to zero the stack.
 279          */
 280         if (thread == NULL)
 281                 goto zero;
 282
 283         regs = (savearea_t *) find_user_regs(thread);
 284         if (regs == NULL)
 285                 goto zero;
 286
 287         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 288         pcstack_limit--;
 289
 290         if (pcstack_limit <= 0)
 291                 return;
 292
 293         pc = get_saved_state_pc(regs);
 294         sp = get_saved_state_sp(regs);
 295         fp = get_saved_state_fp(regs);
 296
 297         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 298                 *pcstack++ = (uint64_t) pc;
 299                 pcstack_limit--;
 300                 if (pcstack_limit <= 0)
 301                         return;
 302
 303                 pc = get_saved_state_lr(regs);
 304         }
 305
 306         n = dtrace_getustack_common(pcstack, pcstack_limit, pc, fp);
 307
 308         ASSERT(n >= 0);
 309         ASSERT(n <= pcstack_limit);
 310
 311         pcstack += n;
 312         pcstack_limit -= n;
 313
 314 zero:
 315         while (pcstack_limit-- > 0)
 316                 *pcstack++ = 0ULL;
 317 }
 318
 319 int
 320 dtrace_getustackdepth(void)
 321 {
 322         thread_t        thread = current_thread();
 323         savearea_t     *regs;
 324         user_addr_t     pc, sp, fp;
 325         int             n = 0;
 326
 327         if (thread == NULL)
 328                 return 0;
 329
 330         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT))
 331                 return (-1);
 332
 333         regs = (savearea_t *) find_user_regs(thread);
 334         if (regs == NULL)
 335                 return 0;
 336
 337         pc = get_saved_state_pc(regs);
 338         sp = get_saved_state_sp(regs);
 339         fp = get_saved_state_fp(regs);
 340
 341         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 342                 n++;
 343                 pc = get_saved_state_lr(regs);
 344         }
 345
 346         /*
 347          * Note that unlike ppc, the arm code does not use
 348          * CPU_DTRACE_USTACK_FP. This is because arm always
 349          * traces from the sp, even in syscall/profile/fbt
 350          * providers.
 351          */
 352
 353         n += dtrace_getustack_common(NULL, 0, pc, fp);
 354
 355         return (n);
 356 }
 357
 358 void
 359 dtrace_getufpstack(uint64_t * pcstack, uint64_t * fpstack, int pcstack_limit)
 360 {
 361         thread_t        thread = current_thread();
 362         boolean_t       is64bit = proc_is64bit(current_proc());
 363         savearea_t      *regs;
 364         user_addr_t     pc, sp;
 365         volatile        uint16_t  *flags = (volatile uint16_t *) & cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 366
 367 #if 0
 368         uintptr_t oldcontext;
 369         size_t          s1, s2;
 370 #endif
 371
 372         if (*flags & CPU_DTRACE_FAULT)
 373                 return;
 374
 375         if (pcstack_limit <= 0)
 376                 return;
 377
 378         /*
 379          * If there's no user context we still need to zero the stack.
 380          */
 381         if (thread == NULL)
 382                 goto zero;
 383
 384         regs = (savearea_t *) find_user_regs(thread);
 385         if (regs == NULL)
 386                 goto zero;
 387
 388         *pcstack++ = (uint64_t)dtrace_proc_selfpid();
 389         pcstack_limit--;
 390
 391         if (pcstack_limit <= 0)
 392                 return;
 393
 394         pc = get_saved_state_pc(regs);
 395         sp = get_saved_state_lr(regs);
 396
 397 #if 0                           /* XXX signal stack crawl */
 398         oldcontext = lwp->lwp_oldcontext;
 399
 400         if (p->p_model == DATAMODEL_NATIVE) {
 401                 s1 = sizeof(struct frame) + 2 * sizeof(long);
 402                 s2 = s1 + sizeof(siginfo_t);
 403         } else {
 404                 s1 = sizeof(struct frame32) + 3 * sizeof(int);
 405                 s2 = s1 + sizeof(siginfo32_t);
 406         }
 407 #endif
 408
 409         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
 410                 *pcstack++ = (uint64_t) pc;
 411                 *fpstack++ = 0;
 412                 pcstack_limit--;
 413                 if (pcstack_limit <= 0)
 414                         return;
 415
 416                 if (is64bit)
 417                         pc = dtrace_fuword64(sp);
 418                 else
 419                         pc = dtrace_fuword32(sp);
 420         }
 421         while (pc != 0 && sp != 0) {
 422                 *pcstack++ = (uint64_t) pc;
 423                 *fpstack++ = sp;
 424                 pcstack_limit--;
 425                 if (pcstack_limit <= 0)
 426                         break;
 427
 428 #if 0                           /* XXX signal stack crawl */
 429                 if (oldcontext == sp + s1 || oldcontext == sp + s2) {
 430                         if (p->p_model == DATAMODEL_NATIVE) {
 431                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 432                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 433
 434                                 sp = dtrace_fulword(&gregs[REG_FP]);
 435                                 pc = dtrace_fulword(&gregs[REG_PC]);
 436
 437                                 oldcontext = dtrace_fulword(&ucp->uc_link);
 438                         } else {
 439                                 ucontext_t     *ucp = (ucontext_t *) oldcontext;
 440                                 greg_t         *gregs = ucp->uc_mcontext.gregs;
 441
 442                                 sp = dtrace_fuword32(&gregs[EBP]);
 443                                 pc = dtrace_fuword32(&gregs[EIP]);
 444
 445                                 oldcontext = dtrace_fuword32(&ucp->uc_link);
 446                         }
 447                 } else
 448 #endif
 449                 {
 450                         if (is64bit) {
 451                                 pc = dtrace_fuword64((sp + RETURN_OFFSET64));
 452                                 sp = dtrace_fuword64(sp);
 453                         } else {
 454                                 pc = dtrace_fuword32((sp + RETURN_OFFSET));
 455                                 sp = dtrace_fuword32(sp);
 456                         }
 457                 }
 458
 459 #if 0
 460                 /* XXX ARMTODO*/
 461                 /*
 462                  * This is totally bogus:  if we faulted, we're going to clear
 463                  * the fault and break.  This is to deal with the apparently
 464                  * broken Java stacks on x86.
 465                  */
 466                 if (*flags & CPU_DTRACE_FAULT) {
 467                         *flags &= ~CPU_DTRACE_FAULT;
 468                         break;
 469                 }
 470 #endif
 471         }
 472
 473 zero:
 474         while (pcstack_limit-- > 0)
 475                 *pcstack++ = 0ULL;
 476 }
 477
 478
 479 void
 480 dtrace_getpcstack(pc_t * pcstack, int pcstack_limit, int aframes,
 481                   uint32_t * intrpc)
 482 {
 483         struct frame   *fp = (struct frame *) __builtin_frame_address(0);
 484         struct frame   *nextfp, *minfp, *stacktop;
 485         int             depth = 0;
 486         int             on_intr;
 487         int             last = 0;
 488         uintptr_t       pc;
 489         uintptr_t       caller = CPU->cpu_dtrace_caller;
 490
 491         if ((on_intr = CPU_ON_INTR(CPU)) != 0)
 492                 stacktop = (struct frame *) dtrace_get_cpu_int_stack_top();
 493         else
 494                 stacktop = (struct frame *) (dtrace_get_kernel_stack(current_thread()) + kernel_stack_size);
 495
 496         minfp = fp;
 497
 498         aframes++;
 499
 500         if (intrpc != NULL && depth < pcstack_limit)
 501                 pcstack[depth++] = (pc_t) intrpc;
 502
 503         while (depth < pcstack_limit) {
 504                 nextfp = *(struct frame **) fp;
 505                 pc = *(uintptr_t *) (((uintptr_t) fp) + RETURN_OFFSET64);
 506
 507                 if (nextfp <= minfp || nextfp >= stacktop) {
 508                         if (on_intr) {
 509                                 /*
 510                                  * Hop from interrupt stack to thread stack.
 511                                  */
 512                                 arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread());
 513                                 if (arm_kern_regs) {
 514                                         nextfp = (struct frame *)(saved_state64(arm_kern_regs)->fp);
 515
 516                                         {
 517                                                 vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
 518
 519                                                 minfp = (struct frame *)kstack_base;
 520                                                 stacktop = (struct frame *)(kstack_base + kernel_stack_size);
 521                                         }
 522
 523                                         on_intr = 0;
 524
 525                                         if (nextfp <= minfp || nextfp >= stacktop) {
 526                                                 last = 1;
 527                                         }
 528                                 } else {
 529                                         /*
 530                                          * If this thread was on the interrupt stack, but did not
 531                                          * take an interrupt (i.e, the idle thread), there is no
 532                                          * explicit saved state for us to use.
 533                                          */
 534                                         last = 1;
 535                                 }
 536                         } else {
 537                                 {
 538                                         /*
 539                                          * This is the last frame we can process; indicate
 540                                          * that we should return after processing this frame.
 541                                          */
 542                                         last = 1;
 543                                 }
 544                         }
 545                 }
 546                 if (aframes > 0) {
 547                         if (--aframes == 0 && caller != (uintptr_t)NULL) {
 548                                 /*
 549                                  * We've just run out of artificial frames,
 550                                  * and we have a valid caller -- fill it in
 551                                  * now.
 552                                  */
 553                                 ASSERT(depth < pcstack_limit);
 554                                 pcstack[depth++] = (pc_t) caller;
 555                                 caller = (uintptr_t)NULL;
 556                         }
 557                 } else {
 558                         if (depth < pcstack_limit)
 559                                 pcstack[depth++] = (pc_t) pc;
 560                 }
 561
 562                 if (last) {
 563                         while (depth < pcstack_limit)
 564                                 pcstack[depth++] = (pc_t) NULL;
 565                         return;
 566                 }
 567                 fp = nextfp;
 568                 minfp = fp;
 569         }
 570 }
 571
 572 /*
 573  * On arm64, we support both 32bit and 64bit user processes.
 574  * This routine is only called when handling 32bit processes
 575  * where thumb_mode is pertinent.
 576  * If this routine is called when handling 64bit processes
 577  * thumb_mode should always be zero.
 578  */
 579 int
 580 dtrace_instr_size(uint32_t instr, int thumb_mode)
 581 {
 582         if (thumb_mode) {
 583                 uint16_t instr16 = *(uint16_t*) &instr;
 584                 if (((instr16 >> 11) & 0x1F) > 0x1C)
 585                         return 4;
 586                 else
 587                         return 2;
 588         } else {
 589                 return 4;
 590         }
 591 }
 592
 593 uint64_t
 594 dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
 595 {
 596 #pragma unused(arg, aframes)
 597         uint64_t val = 0;
 598         struct frame *fp = (struct frame *)__builtin_frame_address(0);
 599         uintptr_t *stack;
 600         uintptr_t pc;
 601         int i;
 602
 603         /*
 604          * A total of 8 arguments are passed via registers; any argument with
 605          * index of 7 or lower is therefore in a register.
 606          */
 607         int inreg = 7;
 608
 609         for (i = 1; i <= aframes; ++i) {
 610                 fp = fp->backchain;
 611                 pc = fp->retaddr;
 612
 613                 if (dtrace_invop_callsite_pre != NULL
 614                     && pc >  (uintptr_t) dtrace_invop_callsite_pre
 615                     && pc <= (uintptr_t) dtrace_invop_callsite_post)
 616                 {
 617                         /* fp points to frame of dtrace_invop() activation */
 618                         fp = fp->backchain; /* to fbt_perfCallback activation */
 619                         fp = fp->backchain; /* to sleh_synchronous activation */
 620                         fp = fp->backchain; /* to fleh_synchronous activation */
 621
 622                         arm_saved_state_t       *tagged_regs = (arm_saved_state_t*) ((void*) &fp[1]);
 623                         arm_saved_state64_t     *saved_state = saved_state64(tagged_regs);
 624
 625                         if (arg <= inreg) {
 626                                 /* the argument will be found in a register */
 627                                 stack = (uintptr_t*) &saved_state->x[0];
 628                         } else {
 629                                 /* the argument will be found in the stack */
 630                                 fp = (struct frame*) saved_state->sp;
 631                                 stack = (uintptr_t*) &fp[1];
 632                                 arg -= (inreg + 1);
 633                         }
 634
 635                         goto load;
 636                 }
 637         }
 638
 639         /*
 640          * We know that we did not come through a trap to get into
 641          * dtrace_probe() --  We arrive here when the provider has
 642          * called dtrace_probe() directly.
 643          * The probe ID is the first argument to dtrace_probe().
 644          * We must advance beyond that to get the argX.
 645          */
 646         arg++; /* Advance past probeID */
 647
 648         if (arg <= inreg) {
 649                 /*
 650                  * This shouldn't happen.  If the argument is passed in a
 651                  * register then it should have been, well, passed in a
 652                  * register...
 653                  */
 654                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
 655                 return (0);
 656         }
 657
 658         arg -= (inreg + 1);
 659         stack = (uintptr_t*) &fp[1]; /* Find marshalled arguments */
 660
 661 load:
 662         if (dtrace_canload((uint64_t)(stack + arg), sizeof(uint64_t),
 663                 mstate, vstate)) {
 664                 /* dtrace_probe arguments arg0 ... arg4 are 64bits wide */
 665                 val = dtrace_load64((uint64_t)(stack + arg));
 666         }
 667
 668         return (val);
 669 }
 670
 671 void
 672 dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
 673                 int fltoffs, int fault, uint64_t illval)
 674 {
 675         /* XXX ARMTODO */
 676         /*
 677          * For the case of the error probe firing lets
 678          * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG.
 679          */
 680         state->dts_arg_error_illval = illval;
 681         dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault );
 682 }
 683
 684 void
 685 dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
 686 {
 687         /* XXX ARMTODO check copied from ppc/x86*/
 688         /*
 689          * "base" is the smallest toxic address in the range, "limit" is the first
 690          * VALID address greater than "base".
 691          */
 692         func(0x0, VM_MIN_KERNEL_ADDRESS);
 693         if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0)
 694                         func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0);
 695 }
 696