osfmk/i386/pmap.h

   1 /*
   2  * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58
  59 /*
  60  *      File:   pmap.h
  61  *
  62  *      Authors:  Avadis Tevanian, Jr., Michael Wayne Young
  63  *      Date:   1985
  64  *
  65  *      Machine-dependent structures for the physical map module.
  66  */
  67 #ifdef KERNEL_PRIVATE
  68 #ifndef _PMAP_MACHINE_
  69 #define _PMAP_MACHINE_  1
  70
  71 #ifndef ASSEMBLER
  72
  73 #include <platforms.h>
  74
  75 #include <mach/kern_return.h>
  76 #include <mach/machine/vm_types.h>
  77 #include <mach/vm_prot.h>
  78 #include <mach/vm_statistics.h>
  79 #include <mach/machine/vm_param.h>
  80 #include <kern/kern_types.h>
  81 #include <kern/thread.h>
  82 #include <kern/lock.h>
  83 #include <mach/branch_predicates.h>
  84
  85 #include <i386/mp.h>
  86 #include <i386/proc_reg.h>
  87
  88 #include <i386/pal_routines.h>
  89
  90 /*
  91  *      Define the generic in terms of the specific
  92  */
  93
  94 #define INTEL_PGBYTES           I386_PGBYTES
  95 #define INTEL_PGSHIFT           I386_PGSHIFT
  96 #define intel_btop(x)           i386_btop(x)
  97 #define intel_ptob(x)           i386_ptob(x)
  98 #define intel_round_page(x)     i386_round_page(x)
  99 #define intel_trunc_page(x)     i386_trunc_page(x)
 100 #define trunc_intel_to_vm(x)    trunc_i386_to_vm(x)
 101 #define round_intel_to_vm(x)    round_i386_to_vm(x)
 102 #define vm_to_intel(x)          vm_to_i386(x)
 103
 104 /*
 105  *      i386/i486/i860 Page Table Entry
 106  */
 107
 108 #endif  /* ASSEMBLER */
 109
 110 #define NPGPTD          4ULL
 111 #define PDESHIFT        21ULL
 112 #define PTEMASK         0x1ffULL
 113 #define PTEINDX         3ULL
 114
 115 #define PTESHIFT        12ULL
 116
 117
 118 #ifdef __x86_64__
 119 #define LOW_4GB_MASK    ((vm_offset_t)0x00000000FFFFFFFFUL)
 120 #endif
 121
 122 #define PDESIZE         sizeof(pd_entry_t) /* for assembly files */
 123 #define PTESIZE         sizeof(pt_entry_t) /* for assembly files */
 124
 125 #define INTEL_OFFMASK   (I386_PGBYTES - 1)
 126 #define INTEL_LOFFMASK  (I386_LPGBYTES - 1)
 127 #define PG_FRAME        0x000FFFFFFFFFF000ULL
 128 #define NPTEPG          (PAGE_SIZE/(sizeof (pt_entry_t)))
 129 #define NPTDPG          (PAGE_SIZE/(sizeof (pd_entry_t)))
 130
 131 #define NBPTD           (NPGPTD << PAGE_SHIFT)
 132 #define NPDEPTD         (NBPTD / (sizeof (pd_entry_t)))
 133 #define NPDEPG          (PAGE_SIZE/(sizeof (pd_entry_t)))
 134 #define NBPDE           (1ULL << PDESHIFT)
 135 #define PDEMASK         (NBPDE - 1)
 136
 137 #define PTE_PER_PAGE    512 /* number of PTE's per page on any level */
 138
 139  /* cleanly define parameters for all the page table levels */
 140 typedef uint64_t        pml4_entry_t;
 141 #define NPML4PG         (PAGE_SIZE/(sizeof (pml4_entry_t)))
 142 #define PML4SHIFT       39
 143 #define PML4PGSHIFT     9
 144 #define NBPML4          (1ULL << PML4SHIFT)
 145 #define PML4MASK        (NBPML4-1)
 146 #define PML4_ENTRY_NULL ((pml4_entry_t *) 0)
 147
 148 typedef uint64_t        pdpt_entry_t;
 149 #define NPDPTPG         (PAGE_SIZE/(sizeof (pdpt_entry_t)))
 150 #define PDPTSHIFT       30
 151 #define PDPTPGSHIFT     9
 152 #define NBPDPT          (1ULL << PDPTSHIFT)
 153 #define PDPTMASK        (NBPDPT-1)
 154 #define PDPT_ENTRY_NULL ((pdpt_entry_t *) 0)
 155
 156 typedef uint64_t        pd_entry_t;
 157 #define NPDPG           (PAGE_SIZE/(sizeof (pd_entry_t)))
 158 #define PDSHIFT         21
 159 #define PDPGSHIFT       9
 160 #define NBPD            (1ULL << PDSHIFT)
 161 #define PDMASK          (NBPD-1)
 162 #define PD_ENTRY_NULL   ((pd_entry_t *) 0)
 163
 164 typedef uint64_t        pt_entry_t;
 165 #define NPTPG           (PAGE_SIZE/(sizeof (pt_entry_t)))
 166 #define PTSHIFT         12
 167 #define PTPGSHIFT       9
 168 #define NBPT            (1ULL << PTSHIFT)
 169 #define PTMASK          (NBPT-1)
 170 #define PT_ENTRY_NULL   ((pt_entry_t *) 0)
 171
 172 typedef uint64_t  pmap_paddr_t;
 173
 174 #if     DEBUG
 175 #define PMAP_ASSERT 1
 176 #endif
 177 #if PMAP_ASSERT
 178 #define pmap_assert(ex) ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex))
 179
 180 #define pmap_assert2(ex, fmt, args...)                                  \
 181         do {                                                            \
 182                 if (!(ex)) {                                            \
 183                         kprintf("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0),  ##args); \
 184                         panic("Assertion %s failed (%s:%d, caller %p) " fmt , #ex, __FILE__, __LINE__, __builtin_return_address(0),  ##args);           \
 185                 }                                                       \
 186         } while(0)
 187 #else
 188 #define pmap_assert(ex)
 189 #define pmap_assert2(ex, fmt, args...)
 190 #endif
 191
 192 /* superpages */
 193 #ifdef __x86_64__
 194 #define SUPERPAGE_NBASEPAGES 512
 195 #else
 196 #define SUPERPAGE_NBASEPAGES 1  /* we don't support superpages on i386 */
 197 #endif
 198
 199 /*
 200  * Atomic 64-bit store of a page table entry.
 201  */
 202 static inline void
 203 pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
 204 {
 205         /*
 206          * In the 32-bit kernel a compare-and-exchange loop was
 207          * required to provide atomicity. For K64, life is easier:
 208          */
 209         *entryp = value;
 210 }
 211
 212 /* in 64 bit spaces, the number of each type of page in the page tables */
 213 #define NPML4PGS        (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
 214 #define NPDPTPGS        (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
 215 #define NPDEPGS         (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t))))
 216 #define NPTEPGS         (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
 217
 218 #define KERNEL_PML4_INDEX               511
 219 #define KERNEL_KEXTS_INDEX      510     /* Home of KEXTs - the basement */
 220 #define KERNEL_PHYSMAP_PML4_INDEX       509     /* virtual to physical map */
 221 #define KERNEL_BASE             (0ULL - NBPML4)
 222 #define KERNEL_BASEMENT         (KERNEL_BASE - NBPML4)
 223
 224 #define VM_WIMG_COPYBACK        VM_MEM_COHERENT
 225 #define VM_WIMG_COPYBACKLW      VM_WIMG_COPYBACK
 226 #define VM_WIMG_DEFAULT         VM_MEM_COHERENT
 227 /* ?? intel ?? */
 228 #define VM_WIMG_IO              (VM_MEM_COHERENT |      \
 229                                 VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED)
 230 #define VM_WIMG_WTHRU           (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED)
 231 /* write combining mode, aka store gather */
 232 #define VM_WIMG_WCOMB           (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT)
 233 #define VM_WIMG_INNERWBACK      VM_MEM_COHERENT
 234 /*
 235  * Pte related macros
 236  */
 237 #define KVADDR(pmi, pdpi, pdi, pti)               \
 238          ((vm_offset_t)                   \
 239                 ((uint64_t) -1    << 47)        | \
 240                 ((uint64_t)(pmi)  << PML4SHIFT) | \
 241                 ((uint64_t)(pdpi) << PDPTSHIFT) | \
 242                 ((uint64_t)(pdi)  << PDESHIFT)  | \
 243                 ((uint64_t)(pti)  << PTESHIFT))
 244
 245 /*
 246  * Size of Kernel address space.  This is the number of page table pages
 247  * (4MB each) to use for the kernel.  256 pages == 1 Gigabyte.
 248  * This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc).
 249  */
 250 #ifndef KVA_PAGES
 251 #define KVA_PAGES       1024
 252 #endif
 253
 254 #ifndef NKPT
 255 #define NKPT            500     /* actual number of kernel page tables */
 256 #endif
 257 #ifndef NKPDE
 258 #define NKPDE   (KVA_PAGES - 1) /* addressable number of page tables/pde's */
 259 #endif
 260
 261
 262
 263 /*
 264  *      Convert address offset to page descriptor index
 265  */
 266 #define pdptnum(pmap, a) (((vm_offset_t)(a) >> PDPTSHIFT) & PDPTMASK)
 267 #define pdenum(pmap, a) (((vm_offset_t)(a) >> PDESHIFT) & PDEMASK)
 268 #define PMAP_INVALID_PDPTNUM (~0ULL)
 269
 270 #define pdeidx(pmap, a)    (((a) >> PDSHIFT)   & ((1ULL<<(48 - PDSHIFT)) -1))
 271 #define pdptidx(pmap, a)   (((a) >> PDPTSHIFT) & ((1ULL<<(48 - PDPTSHIFT)) -1))
 272 #define pml4idx(pmap, a)   (((a) >> PML4SHIFT) & ((1ULL<<(48 - PML4SHIFT)) -1))
 273
 274
 275 /*
 276  *      Convert page descriptor index to user virtual address
 277  */
 278 #define pdetova(a)      ((vm_offset_t)(a) << PDESHIFT)
 279
 280 /*
 281  *      Convert address offset to page table index
 282  */
 283 #define ptenum(a)       (((vm_offset_t)(a) >> PTESHIFT) & PTEMASK)
 284
 285 /*
 286  *      Hardware pte bit definitions (to be used directly on the ptes
 287  *      without using the bit fields).
 288  */
 289
 290 #define INTEL_PTE_VALID         0x00000001ULL
 291 #define INTEL_PTE_WRITE         0x00000002ULL
 292 #define INTEL_PTE_RW            0x00000002ULL
 293 #define INTEL_PTE_USER          0x00000004ULL
 294 #define INTEL_PTE_WTHRU         0x00000008ULL
 295 #define INTEL_PTE_NCACHE        0x00000010ULL
 296 #define INTEL_PTE_REF           0x00000020ULL
 297 #define INTEL_PTE_MOD           0x00000040ULL
 298 #define INTEL_PTE_PS            0x00000080ULL
 299 #define INTEL_PTE_PTA           0x00000080ULL
 300 #define INTEL_PTE_GLOBAL        0x00000100ULL
 301 #define INTEL_PTE_WIRED         0x00000200ULL
 302 #define INTEL_PDPTE_NESTED      0x00000400ULL
 303 #define INTEL_PTE_PFN           PG_FRAME
 304
 305 #define INTEL_PTE_NX            (1ULL << 63)
 306
 307 #define INTEL_PTE_INVALID       0
 308 /* This is conservative, but suffices */
 309 #define INTEL_PTE_RSVD          ((1ULL << 10) | (1ULL << 11) | (0x1FFULL << 54))
 310
 311 #define INTEL_PTE_COMPRESSED    INTEL_PTE_REF /* marker, for invalid PTE only */
 312
 313 #define pa_to_pte(a)            ((a) & INTEL_PTE_PFN) /* XXX */
 314 #define pte_to_pa(p)            ((p) & INTEL_PTE_PFN) /* XXX */
 315 #define pte_increment_pa(p)     ((p) += INTEL_OFFMASK+1)
 316
 317 #define pte_kernel_rw(p)          ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_RW))
 318 #define pte_kernel_ro(p)          ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID))
 319 #define pte_user_rw(p)            ((pt_entry)t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER|INTEL_PTE_RW))
 320 #define pte_user_ro(p)            ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER))
 321
 322 #define PMAP_DEFAULT_CACHE      0
 323 #define PMAP_INHIBIT_CACHE      1
 324 #define PMAP_GUARDED_CACHE      2
 325 #define PMAP_ACTIVATE_CACHE     4
 326 #define PMAP_NO_GUARD_CACHE     8
 327
 328
 329 #ifndef ASSEMBLER
 330
 331 #include <sys/queue.h>
 332
 333 /*
 334  * Address of current and alternate address space page table maps
 335  * and directories.
 336  */
 337
 338 extern pt_entry_t       *PTmap;
 339 extern pdpt_entry_t     *IdlePDPT;
 340 extern pml4_entry_t     *IdlePML4;
 341 extern boolean_t        no_shared_cr3;
 342 extern addr64_t         kernel64_cr3;
 343 extern pd_entry_t       *IdlePTD;       /* physical addr of "Idle" state PTD */
 344
 345 extern uint64_t         pmap_pv_hashlist_walks;
 346 extern uint64_t         pmap_pv_hashlist_cnts;
 347 extern uint32_t         pmap_pv_hashlist_max;
 348 extern uint32_t         pmap_kernel_text_ps;
 349
 350
 351
 352 #ifdef __x86_64__
 353 #define ID_MAP_VTOP(x)  ((void *)(((uint64_t)(x)) & LOW_4GB_MASK))
 354
 355 extern  uint64_t physmap_base, physmap_max;
 356
 357 #define NPHYSMAP (MAX(K64_MAXMEM/GB + 4, 4))
 358
 359 static inline boolean_t physmap_enclosed(addr64_t a) {
 360         return (a < (NPHYSMAP * GB));
 361 }
 362
 363 static  inline void * PHYSMAP_PTOV_check(void *paddr) {
 364         uint64_t pvaddr = (uint64_t)paddr + physmap_base;
 365
 366         if (__improbable(pvaddr >= physmap_max))
 367                 panic("PHYSMAP_PTOV bounds exceeded, 0x%qx, 0x%qx, 0x%qx",
 368                       pvaddr, physmap_base, physmap_max);
 369
 370         return (void *)pvaddr;
 371 }
 372
 373 #define PHYSMAP_PTOV(x) (PHYSMAP_PTOV_check((void*) (x)))
 374
 375 /*
 376  * For KASLR, we alias the master processor's IDT and GDT at fixed
 377  * virtual addresses to defeat SIDT/SGDT address leakage.
 378  * And non-boot processor's GDT aliases likewise (skipping LOWGLOBAL_ALIAS)
 379  * The low global vector page is mapped at a fixed alias also.
 380  */
 381 #define MASTER_IDT_ALIAS        (VM_MIN_KERNEL_ADDRESS + 0x0000)
 382 #define MASTER_GDT_ALIAS        (VM_MIN_KERNEL_ADDRESS + 0x1000)
 383 #define LOWGLOBAL_ALIAS         (VM_MIN_KERNEL_ADDRESS + 0x2000)
 384 #define CPU_GDT_ALIAS(_cpu)     (LOWGLOBAL_ALIAS + (0x1000*(_cpu)))
 385
 386 #endif /*__x86_64__ */
 387
 388 typedef volatile long   cpu_set;        /* set of CPUs - must be <= 32 */
 389                                         /* changed by other processors */
 390 #include <vm/vm_page.h>
 391
 392 /*
 393  *      For each vm_page_t, there is a list of all currently
 394  *      valid virtual mappings of that page.  An entry is
 395  *      a pv_entry_t; the list is the pv_table.
 396  */
 397
 398 struct pmap {
 399         decl_simple_lock_data(,lock)    /* lock on map */
 400         pmap_paddr_t    pm_cr3;         /* physical addr */
 401         boolean_t       pm_shared;
 402         pd_entry_t      *dirbase;        /* page directory pointer */
 403         vm_object_t     pm_obj;         /* object to hold pde's */
 404         task_map_t      pm_task_map;
 405         pdpt_entry_t    *pm_pdpt;       /* KVA of 3rd level page */
 406         pml4_entry_t    *pm_pml4;       /* VKA of top level */
 407         vm_object_t     pm_obj_pdpt;    /* holds pdpt pages */
 408         vm_object_t     pm_obj_pml4;    /* holds pml4 pages */
 409 #define PMAP_PCID_MAX_CPUS      (48)    /* Must be a multiple of 8 */
 410         pcid_t          pmap_pcid_cpus[PMAP_PCID_MAX_CPUS];
 411         volatile uint8_t pmap_pcid_coherency_vector[PMAP_PCID_MAX_CPUS];
 412         struct pmap_statistics  stats;  /* map statistics */
 413         int             ref_count;      /* reference count */
 414         int             nx_enabled;
 415         ledger_t        ledger;         /* ledger tracking phys mappings */
 416 };
 417
 418
 419 #if NCOPY_WINDOWS > 0
 420 #define PMAP_PDPT_FIRST_WINDOW 0
 421 #define PMAP_PDPT_NWINDOWS 4
 422 #define PMAP_PDE_FIRST_WINDOW (PMAP_PDPT_NWINDOWS)
 423 #define PMAP_PDE_NWINDOWS 4
 424 #define PMAP_PTE_FIRST_WINDOW (PMAP_PDE_FIRST_WINDOW + PMAP_PDE_NWINDOWS)
 425 #define PMAP_PTE_NWINDOWS 4
 426
 427 #define PMAP_NWINDOWS_FIRSTFREE (PMAP_PTE_FIRST_WINDOW + PMAP_PTE_NWINDOWS)
 428 #define PMAP_WINDOW_SIZE 8
 429 #define PMAP_NWINDOWS (PMAP_NWINDOWS_FIRSTFREE + PMAP_WINDOW_SIZE)
 430
 431 typedef struct {
 432         pt_entry_t      *prv_CMAP;
 433         caddr_t         prv_CADDR;
 434 } mapwindow_t;
 435
 436 typedef struct cpu_pmap {
 437         int                     pdpt_window_index;
 438         int                     pde_window_index;
 439         int                     pte_window_index;
 440         mapwindow_t             mapwindow[PMAP_NWINDOWS];
 441 } cpu_pmap_t;
 442
 443
 444 extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry);
 445 extern void         pmap_put_mapwindow(mapwindow_t *map);
 446 #endif
 447
 448 typedef struct pmap_memory_regions {
 449         ppnum_t base;           /* first page of this region */
 450         ppnum_t alloc_up;       /* pages below this one have been "stolen" */
 451         ppnum_t alloc_down;     /* pages above this one have been "stolen" */
 452         ppnum_t end;            /* last page of this region */
 453         uint32_t type;
 454         uint64_t attribute;
 455 } pmap_memory_region_t;
 456
 457 extern unsigned pmap_memory_region_count;
 458 extern unsigned pmap_memory_region_current;
 459
 460 #define PMAP_MEMORY_REGIONS_SIZE 128
 461
 462 extern pmap_memory_region_t pmap_memory_regions[];
 463 #include <i386/pmap_pcid.h>
 464
 465 static inline void
 466 set_dirbase(pmap_t tpmap, __unused thread_t thread) {
 467         int ccpu = cpu_number();
 468         cpu_datap(ccpu)->cpu_task_cr3 = tpmap->pm_cr3;
 469         cpu_datap(ccpu)->cpu_task_map = tpmap->pm_task_map;
 470         /*
 471          * Switch cr3 if necessary
 472          * - unless running with no_shared_cr3 debugging mode
 473          *   and we're not on the kernel's cr3 (after pre-empted copyio)
 474          */
 475         if (__probable(!no_shared_cr3)) {
 476                 if (get_cr3_base() != tpmap->pm_cr3) {
 477                         if (pmap_pcid_ncpus) {
 478                                 pmap_pcid_activate(tpmap, ccpu);
 479                         }
 480                         else
 481                                 set_cr3_raw(tpmap->pm_cr3);
 482                 }
 483         } else {
 484                 if (get_cr3_base() != cpu_datap(ccpu)->cpu_kernel_cr3)
 485                         set_cr3_raw(cpu_datap(ccpu)->cpu_kernel_cr3);
 486         }
 487 }
 488
 489 /*
 490  *      External declarations for PMAP_ACTIVATE.
 491  */
 492
 493 extern void             process_pmap_updates(void);
 494 extern void             pmap_update_interrupt(void);
 495
 496 /*
 497  *      Machine dependent routines that are used only for i386/i486/i860.
 498  */
 499
 500 extern addr64_t         (kvtophys)(
 501                                 vm_offset_t     addr);
 502
 503 extern kern_return_t    pmap_expand(
 504                                 pmap_t          pmap,
 505                                 vm_map_offset_t addr,
 506                                 unsigned int options);
 507 #if     !defined(__x86_64__)
 508 extern pt_entry_t       *pmap_pte(
 509                                 struct pmap     *pmap,
 510                                 vm_map_offset_t addr);
 511
 512 extern pd_entry_t       *pmap_pde(
 513                                 struct pmap     *pmap,
 514                                 vm_map_offset_t addr);
 515
 516 extern pd_entry_t       *pmap64_pde(
 517                                 struct pmap     *pmap,
 518                                 vm_map_offset_t addr);
 519
 520 extern pdpt_entry_t     *pmap64_pdpt(
 521                                 struct pmap     *pmap,
 522                                 vm_map_offset_t addr);
 523 #endif
 524 extern vm_offset_t      pmap_map(
 525                                 vm_offset_t     virt,
 526                                 vm_map_offset_t start,
 527                                 vm_map_offset_t end,
 528                                 vm_prot_t       prot,
 529                                 unsigned int    flags);
 530
 531 extern vm_offset_t      pmap_map_bd(
 532                                 vm_offset_t     virt,
 533                                 vm_map_offset_t start,
 534                                 vm_map_offset_t end,
 535                                 vm_prot_t       prot,
 536                                 unsigned int    flags);
 537
 538 extern void             pmap_bootstrap(
 539                                 vm_offset_t     load_start,
 540                                 boolean_t       IA32e);
 541
 542 extern boolean_t        pmap_valid_page(
 543                                 ppnum_t pn);
 544
 545 extern int              pmap_list_resident_pages(
 546                                 struct pmap     *pmap,
 547                                 vm_offset_t     *listp,
 548                                 int             space);
 549 extern void             x86_filter_TLB_coherency_interrupts(boolean_t);
 550 /*
 551  * Get cache attributes (as pagetable bits) for the specified phys page
 552  */
 553 extern  unsigned        pmap_get_cache_attributes(ppnum_t);
 554 #if NCOPY_WINDOWS > 0
 555 extern struct cpu_pmap  *pmap_cpu_alloc(
 556                                 boolean_t       is_boot_cpu);
 557 extern void             pmap_cpu_free(
 558                                 struct cpu_pmap *cp);
 559 #endif
 560
 561 extern void             pmap_map_block(
 562                                 pmap_t pmap,
 563                                 addr64_t va,
 564                                 ppnum_t pa,
 565                                 uint32_t size,
 566                                 vm_prot_t prot,
 567                                 int attr,
 568                                 unsigned int flags);
 569
 570 extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
 571 extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
 572 extern ppnum_t          pmap_find_phys(pmap_t map, addr64_t va);
 573
 574 extern void pmap_cpu_init(void);
 575 extern void pmap_disable_NX(pmap_t pmap);
 576
 577 extern void pt_fake_zone_init(int);
 578 extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *,
 579                               uint64_t *, int *, int *, int *);
 580 extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__printflike(1,2));
 581
 582 /*
 583  *      Macros for speed.
 584  */
 585
 586
 587 #include <kern/spl.h>
 588
 589
 590 #define PMAP_ACTIVATE_MAP(map, thread)  {                               \
 591         register pmap_t         tpmap;                                  \
 592                                                                         \
 593         tpmap = vm_map_pmap(map);                                       \
 594         set_dirbase(tpmap, thread);                                     \
 595 }
 596
 597 #if   defined(__x86_64__)
 598 #define PMAP_DEACTIVATE_MAP(map, thread)                                \
 599         pmap_assert(pmap_pcid_ncpus ? (pcid_for_pmap_cpu_tuple(map->pmap, cpu_number()) == (get_cr3_raw() & 0xFFF)) : TRUE);
 600 #else
 601 #define PMAP_DEACTIVATE_MAP(map, thread)
 602 #endif
 603
 604 #define PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) {                   \
 605                                                                         \
 606         pmap_assert(ml_get_interrupts_enabled() == FALSE);              \
 607         if (old_th->map != new_th->map) {                               \
 608                 PMAP_DEACTIVATE_MAP(old_th->map, old_th);               \
 609                 PMAP_ACTIVATE_MAP(new_th->map, new_th);                 \
 610         }                                                               \
 611 }
 612
 613 #if NCOPY_WINDOWS > 0
 614 #define PMAP_SWITCH_USER(th, new_map, my_cpu) {                         \
 615         spl_t           spl;                                            \
 616                                                                         \
 617         spl = splhigh();                                                \
 618         PMAP_DEACTIVATE_MAP(th->map, th);                               \
 619         th->map = new_map;                                              \
 620         PMAP_ACTIVATE_MAP(th->map, th);                                 \
 621         splx(spl);                                                      \
 622         inval_copy_windows(th);                                         \
 623 }
 624 #else
 625 #define PMAP_SWITCH_USER(th, new_map, my_cpu) {                         \
 626         spl_t           spl;                                            \
 627                                                                         \
 628         spl = splhigh();                                                \
 629         PMAP_DEACTIVATE_MAP(th->map, th);                               \
 630         th->map = new_map;                                              \
 631         PMAP_ACTIVATE_MAP(th->map, th);                                 \
 632         splx(spl);                                                      \
 633 }
 634 #endif
 635
 636 /*
 637  * Marking the current cpu's cr3 inactive is achieved by setting its lsb.
 638  * Marking the current cpu's cr3 active once more involves clearng this bit.
 639  * Note that valid page tables are page-aligned and so the bottom 12 bits
 640  * are normally zero, modulo PCID.
 641  * We can only mark the current cpu active/inactive but we can test any cpu.
 642  */
 643 #define CPU_CR3_MARK_INACTIVE()                                         \
 644         current_cpu_datap()->cpu_active_cr3 |= 1
 645
 646 #define CPU_CR3_MARK_ACTIVE()                                           \
 647         current_cpu_datap()->cpu_active_cr3 &= ~1
 648
 649 #define CPU_CR3_IS_ACTIVE(cpu)                                          \
 650         ((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0)
 651
 652 #define CPU_GET_ACTIVE_CR3(cpu)                                         \
 653         (cpu_datap(cpu)->cpu_active_cr3 & ~1)
 654
 655 #define CPU_GET_TASK_CR3(cpu)                                           \
 656         (cpu_datap(cpu)->cpu_task_cr3)
 657
 658 /*
 659  *      Mark this cpu idle, and remove it from the active set,
 660  *      since it is not actively using any pmap.  Signal_cpus
 661  *      will notice that it is idle, and avoid signaling it,
 662  *      but will queue the update request for when the cpu
 663  *      becomes active.
 664  */
 665 #define MARK_CPU_IDLE(my_cpu)   {                                       \
 666         assert(ml_get_interrupts_enabled() == FALSE);                   \
 667         CPU_CR3_MARK_INACTIVE();                                        \
 668         mfence();                                                                       \
 669 }
 670
 671 #define MARK_CPU_ACTIVE(my_cpu) {                                       \
 672         assert(ml_get_interrupts_enabled() == FALSE);                   \
 673         /*                                                              \
 674          *      If a kernel_pmap update was requested while this cpu    \
 675          *      was idle, process it as if we got the interrupt.        \
 676          *      Before doing so, remove this cpu from the idle set.     \
 677          *      Since we do not grab any pmap locks while we flush      \
 678          *      our TLB, another cpu may start an update operation      \
 679          *      before we finish.  Removing this cpu from the idle      \
 680          *      set assures that we will receive another update         \
 681          *      interrupt if this happens.                              \
 682          */                                                             \
 683         CPU_CR3_MARK_ACTIVE();                                          \
 684         mfence();                                                                       \
 685                                                                         \
 686         if (current_cpu_datap()->cpu_tlb_invalid)                       \
 687             process_pmap_updates();                                     \
 688 }
 689
 690 #define PMAP_CONTEXT(pmap, thread)
 691
 692 #define pmap_kernel_va(VA)      \
 693         ((((vm_offset_t) (VA)) >= vm_min_kernel_address) &&     \
 694          (((vm_offset_t) (VA)) <= vm_max_kernel_address))
 695
 696
 697 #define pmap_resident_count(pmap)       ((pmap)->stats.resident_count)
 698 #define pmap_resident_max(pmap)         ((pmap)->stats.resident_max)
 699 #define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr)
 700 #define pmap_attribute(pmap,addr,size,attr,value) \
 701                                         (KERN_INVALID_ADDRESS)
 702 #define pmap_attribute_cache_sync(addr,size,attr,value) \
 703                                         (KERN_INVALID_ADDRESS)
 704
 705 #define MACHINE_PMAP_IS_EMPTY   1
 706 extern boolean_t pmap_is_empty(pmap_t           pmap,
 707                                vm_map_offset_t  start,
 708                                vm_map_offset_t  end);
 709
 710 #define MACHINE_BOOTSTRAPPTD    1       /* Static bootstrap page-tables */
 711
 712 kern_return_t
 713 pmap_permissions_verify(pmap_t, vm_map_t, vm_offset_t, vm_offset_t);
 714
 715 #endif  /* ASSEMBLER */
 716
 717
 718 #endif  /* _PMAP_MACHINE_ */
 719
 720
 721 #endif  /* KERNEL_PRIVATE */