osfmk/i386/pmap.h

   1 /*
   2  * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58
  59 /*
  60  *      File:   pmap.h
  61  *
  62  *      Authors:  Avadis Tevanian, Jr., Michael Wayne Young
  63  *      Date:   1985
  64  *
  65  *      Machine-dependent structures for the physical map module.
  66  */
  67 #ifdef KERNEL_PRIVATE
  68 #ifndef _PMAP_MACHINE_
  69 #define _PMAP_MACHINE_  1
  70
  71 #ifndef ASSEMBLER
  72
  73 #include <platforms.h>
  74
  75 #include <mach/kern_return.h>
  76 #include <mach/machine/vm_types.h>
  77 #include <mach/vm_prot.h>
  78 #include <mach/vm_statistics.h>
  79 #include <mach/machine/vm_param.h>
  80 #include <kern/kern_types.h>
  81 #include <kern/thread.h>
  82 #include <kern/lock.h>
  83
  84 #include <i386/mp.h>
  85 #include <i386/proc_reg.h>
  86
  87 /*
  88  *      Define the generic in terms of the specific
  89  */
  90
  91 #define INTEL_PGBYTES           I386_PGBYTES
  92 #define INTEL_PGSHIFT           I386_PGSHIFT
  93 #define intel_btop(x)           i386_btop(x)
  94 #define intel_ptob(x)           i386_ptob(x)
  95 #define intel_round_page(x)     i386_round_page(x)
  96 #define intel_trunc_page(x)     i386_trunc_page(x)
  97 #define trunc_intel_to_vm(x)    trunc_i386_to_vm(x)
  98 #define round_intel_to_vm(x)    round_i386_to_vm(x)
  99 #define vm_to_intel(x)          vm_to_i386(x)
 100
 101 /*
 102  *      i386/i486/i860 Page Table Entry
 103  */
 104
 105 #endif  /* ASSEMBLER */
 106
 107 #define NPGPTD          4
 108 #define PDESHIFT        21
 109 #define PTEMASK         0x1ff
 110 #define PTEINDX         3
 111
 112 #define PTESHIFT        12
 113
 114 #define PDESIZE         sizeof(pd_entry_t) /* for assembly files */
 115 #define PTESIZE         sizeof(pt_entry_t) /* for assembly files */
 116
 117 #define INTEL_OFFMASK   (I386_PGBYTES - 1)
 118 #define PG_FRAME        0x000FFFFFFFFFF000ULL
 119 #define NPTEPG          (PAGE_SIZE/(sizeof (pt_entry_t)))
 120 #define NPTDPG          (PAGE_SIZE/(sizeof (pd_entry_t)))
 121
 122 #define NBPTD           (NPGPTD << PAGE_SHIFT)
 123 #define NPDEPTD         (NBPTD / (sizeof (pd_entry_t)))
 124 #define NPDEPG          (PAGE_SIZE/(sizeof (pd_entry_t)))
 125 #define NBPDE           (1 << PDESHIFT)
 126 #define PDEMASK         (NBPDE - 1)
 127
 128  /* cleanly define parameters for all the page table levels */
 129 typedef uint64_t        pml4_entry_t;
 130 #define NPML4PG         (PAGE_SIZE/(sizeof (pml4_entry_t)))
 131 #define PML4SHIFT       39
 132 #define PML4PGSHIFT     9
 133 #define NBPML4          (1ULL << PML4SHIFT)
 134 #define PML4MASK        (NBPML4-1)
 135 #define PML4_ENTRY_NULL ((pml4_entry_t *) 0)
 136
 137 typedef uint64_t        pdpt_entry_t;
 138 #define NPDPTPG         (PAGE_SIZE/(sizeof (pdpt_entry_t)))
 139 #define PDPTSHIFT       30
 140 #define PDPTPGSHIFT     9
 141 #define NBPDPT          (1 << PDPTSHIFT)
 142 #define PDPTMASK        (NBPDPT-1)
 143 #define PDPT_ENTRY_NULL ((pdpt_entry_t *) 0)
 144
 145 typedef uint64_t        pd_entry_t;
 146 #define NPDPG           (PAGE_SIZE/(sizeof (pd_entry_t)))
 147 #define PDSHIFT         21
 148 #define PDPGSHIFT       9
 149 #define NBPD            (1 << PDSHIFT)
 150 #define PDMASK          (NBPD-1)
 151 #define PD_ENTRY_NULL   ((pd_entry_t *) 0)
 152
 153 typedef uint64_t        pt_entry_t;
 154 #define NPTPG           (PAGE_SIZE/(sizeof (pt_entry_t)))
 155 #define PTSHIFT         12
 156 #define PTPGSHIFT       9
 157 #define NBPT            (1 << PTSHIFT)
 158 #define PTMASK          (NBPT-1)
 159 #define PT_ENTRY_NULL   ((pt_entry_t *) 0)
 160
 161 typedef uint64_t  pmap_paddr_t;
 162
 163 /*
 164  * Atomic 64-bit store of a page table entry.
 165  */
 166 static inline void
 167 pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
 168 {
 169         /*
 170          * Load the new value into %ecx:%ebx
 171          * Load the old value into %edx:%eax
 172          * Compare-exchange-8bytes at address entryp (loaded in %edi)
 173          * If the compare succeeds, the new value will have been stored.
 174          * Otherwise, the old value changed and reloaded, so try again.
 175          */
 176         __asm__ volatile(
 177                 "       movl    (%0), %%eax     \n\t"
 178                 "       movl    4(%0), %%edx    \n\t"
 179                 "1:                             \n\t"
 180                 "       cmpxchg8b (%0)          \n\t"
 181                 "       jnz 1b"
 182                 :
 183                 : "D" (entryp),
 184                   "b" ((uint32_t)value),
 185                   "c" ((uint32_t)(value >> 32))
 186                 : "eax", "edx", "memory");
 187 }
 188
 189 /*
 190  * Atomic 64-bit compare and exchange of a page table entry.
 191  */
 192 static inline boolean_t
 193 pmap_cmpx_pte(pt_entry_t *entryp, pt_entry_t old, pt_entry_t new)
 194 {
 195         boolean_t               ret;
 196
 197         /*
 198          * Load the old value into %edx:%eax
 199          * Load the new value into %ecx:%ebx
 200          * Compare-exchange-8bytes at address entryp (loaded in %edi)
 201          * If the compare succeeds, the new value is stored, return TRUE.
 202          * Otherwise, no swap is made, return FALSE.
 203          */
 204         asm volatile(
 205                 "       lock; cmpxchg8b (%1)    \n\t"
 206                 "       setz    %%al            \n\t"
 207                 "       movzbl  %%al,%0"
 208                 : "=a" (ret)
 209                 : "D" (entryp),
 210                   "a" ((uint32_t)old),
 211                   "d" ((uint32_t)(old >> 32)),
 212                   "b" ((uint32_t)new),
 213                   "c" ((uint32_t)(new >> 32))
 214                 : "memory");
 215         return ret;
 216 }
 217
 218 #define pmap_update_pte(entryp, old, new) \
 219         while (!pmap_cmpx_pte((entryp), (old), (new)))
 220
 221
 222 /* in 64 bit spaces, the number of each type of page in the page tables */
 223 #define NPML4PGS        (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
 224 #define NPDPTPGS        (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
 225 #define NPDEPGS         (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t))))
 226 #define NPTEPGS         (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
 227
 228 /*
 229  * The 64-bit kernel is remapped in uber-space which is at the base
 230  * the highest 4th-level directory (KERNEL_UBER_PML4_INDEX). That is,
 231  * 512GB from the top of virtual space (or zero).
 232  */
 233 #define KERNEL_UBER_PML4_INDEX  511
 234 #define KERNEL_UBER_BASE        (0ULL - NBPML4)
 235 #define KERNEL_UBER_BASE_HI32   ((uint32_t)(KERNEL_UBER_BASE >> 32))
 236
 237 #define VM_WIMG_COPYBACK        VM_MEM_COHERENT
 238 #define VM_WIMG_DEFAULT         VM_MEM_COHERENT
 239 /* ?? intel ?? */
 240 #define VM_WIMG_IO              (VM_MEM_COHERENT |      \
 241                                 VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED)
 242 #define VM_WIMG_WTHRU           (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED)
 243 /* write combining mode, aka store gather */
 244 #define VM_WIMG_WCOMB           (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT)
 245
 246 /*
 247  * Pte related macros
 248  */
 249 #define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDESHIFT)|((pti)<<PTESHIFT)))
 250 #define VADDR64(pmi, pdi, pti) ((vm_offset_t)(((pmi)<<PLM4SHIFT))((pdi)<<PDESHIFT)|((pti)<<PTESHIFT))
 251
 252 /*
 253  * Size of Kernel address space.  This is the number of page table pages
 254  * (4MB each) to use for the kernel.  256 pages == 1 Gigabyte.
 255  * This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc).
 256  */
 257 #ifndef KVA_PAGES
 258 #define KVA_PAGES       1024
 259 #endif
 260
 261 #ifndef NKPT
 262 #define NKPT            500     /* actual number of kernel page tables */
 263 #endif
 264 #ifndef NKPDE
 265 #define NKPDE   (KVA_PAGES - 1) /* addressable number of page tables/pde's */
 266 #endif
 267
 268
 269 enum high_cpu_types {
 270   HIGH_CPU_ISS0,
 271   HIGH_CPU_ISS1,
 272   HIGH_CPU_DESC,
 273   HIGH_CPU_LDT_BEGIN,
 274   HIGH_CPU_LDT_END = HIGH_CPU_LDT_BEGIN + (LDTSZ / 512) - 1,
 275   HIGH_CPU_END
 276 };
 277
 278 enum  high_fixed_addresses {
 279   HIGH_FIXED_TRAMPS,  /* must be first */
 280   HIGH_FIXED_TRAMPS_END,
 281   HIGH_FIXED_GDT,
 282   HIGH_FIXED_IDT,
 283   HIGH_FIXED_LDT_BEGIN,
 284   HIGH_FIXED_LDT_END = HIGH_FIXED_LDT_BEGIN + (LDTSZ / 512) - 1,
 285   HIGH_FIXED_KTSS,
 286   HIGH_FIXED_DFTSS,
 287   HIGH_FIXED_DBTSS,
 288   HIGH_FIXED_CPUS_BEGIN,
 289   HIGH_FIXED_CPUS_END = HIGH_FIXED_CPUS_BEGIN + (HIGH_CPU_END * MAX_CPUS) - 1,
 290 };
 291
 292
 293 /* XXX64  below PTDI values need cleanup */
 294 /*
 295  * The *PTDI values control the layout of virtual memory
 296  *
 297  */
 298 #define        KPTDI           (0x000)/* start of kernel virtual pde's */
 299 #define        PTDPTDI         (0x7F4) /* ptd entry that points to ptd! */
 300 #define        APTDPTDI        (0x7F8) /* alt ptd entry that points to APTD */
 301 #define        UMAXPTDI        (0x7F8) /* ptd entry for user space end */
 302 #define UMAXPTEOFF      (NPTEPG)        /* pte entry for user space end */
 303
 304 #define KERNBASE       VADDR(KPTDI,0)
 305
 306 /*
 307  *      Convert address offset to directory address
 308  *      containing the page table pointer - legacy
 309  */
 310 /*#define pmap_pde(m,v) (&((m)->dirbase[(vm_offset_t)(v) >> PDESHIFT]))*/
 311
 312 #define HIGH_MEM_BASE  ((uint32_t)( -NBPDE) )  /* shared gdt etc seg addr */ /* XXX64 ?? */
 313 #define pmap_index_to_virt(x)  (HIGH_MEM_BASE | ((unsigned)(x) << PAGE_SHIFT))
 314
 315 /*
 316  *      Convert address offset to page descriptor index
 317  */
 318 #define pdenum(pmap, a) (((vm_offset_t)(a) >> PDESHIFT) & PDEMASK)
 319
 320 #define pdeidx(pmap, a)    (((a) >> PDSHIFT)   & ((1ULL<<(48 - PDSHIFT)) -1))
 321 #define pdptidx(pmap, a)   (((a) >> PDPTSHIFT) & ((1ULL<<(48 - PDPTSHIFT)) -1))
 322 #define pml4idx(pmap, a)   (((a) >> PML4SHIFT) & ((1ULL<<(48 - PML4SHIFT)) -1))
 323
 324 /*
 325  *      Convert page descriptor index to user virtual address
 326  */
 327 #define pdetova(a)      ((vm_offset_t)(a) << PDESHIFT)
 328
 329 /*
 330  *      Convert address offset to page table index
 331  */
 332 #define ptenum(a)       (((vm_offset_t)(a) >> PTESHIFT) & PTEMASK)
 333
 334 /*
 335  *      Hardware pte bit definitions (to be used directly on the ptes
 336  *      without using the bit fields).
 337  */
 338
 339 #define INTEL_PTE_VALID         0x00000001
 340 #define INTEL_PTE_WRITE         0x00000002
 341 #define INTEL_PTE_RW            0x00000002
 342 #define INTEL_PTE_USER          0x00000004
 343 #define INTEL_PTE_WTHRU         0x00000008
 344 #define INTEL_PTE_NCACHE        0x00000010
 345 #define INTEL_PTE_REF           0x00000020
 346 #define INTEL_PTE_MOD           0x00000040
 347 #define INTEL_PTE_PS            0x00000080
 348 #define INTEL_PTE_GLOBAL        0x00000100
 349 #define INTEL_PTE_WIRED         0x00000200
 350 #define INTEL_PTE_PFN           PG_FRAME
 351 #define INTEL_PTE_PTA           0x00000080
 352
 353 #define INTEL_PTE_NX            (1ULL << 63)
 354
 355 #define INTEL_PTE_INVALID       0
 356
 357 #define pa_to_pte(a)            ((a) & INTEL_PTE_PFN) /* XXX */
 358 #define pte_to_pa(p)            ((p) & INTEL_PTE_PFN) /* XXX */
 359 #define pte_increment_pa(p)     ((p) += INTEL_OFFMASK+1)
 360
 361 #define pte_kernel_rw(p)          ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_RW))
 362 #define pte_kernel_ro(p)          ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID))
 363 #define pte_user_rw(p)            ((pt_entry)t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER|INTEL_PTE_RW))
 364 #define pte_user_ro(p)            ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER))
 365
 366 #define PMAP_DEFAULT_CACHE      0
 367 #define PMAP_INHIBIT_CACHE      1
 368 #define PMAP_GUARDED_CACHE      2
 369 #define PMAP_ACTIVATE_CACHE     4
 370 #define PMAP_NO_GUARD_CACHE     8
 371
 372
 373 #ifndef ASSEMBLER
 374
 375 #include <sys/queue.h>
 376
 377 /*
 378  * Address of current and alternate address space page table maps
 379  * and directories.
 380  */
 381
 382 extern pt_entry_t PTmap[], APTmap[], Upte;
 383 extern pd_entry_t PTD[], APTD[], PTDpde[], APTDpde[], Upde;
 384
 385 extern pd_entry_t *IdlePTD;     /* physical address of "Idle" state directory */
 386 extern pdpt_entry_t *IdlePDPT;
 387
 388 extern pmap_paddr_t lo_kernel_cr3;
 389
 390 extern pml4_entry_t *IdlePML4;
 391 extern pdpt_entry_t *IdlePDPT64;
 392 extern addr64_t     kernel64_cr3;
 393 extern boolean_t    no_shared_cr3;
 394
 395 extern uint64_t pmap_pv_hashlist_walks;
 396 extern uint64_t pmap_pv_hashlist_cnts;
 397 extern uint32_t pmap_pv_hashlist_max;
 398
 399 /*
 400  * virtual address to page table entry and
 401  * to physical address. Likewise for alternate address space.
 402  * Note: these work recursively, thus vtopte of a pte will give
 403  * the corresponding pde that in turn maps it.
 404  */
 405 #define vtopte(va)      (PTmap + i386_btop((vm_offset_t)va))
 406
 407
 408 typedef volatile long   cpu_set;        /* set of CPUs - must be <= 32 */
 409                                         /* changed by other processors */
 410 struct md_page {
 411   int pv_list_count;
 412   TAILQ_HEAD(,pv_entry)  pv_list;
 413 };
 414
 415 #include <vm/vm_page.h>
 416
 417 /*
 418  *      For each vm_page_t, there is a list of all currently
 419  *      valid virtual mappings of that page.  An entry is
 420  *      a pv_entry_t; the list is the pv_table.
 421  */
 422
 423 struct pmap {
 424         pd_entry_t      *dirbase;        /* page directory pointer */
 425         pmap_paddr_t    pdirbase;        /* phys. address of dirbase */
 426         vm_object_t     pm_obj;         /* object to hold pde's */
 427         int             ref_count;      /* reference count */
 428         int             nx_enabled;
 429         task_map_t      pm_task_map;
 430         decl_simple_lock_data(,lock)    /* lock on map */
 431         struct pmap_statistics  stats;  /* map statistics */
 432         vm_offset_t     pm_hold;        /* true pdpt zalloc addr */
 433         pmap_paddr_t    pm_cr3;         /* physical addr */
 434         pdpt_entry_t    *pm_pdpt;       /* KVA of 3rd level page */
 435         pml4_entry_t    *pm_pml4;       /* VKA of top level */
 436         vm_object_t     pm_obj_pdpt;    /* holds pdpt pages */
 437         vm_object_t     pm_obj_pml4;    /* holds pml4 pages */
 438         vm_object_t     pm_obj_top;     /* holds single top level page */
 439         boolean_t       pm_shared;
 440 };
 441
 442
 443 #define PMAP_PDPT_FIRST_WINDOW 0
 444 #define PMAP_PDPT_NWINDOWS 4
 445 #define PMAP_PDE_FIRST_WINDOW (PMAP_PDPT_NWINDOWS)
 446 #define PMAP_PDE_NWINDOWS 4
 447 #define PMAP_PTE_FIRST_WINDOW (PMAP_PDE_FIRST_WINDOW + PMAP_PDE_NWINDOWS)
 448 #define PMAP_PTE_NWINDOWS 4
 449
 450 #define PMAP_NWINDOWS_FIRSTFREE (PMAP_PTE_FIRST_WINDOW + PMAP_PTE_NWINDOWS)
 451 #define PMAP_WINDOW_SIZE 8
 452 #define PMAP_NWINDOWS (PMAP_NWINDOWS_FIRSTFREE + PMAP_WINDOW_SIZE)
 453
 454 typedef struct {
 455         pt_entry_t      *prv_CMAP;
 456         caddr_t         prv_CADDR;
 457 } mapwindow_t;
 458
 459 typedef struct cpu_pmap {
 460         int                     pdpt_window_index;
 461         int                     pde_window_index;
 462         int                     pte_window_index;
 463         mapwindow_t             mapwindow[PMAP_NWINDOWS];
 464 } cpu_pmap_t;
 465
 466
 467 extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry);
 468 extern void         pmap_put_mapwindow(mapwindow_t *map);
 469
 470
 471 typedef struct pmap_memory_regions {
 472   ppnum_t base;
 473   ppnum_t end;
 474   ppnum_t alloc;
 475   uint32_t type;
 476 } pmap_memory_region_t;
 477
 478 unsigned pmap_memory_region_count;
 479 unsigned pmap_memory_region_current;
 480
 481 #define PMAP_MEMORY_REGIONS_SIZE 128
 482
 483 extern pmap_memory_region_t pmap_memory_regions[];
 484
 485 static inline void set_dirbase(pmap_t tpmap, __unused int tcpu) {
 486         current_cpu_datap()->cpu_task_cr3 = (pmap_paddr_t)((tpmap)->pm_cr3);
 487         current_cpu_datap()->cpu_task_map = tpmap->pm_task_map;
 488 }
 489
 490 /*
 491  *      External declarations for PMAP_ACTIVATE.
 492  */
 493
 494 extern void             process_pmap_updates(void);
 495 extern void             pmap_update_interrupt(void);
 496
 497 /*
 498  *      Machine dependent routines that are used only for i386/i486/i860.
 499  */
 500
 501 extern addr64_t         (kvtophys)(
 502                                 vm_offset_t     addr);
 503
 504 extern void             pmap_expand(
 505                                 pmap_t          pmap,
 506                                 vm_map_offset_t addr);
 507
 508 extern pt_entry_t       *pmap_pte(
 509                                 struct pmap     *pmap,
 510                                 vm_map_offset_t addr);
 511
 512 extern pd_entry_t       *pmap_pde(
 513                                 struct pmap     *pmap,
 514                                 vm_map_offset_t addr);
 515
 516 extern pd_entry_t       *pmap64_pde(
 517                                 struct pmap     *pmap,
 518                                 vm_map_offset_t addr);
 519
 520 extern pdpt_entry_t     *pmap64_pdpt(
 521                                 struct pmap     *pmap,
 522                                 vm_map_offset_t addr);
 523
 524 extern vm_offset_t      pmap_map(
 525                                 vm_offset_t     virt,
 526                                 vm_map_offset_t start,
 527                                 vm_map_offset_t end,
 528                                 vm_prot_t       prot,
 529                                 unsigned int    flags);
 530
 531 extern vm_offset_t      pmap_map_bd(
 532                                 vm_offset_t     virt,
 533                                 vm_map_offset_t start,
 534                                 vm_map_offset_t end,
 535                                 vm_prot_t       prot,
 536                                 unsigned int    flags);
 537
 538 extern void             pmap_bootstrap(
 539                                 vm_offset_t     load_start,
 540                                 boolean_t       IA32e);
 541
 542 extern boolean_t        pmap_valid_page(
 543                                 ppnum_t pn);
 544
 545 extern int              pmap_list_resident_pages(
 546                                 struct pmap     *pmap,
 547                                 vm_offset_t     *listp,
 548                                 int             space);
 549
 550 extern void             pmap_commpage32_init(
 551                                            vm_offset_t kernel,
 552                                            vm_offset_t user,
 553                                            int count);
 554 extern void             pmap_commpage64_init(
 555                                            vm_offset_t  kernel,
 556                                            vm_map_offset_t user,
 557                                            int count);
 558
 559 extern struct cpu_pmap  *pmap_cpu_alloc(
 560                                 boolean_t       is_boot_cpu);
 561 extern void             pmap_cpu_free(
 562                                 struct cpu_pmap *cp);
 563
 564 extern void             pmap_map_block(
 565                                 pmap_t pmap,
 566                                 addr64_t va,
 567                                 ppnum_t pa,
 568                                 uint32_t size,
 569                                 vm_prot_t prot,
 570                                 int attr,
 571                                 unsigned int flags);
 572
 573 extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
 574 extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
 575 extern ppnum_t          pmap_find_phys(pmap_t map, addr64_t va);
 576
 577 extern void pmap_cpu_init(void);
 578 extern void pmap_disable_NX(pmap_t pmap);
 579 extern void pmap_set_4GB_pagezero(pmap_t pmap);
 580 extern void pmap_clear_4GB_pagezero(pmap_t pmap);
 581 extern void pmap_load_kernel_cr3(void);
 582 extern vm_offset_t pmap_cpu_high_map_vaddr(int, enum high_cpu_types);
 583 extern vm_offset_t pmap_high_map_vaddr(enum high_cpu_types);
 584 extern vm_offset_t pmap_high_map(pt_entry_t, enum high_cpu_types);
 585 extern vm_offset_t pmap_cpu_high_shared_remap(int, enum high_cpu_types, vm_offset_t, int);
 586 extern vm_offset_t pmap_high_shared_remap(enum high_fixed_addresses, vm_offset_t, int);
 587
 588 extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, int *, int *);
 589
 590
 591
 592 /*
 593  *      Macros for speed.
 594  */
 595
 596
 597 #include <kern/spl.h>
 598
 599 #if defined(PMAP_ACTIVATE_KERNEL)
 600 #undef PMAP_ACTIVATE_KERNEL
 601 #undef PMAP_DEACTIVATE_KERNEL
 602 #undef PMAP_ACTIVATE_USER
 603 #undef PMAP_DEACTIVATE_USER
 604 #endif
 605
 606
 607 #define PMAP_ACTIVATE_KERNEL(my_cpu)  {                                 \
 608         spl_t           spl;                                            \
 609                                                                         \
 610         spl = splhigh();                                                \
 611         if (current_cpu_datap()->cpu_tlb_invalid)                       \
 612                 process_pmap_updates();                                 \
 613         splx(spl);                                                      \
 614 }
 615
 616 #define PMAP_DEACTIVATE_KERNEL(my_cpu) {                                \
 617         spl_t           spl;                                            \
 618                                                                         \
 619         spl = splhigh();                                                \
 620         process_pmap_updates();                                         \
 621         splx(spl);                                                      \
 622 }
 623
 624
 625 #define PMAP_ACTIVATE_MAP(map, my_cpu)  {                               \
 626         register pmap_t         tpmap;                                  \
 627                                                                         \
 628         tpmap = vm_map_pmap(map);                                       \
 629         set_dirbase(tpmap, my_cpu);                                     \
 630 }
 631
 632 #define PMAP_DEACTIVATE_MAP(map, my_cpu)                                \
 633         if (vm_map_pmap(map)->pm_task_map == TASK_MAP_64BIT_SHARED)     \
 634                 pmap_load_kernel_cr3();
 635
 636
 637 #define PMAP_ACTIVATE_USER(th, my_cpu)  {                               \
 638         spl_t           spl;                                            \
 639                                                                         \
 640         spl = splhigh();                                                \
 641         PMAP_ACTIVATE_MAP(th->map, my_cpu)                              \
 642         splx(spl);                                                      \
 643 }
 644
 645 #define PMAP_DEACTIVATE_USER(th, my_cpu)
 646
 647
 648 #define PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) {                   \
 649         spl_t           spl;                                            \
 650         pt_entry_t      *kpdp;                                          \
 651         pt_entry_t      *updp;                                          \
 652         int             i;                                              \
 653         int             need_flush;                                     \
 654                                                                         \
 655         need_flush = 0;                                                 \
 656         spl = splhigh();                                                \
 657         if (old_th->map != new_th->map) {                               \
 658                 PMAP_DEACTIVATE_MAP(old_th->map, my_cpu);               \
 659                 PMAP_ACTIVATE_MAP(new_th->map, my_cpu);                 \
 660         }                                                               \
 661         kpdp = current_cpu_datap()->cpu_copywindow_pdp;                 \
 662         for (i = 0; i < NCOPY_WINDOWS; i++) {                           \
 663                 if (new_th->machine.copy_window[i].user_base != (user_addr_t)-1) {      \
 664                         updp = pmap_pde(new_th->map->pmap,              \
 665                               new_th->machine.copy_window[i].user_base);\
 666                         pmap_store_pte(kpdp, updp ? *updp : 0);         \
 667                 }                                                       \
 668                 kpdp++;                                                 \
 669         }                                                               \
 670         splx(spl);                                                      \
 671         if (new_th->machine.copyio_state == WINDOWS_OPENED)             \
 672                 need_flush = 1;                                         \
 673         else                                                            \
 674                 new_th->machine.copyio_state = WINDOWS_DIRTY;           \
 675         if (new_th->machine.physwindow_pte) {                           \
 676           pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep),    \
 677                                new_th->machine.physwindow_pte);         \
 678                 if (need_flush == 0)                                    \
 679                         invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);\
 680         }                                                               \
 681         if (need_flush)                                                 \
 682                 flush_tlb();                                            \
 683 }
 684
 685 #define PMAP_SWITCH_USER(th, new_map, my_cpu) {                         \
 686         spl_t           spl;                                            \
 687                                                                         \
 688         spl = splhigh();                                                \
 689         PMAP_DEACTIVATE_MAP(th->map, my_cpu);                           \
 690         th->map = new_map;                                              \
 691         PMAP_ACTIVATE_MAP(th->map, my_cpu);                             \
 692         splx(spl);                                                      \
 693         inval_copy_windows(th);                                         \
 694 }
 695
 696 /*
 697  * Marking the current cpu's cr3 inactive is achieved by setting its lsb.
 698  * Marking the current cpu's cr3 active once more involves clearng this bit.
 699  * Note that valid page tables are page-aligned and so the bottom 12 bits
 700  * are noramlly zero.
 701  * We can only mark the current cpu active/inactive but we can test any cpu.
 702  */
 703 #define CPU_CR3_MARK_INACTIVE()                                         \
 704         current_cpu_datap()->cpu_active_cr3 |= 1
 705
 706 #define CPU_CR3_MARK_ACTIVE()                                           \
 707         current_cpu_datap()->cpu_active_cr3 &= ~1
 708
 709 #define CPU_CR3_IS_ACTIVE(cpu)                                          \
 710         ((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0)
 711
 712 #define CPU_GET_ACTIVE_CR3(cpu)                                         \
 713         (cpu_datap(cpu)->cpu_active_cr3 & ~1)
 714
 715 #define MARK_CPU_IDLE(my_cpu)   {                                       \
 716         /*                                                              \
 717          *      Mark this cpu idle, and remove it from the active set,  \
 718          *      since it is not actively using any pmap.  Signal_cpus   \
 719          *      will notice that it is idle, and avoid signaling it,    \
 720          *      but will queue the update request for when the cpu      \
 721          *      becomes active.                                         \
 722          */                                                             \
 723         int     s = splhigh();                                          \
 724         if (!cpu_mode_is64bit() || no_shared_cr3)                       \
 725                 process_pmap_updates();                                 \
 726         else                                                            \
 727                 pmap_load_kernel_cr3();                                 \
 728         CPU_CR3_MARK_INACTIVE();                                        \
 729         __asm__ volatile("mfence");                                     \
 730         splx(s);                                                        \
 731 }
 732
 733 #define MARK_CPU_ACTIVE(my_cpu) {                                       \
 734                                                                         \
 735         int     s = splhigh();                                          \
 736         /*                                                              \
 737          *      If a kernel_pmap update was requested while this cpu    \
 738          *      was idle, process it as if we got the interrupt.        \
 739          *      Before doing so, remove this cpu from the idle set.     \
 740          *      Since we do not grab any pmap locks while we flush      \
 741          *      our TLB, another cpu may start an update operation      \
 742          *      before we finish.  Removing this cpu from the idle      \
 743          *      set assures that we will receive another update         \
 744          *      interrupt if this happens.                              \
 745          */                                                             \
 746         CPU_CR3_MARK_ACTIVE();                                          \
 747         __asm__ volatile("mfence");                                     \
 748                                                                         \
 749         if (current_cpu_datap()->cpu_tlb_invalid)                       \
 750             process_pmap_updates();                                     \
 751         splx(s);                                                        \
 752 }
 753
 754 #define PMAP_CONTEXT(pmap, thread)
 755
 756 #define pmap_kernel_va(VA)      \
 757         ((((vm_offset_t) (VA)) >= vm_min_kernel_address) &&     \
 758          (((vm_offset_t) (VA)) <= vm_max_kernel_address))
 759
 760
 761 #define pmap_resident_count(pmap)       ((pmap)->stats.resident_count)
 762 #define pmap_resident_max(pmap)         ((pmap)->stats.resident_max)
 763 #define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr)
 764 #define pmap_attribute(pmap,addr,size,attr,value) \
 765                                         (KERN_INVALID_ADDRESS)
 766 #define pmap_attribute_cache_sync(addr,size,attr,value) \
 767                                         (KERN_INVALID_ADDRESS)
 768
 769 #define MACHINE_PMAP_IS_EMPTY 1
 770 extern boolean_t pmap_is_empty(pmap_t           pmap,
 771                                vm_map_offset_t  start,
 772                                vm_map_offset_t  end);
 773
 774 #endif  /* ASSEMBLER */
 775
 776
 777 #endif  /* _PMAP_MACHINE_ */
 778
 779
 780 #endif  /* KERNEL_PRIVATE */