osfmk/i386/pmap.h

   1 /*
   2  * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58
  59 /*
  60  *      File:   pmap.h
  61  *
  62  *      Authors:  Avadis Tevanian, Jr., Michael Wayne Young
  63  *      Date:   1985
  64  *
  65  *      Machine-dependent structures for the physical map module.
  66  */
  67 #ifdef KERNEL_PRIVATE
  68 #ifndef _PMAP_MACHINE_
  69 #define _PMAP_MACHINE_  1
  70
  71 #ifndef ASSEMBLER
  72
  73 #include <platforms.h>
  74
  75 #include <mach/kern_return.h>
  76 #include <mach/machine/vm_types.h>
  77 #include <mach/vm_prot.h>
  78 #include <mach/vm_statistics.h>
  79 #include <mach/machine/vm_param.h>
  80 #include <kern/kern_types.h>
  81 #include <kern/thread.h>
  82 #include <kern/lock.h>
  83
  84 #include <i386/mp.h>
  85 #include <i386/proc_reg.h>
  86
  87 /*
  88  *      Define the generic in terms of the specific
  89  */
  90
  91 #define INTEL_PGBYTES           I386_PGBYTES
  92 #define INTEL_PGSHIFT           I386_PGSHIFT
  93 #define intel_btop(x)           i386_btop(x)
  94 #define intel_ptob(x)           i386_ptob(x)
  95 #define intel_round_page(x)     i386_round_page(x)
  96 #define intel_trunc_page(x)     i386_trunc_page(x)
  97 #define trunc_intel_to_vm(x)    trunc_i386_to_vm(x)
  98 #define round_intel_to_vm(x)    round_i386_to_vm(x)
  99 #define vm_to_intel(x)          vm_to_i386(x)
 100
 101 /*
 102  *      i386/i486/i860 Page Table Entry
 103  */
 104
 105 #endif  /* ASSEMBLER */
 106
 107 #define NPGPTD          4
 108 #define PDESHIFT        21
 109 #define PTEMASK         0x1ff
 110 #define PTEINDX         3
 111
 112 #define PTESHIFT        12
 113
 114
 115 #define INITPT_SEG_BASE  0x100000
 116 #define INITGDT_SEG_BASE 0x106000
 117 #define SLEEP_SEG_BASE   0x107000
 118
 119 #ifdef __x86_64__
 120 #define LOW_4GB_MASK    ((vm_offset_t)0x00000000FFFFFFFFUL)
 121 #endif
 122
 123 #define PDESIZE         sizeof(pd_entry_t) /* for assembly files */
 124 #define PTESIZE         sizeof(pt_entry_t) /* for assembly files */
 125
 126 #define INTEL_OFFMASK   (I386_PGBYTES - 1)
 127 #define INTEL_LOFFMASK  (I386_LPGBYTES - 1)
 128 #define PG_FRAME        0x000FFFFFFFFFF000ULL
 129 #define NPTEPG          (PAGE_SIZE/(sizeof (pt_entry_t)))
 130 #define NPTDPG          (PAGE_SIZE/(sizeof (pd_entry_t)))
 131
 132 #define NBPTD           (NPGPTD << PAGE_SHIFT)
 133 #define NPDEPTD         (NBPTD / (sizeof (pd_entry_t)))
 134 #define NPDEPG          (PAGE_SIZE/(sizeof (pd_entry_t)))
 135 #define NBPDE           (1 << PDESHIFT)
 136 #define PDEMASK         (NBPDE - 1)
 137
 138 #define PTE_PER_PAGE    512 /* number of PTE's per page on any level */
 139
 140  /* cleanly define parameters for all the page table levels */
 141 typedef uint64_t        pml4_entry_t;
 142 #define NPML4PG         (PAGE_SIZE/(sizeof (pml4_entry_t)))
 143 #define PML4SHIFT       39
 144 #define PML4PGSHIFT     9
 145 #define NBPML4          (1ULL << PML4SHIFT)
 146 #define PML4MASK        (NBPML4-1)
 147 #define PML4_ENTRY_NULL ((pml4_entry_t *) 0)
 148
 149 typedef uint64_t        pdpt_entry_t;
 150 #define NPDPTPG         (PAGE_SIZE/(sizeof (pdpt_entry_t)))
 151 #define PDPTSHIFT       30
 152 #define PDPTPGSHIFT     9
 153 #define NBPDPT          (1 << PDPTSHIFT)
 154 #define PDPTMASK        (NBPDPT-1)
 155 #define PDPT_ENTRY_NULL ((pdpt_entry_t *) 0)
 156
 157 typedef uint64_t        pd_entry_t;
 158 #define NPDPG           (PAGE_SIZE/(sizeof (pd_entry_t)))
 159 #define PDSHIFT         21
 160 #define PDPGSHIFT       9
 161 #define NBPD            (1 << PDSHIFT)
 162 #define PDMASK          (NBPD-1)
 163 #define PD_ENTRY_NULL   ((pd_entry_t *) 0)
 164
 165 typedef uint64_t        pt_entry_t;
 166 #define NPTPG           (PAGE_SIZE/(sizeof (pt_entry_t)))
 167 #define PTSHIFT         12
 168 #define PTPGSHIFT       9
 169 #define NBPT            (1 << PTSHIFT)
 170 #define PTMASK          (NBPT-1)
 171 #define PT_ENTRY_NULL   ((pt_entry_t *) 0)
 172
 173 typedef uint64_t  pmap_paddr_t;
 174
 175 /* superpages */
 176 #ifdef __x86_64__
 177 #define SUPERPAGE_NBASEPAGES 512
 178 #else
 179 #define SUPERPAGE_NBASEPAGES 1  /* we don't support superpages on i386 */
 180 #endif
 181
 182 /*
 183  * Atomic 64-bit store of a page table entry.
 184  */
 185 static inline void
 186 pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
 187 {
 188 #ifdef __i386__
 189         /*
 190          * Load the new value into %ecx:%ebx
 191          * Load the old value into %edx:%eax
 192          * Compare-exchange-8bytes at address entryp (loaded in %edi)
 193          * If the compare succeeds, the new value will have been stored.
 194          * Otherwise, the old value changed and reloaded, so try again.
 195          */
 196         __asm__ volatile(
 197                 "       movl    (%0), %%eax     \n\t"
 198                 "       movl    4(%0), %%edx    \n\t"
 199                 "1:                             \n\t"
 200                 "       cmpxchg8b (%0)          \n\t"
 201                 "       jnz 1b"
 202                 :
 203                 : "D" (entryp),
 204                   "b" ((uint32_t)value),
 205                   "c" ((uint32_t)(value >> 32))
 206                 : "eax", "edx", "memory");
 207 #else
 208         /*
 209          * In the 32-bit kernel a compare-and-exchange loop was
 210          * required to provide atomicity. For K64, life is easier:
 211          */
 212         *entryp = value;
 213 #endif
 214 }
 215
 216 /*
 217  * Atomic 64-bit compare and exchange of a page table entry.
 218  */
 219 static inline boolean_t
 220 pmap_cmpx_pte(pt_entry_t *entryp, pt_entry_t old, pt_entry_t new)
 221 {
 222         boolean_t               ret;
 223
 224 #ifdef __i386__
 225         /*
 226          * Load the old value into %edx:%eax
 227          * Load the new value into %ecx:%ebx
 228          * Compare-exchange-8bytes at address entryp (loaded in %edi)
 229          * If the compare succeeds, the new value is stored, return TRUE.
 230          * Otherwise, no swap is made, return FALSE.
 231          */
 232         asm volatile(
 233                 "       lock; cmpxchg8b (%1)    \n\t"
 234                 "       setz    %%al            \n\t"
 235                 "       movzbl  %%al,%0"
 236                 : "=a" (ret)
 237                 : "D" (entryp),
 238                   "a" ((uint32_t)old),
 239                   "d" ((uint32_t)(old >> 32)),
 240                   "b" ((uint32_t)new),
 241                   "c" ((uint32_t)(new >> 32))
 242                 : "memory");
 243 #else
 244         /*
 245          * Load the old value into %rax
 246          * Load the new value into another register
 247          * Compare-exchange-quad at address entryp
 248          * If the compare succeeds, the new value is stored, return TRUE.
 249          * Otherwise, no swap is made, return FALSE.
 250          */
 251         asm volatile(
 252                 "       lock; cmpxchgq %2,(%3)  \n\t"
 253                 "       setz    %%al            \n\t"
 254                 "       movzbl  %%al,%0"
 255                 : "=a" (ret)
 256                 : "a" (old),
 257                   "r" (new),
 258                   "r" (entryp)
 259                 : "memory");
 260 #endif
 261         return ret;
 262 }
 263
 264 #define pmap_update_pte(entryp, old, new) \
 265         while (!pmap_cmpx_pte((entryp), (old), (new)))
 266
 267
 268 /* in 64 bit spaces, the number of each type of page in the page tables */
 269 #define NPML4PGS        (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
 270 #define NPDPTPGS        (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
 271 #define NPDEPGS         (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t))))
 272 #define NPTEPGS         (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
 273
 274 #ifdef __i386__
 275 /*
 276  * The 64-bit kernel is remapped in uber-space which is at the base
 277  * the highest 4th-level directory (KERNEL_UBER_PML4_INDEX). That is,
 278  * 512GB from the top of virtual space (or zero).
 279  */
 280 #define KERNEL_UBER_PML4_INDEX  511
 281 #define KERNEL_UBER_BASE        (0ULL - NBPML4)
 282 #define KERNEL_UBER_BASE_HI32   ((uint32_t)(KERNEL_UBER_BASE >> 32))
 283 #else
 284 #define KERNEL_PML4_INDEX       511
 285 #define KERNEL_KEXTS_INDEX      510     /* Home of KEXTs - the basement */
 286 #define KERNEL_PHYSMAP_INDEX    509     /* virtual to physical map */
 287 #define KERNEL_BASE             (0ULL - NBPML4)
 288 #define KERNEL_BASEMENT         (KERNEL_BASE - NBPML4)
 289 #endif
 290
 291 #define VM_WIMG_COPYBACK        VM_MEM_COHERENT
 292 #define VM_WIMG_DEFAULT         VM_MEM_COHERENT
 293 /* ?? intel ?? */
 294 #define VM_WIMG_IO              (VM_MEM_COHERENT |      \
 295                                 VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED)
 296 #define VM_WIMG_WTHRU           (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED)
 297 /* write combining mode, aka store gather */
 298 #define VM_WIMG_WCOMB           (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT)
 299
 300 /*
 301  * Pte related macros
 302  */
 303 #ifdef __i386__
 304 #define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDESHIFT)|((pti)<<PTESHIFT)))
 305 #define VADDR64(pmi, pdi, pti) ((vm_offset_t)(((pmi)<<PLM4SHIFT))((pdi)<<PDESHIFT)|((pti)<<PTESHIFT))
 306 #else
 307 #define KVADDR(pmi, pdpi, pdi, pti)               \
 308          ((vm_offset_t)                   \
 309                 ((uint64_t) -1    << 47)        | \
 310                 ((uint64_t)(pmi)  << PML4SHIFT) | \
 311                 ((uint64_t)(pdpi) << PDPTSHIFT) | \
 312                 ((uint64_t)(pdi)  << PDESHIFT)  | \
 313                 ((uint64_t)(pti)  << PTESHIFT))
 314 #endif
 315
 316 /*
 317  * Size of Kernel address space.  This is the number of page table pages
 318  * (4MB each) to use for the kernel.  256 pages == 1 Gigabyte.
 319  * This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc).
 320  */
 321 #ifndef KVA_PAGES
 322 #define KVA_PAGES       1024
 323 #endif
 324
 325 #ifndef NKPT
 326 #define NKPT            500     /* actual number of kernel page tables */
 327 #endif
 328 #ifndef NKPDE
 329 #define NKPDE   (KVA_PAGES - 1) /* addressable number of page tables/pde's */
 330 #endif
 331
 332
 333 #ifdef __i386__
 334 enum high_cpu_types {
 335   HIGH_CPU_ISS0,
 336   HIGH_CPU_ISS1,
 337   HIGH_CPU_DESC,
 338   HIGH_CPU_LDT_BEGIN,
 339   HIGH_CPU_LDT_END = HIGH_CPU_LDT_BEGIN + (LDTSZ / 512) - 1,
 340   HIGH_CPU_END
 341 };
 342
 343 enum  high_fixed_addresses {
 344   HIGH_FIXED_TRAMPS,  /* must be first */
 345   HIGH_FIXED_TRAMPS_END,
 346   HIGH_FIXED_GDT,
 347   HIGH_FIXED_IDT,
 348   HIGH_FIXED_LDT_BEGIN,
 349   HIGH_FIXED_LDT_END = HIGH_FIXED_LDT_BEGIN + (LDTSZ / 512) - 1,
 350   HIGH_FIXED_KTSS,
 351   HIGH_FIXED_DFTSS,
 352   HIGH_FIXED_DBTSS,
 353   HIGH_FIXED_CPUS_BEGIN,
 354   HIGH_FIXED_CPUS_END = HIGH_FIXED_CPUS_BEGIN + (HIGH_CPU_END * MAX_CPUS) - 1,
 355 };
 356
 357
 358 /* XXX64  below PTDI values need cleanup */
 359 /*
 360  * The *PTDI values control the layout of virtual memory
 361  *
 362  */
 363 #define        KPTDI           (0x000)/* start of kernel virtual pde's */
 364 #define        PTDPTDI         (0x7F4) /* ptd entry that points to ptd! */
 365 #define        APTDPTDI        (0x7F8) /* alt ptd entry that points to APTD */
 366 #define        UMAXPTDI        (0x7F8) /* ptd entry for user space end */
 367 #define UMAXPTEOFF      (NPTEPG)        /* pte entry for user space end */
 368
 369 #define KERNBASE       VADDR(KPTDI,0)
 370
 371 /*
 372  *      Convert address offset to directory address
 373  *      containing the page table pointer - legacy
 374  */
 375 /*#define pmap_pde(m,v) (&((m)->dirbase[(vm_offset_t)(v) >> PDESHIFT]))*/
 376
 377 #define HIGH_MEM_BASE  ((uint32_t)( -NBPDE) )  /* shared gdt etc seg addr */ /* XXX64 ?? */
 378 #define pmap_index_to_virt(x)  (HIGH_MEM_BASE | ((unsigned)(x) << PAGE_SHIFT))
 379 #endif
 380
 381 /*
 382  *      Convert address offset to page descriptor index
 383  */
 384 #define pdptnum(pmap, a) (((vm_offset_t)(a) >> PDPTSHIFT) & PDPTMASK)
 385 #define pdenum(pmap, a) (((vm_offset_t)(a) >> PDESHIFT) & PDEMASK)
 386 #define PMAP_INVALID_PDPTNUM (~0ULL)
 387
 388 #ifdef __i386__
 389 #define pdeidx(pmap, a)    (((a) >> PDSHIFT)   & ((1ULL<<(48 - PDSHIFT)) -1))
 390 #define pdptidx(pmap, a)   (((a) >> PDPTSHIFT) & ((1ULL<<(48 - PDPTSHIFT)) -1))
 391 #define pml4idx(pmap, a)   (((a) >> PML4SHIFT) & ((1ULL<<(48 - PML4SHIFT)) -1))
 392 #else
 393 #define VAMASK             ((1ULL<<48)-1)
 394 #define pml4idx(pmap, a)   ((((a) & VAMASK) >> PML4SHIFT) &     \
 395                                 ((1ULL<<(48 - PML4SHIFT))-1))
 396 #define pdptidx(pmap, a)   ((((a) & PML4MASK) >> PDPTSHIFT) &   \
 397                                 ((1ULL<<(48 - PDPTSHIFT))-1))
 398 #define pdeidx(pmap, a)    ((((a) & PML4MASK) >> PDSHIFT) &     \
 399                                 ((1ULL<<(48 - PDSHIFT)) - 1))
 400 #endif
 401
 402 /*
 403  *      Convert page descriptor index to user virtual address
 404  */
 405 #define pdetova(a)      ((vm_offset_t)(a) << PDESHIFT)
 406
 407 /*
 408  *      Convert address offset to page table index
 409  */
 410 #define ptenum(a)       (((vm_offset_t)(a) >> PTESHIFT) & PTEMASK)
 411
 412 /*
 413  *      Hardware pte bit definitions (to be used directly on the ptes
 414  *      without using the bit fields).
 415  */
 416
 417 #define INTEL_PTE_VALID         0x00000001
 418 #define INTEL_PTE_WRITE         0x00000002
 419 #define INTEL_PTE_RW            0x00000002
 420 #define INTEL_PTE_USER          0x00000004
 421 #define INTEL_PTE_WTHRU         0x00000008
 422 #define INTEL_PTE_NCACHE        0x00000010
 423 #define INTEL_PTE_REF           0x00000020
 424 #define INTEL_PTE_MOD           0x00000040
 425 #define INTEL_PTE_PS            0x00000080
 426 #define INTEL_PTE_PTA           0x00000080
 427 #define INTEL_PTE_GLOBAL        0x00000100
 428 #define INTEL_PTE_WIRED         0x00000200
 429 #define INTEL_PDPTE_NESTED      0x00000400
 430 #define INTEL_PTE_PFN           PG_FRAME
 431
 432 #define INTEL_PTE_NX            (1ULL << 63)
 433
 434 #define INTEL_PTE_INVALID       0
 435 /* This is conservative, but suffices */
 436 #define INTEL_PTE_RSVD          ((1ULL << 8) | (1ULL << 9) | (1ULL << 10) | (1ULL << 11) | (0x1FFULL << 54))
 437 #define pa_to_pte(a)            ((a) & INTEL_PTE_PFN) /* XXX */
 438 #define pte_to_pa(p)            ((p) & INTEL_PTE_PFN) /* XXX */
 439 #define pte_increment_pa(p)     ((p) += INTEL_OFFMASK+1)
 440
 441 #define pte_kernel_rw(p)          ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_RW))
 442 #define pte_kernel_ro(p)          ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID))
 443 #define pte_user_rw(p)            ((pt_entry)t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER|INTEL_PTE_RW))
 444 #define pte_user_ro(p)            ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER))
 445
 446 #define PMAP_DEFAULT_CACHE      0
 447 #define PMAP_INHIBIT_CACHE      1
 448 #define PMAP_GUARDED_CACHE      2
 449 #define PMAP_ACTIVATE_CACHE     4
 450 #define PMAP_NO_GUARD_CACHE     8
 451
 452
 453 #ifndef ASSEMBLER
 454
 455 #include <sys/queue.h>
 456
 457 /*
 458  * Address of current and alternate address space page table maps
 459  * and directories.
 460  */
 461
 462 #ifdef __i386__
 463 extern pt_entry_t       PTmap[], APTmap[], Upte;
 464 extern pd_entry_t       PTD[], APTD[], PTDpde[], APTDpde[], Upde;
 465 extern pmap_paddr_t     lo_kernel_cr3;
 466 extern pdpt_entry_t     *IdlePDPT64;
 467 #else
 468 extern pt_entry_t       *PTmap;
 469 #endif
 470 extern boolean_t        no_shared_cr3;
 471 extern addr64_t         kernel64_cr3;
 472 extern pd_entry_t       *IdlePTD;       /* physical addr of "Idle" state PTD */
 473 extern pdpt_entry_t     IdlePDPT[];
 474 extern pml4_entry_t     IdlePML4[];
 475
 476 extern uint64_t         pmap_pv_hashlist_walks;
 477 extern uint64_t         pmap_pv_hashlist_cnts;
 478 extern uint32_t         pmap_pv_hashlist_max;
 479 extern uint32_t         pmap_kernel_text_ps;
 480
 481 #ifdef __i386__
 482 /*
 483  * ** i386 **
 484  * virtual address to page table entry and
 485  * to physical address. Likewise for alternate address space.
 486  * Note: these work recursively, thus vtopte of a pte will give
 487  * the corresponding pde that in turn maps it.
 488  */
 489
 490 #define vtopte(va)      (PTmap + i386_btop((vm_offset_t)va))
 491 #endif
 492
 493 #ifdef __x86_64__
 494 #define ID_MAP_VTOP(x)  ((void *)(((uint64_t)(x)) & LOW_4GB_MASK))
 495
 496 #define PHYSMAP_BASE    KVADDR(KERNEL_PHYSMAP_INDEX,0,0,0)
 497 #define PHYSMAP_PTOV(x) ((void *)(((uint64_t)(x)) + PHYSMAP_BASE))
 498 #endif
 499
 500 typedef volatile long   cpu_set;        /* set of CPUs - must be <= 32 */
 501                                         /* changed by other processors */
 502 struct md_page {
 503   int pv_list_count;
 504   TAILQ_HEAD(,pv_entry)  pv_list;
 505 };
 506
 507 #include <vm/vm_page.h>
 508
 509 /*
 510  *      For each vm_page_t, there is a list of all currently
 511  *      valid virtual mappings of that page.  An entry is
 512  *      a pv_entry_t; the list is the pv_table.
 513  */
 514
 515 struct pmap {
 516         pd_entry_t      *dirbase;        /* page directory pointer */
 517 #ifdef __i386__
 518         pmap_paddr_t    pdirbase;        /* phys. address of dirbase */
 519 #endif
 520         vm_object_t     pm_obj;         /* object to hold pde's */
 521         int             ref_count;      /* reference count */
 522         int             nx_enabled;
 523         task_map_t      pm_task_map;
 524         decl_simple_lock_data(,lock)    /* lock on map */
 525         struct pmap_statistics  stats;  /* map statistics */
 526 #ifdef __i386__
 527         vm_offset_t     pm_hold;        /* true pdpt zalloc addr */
 528 #endif
 529         pmap_paddr_t    pm_cr3;         /* physical addr */
 530         pdpt_entry_t    *pm_pdpt;       /* KVA of 3rd level page */
 531         pml4_entry_t    *pm_pml4;       /* VKA of top level */
 532         vm_object_t     pm_obj_pdpt;    /* holds pdpt pages */
 533         vm_object_t     pm_obj_pml4;    /* holds pml4 pages */
 534         vm_object_t     pm_obj_top;     /* holds single top level page */
 535         boolean_t       pm_shared;
 536 };
 537
 538
 539 #if NCOPY_WINDOWS > 0
 540 #define PMAP_PDPT_FIRST_WINDOW 0
 541 #define PMAP_PDPT_NWINDOWS 4
 542 #define PMAP_PDE_FIRST_WINDOW (PMAP_PDPT_NWINDOWS)
 543 #define PMAP_PDE_NWINDOWS 4
 544 #define PMAP_PTE_FIRST_WINDOW (PMAP_PDE_FIRST_WINDOW + PMAP_PDE_NWINDOWS)
 545 #define PMAP_PTE_NWINDOWS 4
 546
 547 #define PMAP_NWINDOWS_FIRSTFREE (PMAP_PTE_FIRST_WINDOW + PMAP_PTE_NWINDOWS)
 548 #define PMAP_WINDOW_SIZE 8
 549 #define PMAP_NWINDOWS (PMAP_NWINDOWS_FIRSTFREE + PMAP_WINDOW_SIZE)
 550
 551 typedef struct {
 552         pt_entry_t      *prv_CMAP;
 553         caddr_t         prv_CADDR;
 554 } mapwindow_t;
 555
 556 typedef struct cpu_pmap {
 557         int                     pdpt_window_index;
 558         int                     pde_window_index;
 559         int                     pte_window_index;
 560         mapwindow_t             mapwindow[PMAP_NWINDOWS];
 561 } cpu_pmap_t;
 562
 563
 564 extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry);
 565 extern void         pmap_put_mapwindow(mapwindow_t *map);
 566 #endif
 567
 568 typedef struct pmap_memory_regions {
 569   ppnum_t base;
 570   ppnum_t end;
 571   ppnum_t alloc;
 572   uint32_t type;
 573 } pmap_memory_region_t;
 574
 575 extern unsigned pmap_memory_region_count;
 576 extern unsigned pmap_memory_region_current;
 577
 578 #define PMAP_MEMORY_REGIONS_SIZE 128
 579
 580 extern pmap_memory_region_t pmap_memory_regions[];
 581
 582 static inline void
 583 set_dirbase(pmap_t tpmap, __unused thread_t thread) {
 584         current_cpu_datap()->cpu_task_cr3 = tpmap->pm_cr3;
 585         current_cpu_datap()->cpu_task_map = tpmap->pm_task_map;
 586 #ifndef __i386__
 587         /*
 588          * Switch cr3 if necessary
 589          * - unless running with no_shared_cr3 debugging mode
 590          *   and we're not on the kernel's cr3 (after pre-empted copyio)
 591          */
 592         if (!no_shared_cr3) {
 593                 if (get_cr3() != tpmap->pm_cr3)
 594                         set_cr3(tpmap->pm_cr3);
 595         } else {
 596                 if (get_cr3() != current_cpu_datap()->cpu_kernel_cr3)
 597                         set_cr3(current_cpu_datap()->cpu_kernel_cr3);
 598         }
 599 #endif
 600 }
 601
 602 /*
 603  *      External declarations for PMAP_ACTIVATE.
 604  */
 605
 606 extern void             process_pmap_updates(void);
 607 extern void             pmap_update_interrupt(void);
 608
 609 /*
 610  *      Machine dependent routines that are used only for i386/i486/i860.
 611  */
 612
 613 extern addr64_t         (kvtophys)(
 614                                 vm_offset_t     addr);
 615
 616 extern void             pmap_expand(
 617                                 pmap_t          pmap,
 618                                 vm_map_offset_t addr);
 619
 620 extern pt_entry_t       *pmap_pte(
 621                                 struct pmap     *pmap,
 622                                 vm_map_offset_t addr);
 623
 624 extern pd_entry_t       *pmap_pde(
 625                                 struct pmap     *pmap,
 626                                 vm_map_offset_t addr);
 627
 628 extern pd_entry_t       *pmap64_pde(
 629                                 struct pmap     *pmap,
 630                                 vm_map_offset_t addr);
 631
 632 extern pdpt_entry_t     *pmap64_pdpt(
 633                                 struct pmap     *pmap,
 634                                 vm_map_offset_t addr);
 635
 636 extern vm_offset_t      pmap_map(
 637                                 vm_offset_t     virt,
 638                                 vm_map_offset_t start,
 639                                 vm_map_offset_t end,
 640                                 vm_prot_t       prot,
 641                                 unsigned int    flags);
 642
 643 extern vm_offset_t      pmap_map_bd(
 644                                 vm_offset_t     virt,
 645                                 vm_map_offset_t start,
 646                                 vm_map_offset_t end,
 647                                 vm_prot_t       prot,
 648                                 unsigned int    flags);
 649
 650 extern void             pmap_bootstrap(
 651                                 vm_offset_t     load_start,
 652                                 boolean_t       IA32e);
 653
 654 extern boolean_t        pmap_valid_page(
 655                                 ppnum_t pn);
 656
 657 extern int              pmap_list_resident_pages(
 658                                 struct pmap     *pmap,
 659                                 vm_offset_t     *listp,
 660                                 int             space);
 661
 662 #ifdef __i386__
 663 extern void             pmap_commpage32_init(
 664                                            vm_offset_t kernel,
 665                                            vm_offset_t user,
 666                                            int count);
 667 extern void             pmap_commpage64_init(
 668                                            vm_offset_t  kernel,
 669                                            vm_map_offset_t user,
 670                                            int count);
 671
 672 #endif
 673
 674 #if NCOPY_WINDOWS > 0
 675 extern struct cpu_pmap  *pmap_cpu_alloc(
 676                                 boolean_t       is_boot_cpu);
 677 extern void             pmap_cpu_free(
 678                                 struct cpu_pmap *cp);
 679 #endif
 680
 681 extern void             pmap_map_block(
 682                                 pmap_t pmap,
 683                                 addr64_t va,
 684                                 ppnum_t pa,
 685                                 uint32_t size,
 686                                 vm_prot_t prot,
 687                                 int attr,
 688                                 unsigned int flags);
 689
 690 extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
 691 extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
 692 extern ppnum_t          pmap_find_phys(pmap_t map, addr64_t va);
 693
 694 extern void pmap_cpu_init(void);
 695 extern void pmap_disable_NX(pmap_t pmap);
 696 #ifdef __i386__
 697 extern void pmap_set_4GB_pagezero(pmap_t pmap);
 698 extern void pmap_clear_4GB_pagezero(pmap_t pmap);
 699 extern void pmap_load_kernel_cr3(void);
 700 extern vm_offset_t pmap_cpu_high_map_vaddr(int, enum high_cpu_types);
 701 extern vm_offset_t pmap_high_map_vaddr(enum high_cpu_types);
 702 extern vm_offset_t pmap_high_map(pt_entry_t, enum high_cpu_types);
 703 extern vm_offset_t pmap_cpu_high_shared_remap(int, enum high_cpu_types, vm_offset_t, int);
 704 extern vm_offset_t pmap_high_shared_remap(enum high_fixed_addresses, vm_offset_t, int);
 705 #endif
 706
 707 extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, int *, int *);
 708 extern void pmap_pagetable_corruption_msg_log(int (*)(const char * fmt, ...)__printflike(1,2));
 709
 710
 711 /*
 712  *      Macros for speed.
 713  */
 714
 715
 716 #include <kern/spl.h>
 717
 718
 719 #define PMAP_ACTIVATE_MAP(map, thread)  {                               \
 720         register pmap_t         tpmap;                                  \
 721                                                                         \
 722         tpmap = vm_map_pmap(map);                                       \
 723         set_dirbase(tpmap, thread);                                     \
 724 }
 725
 726 #ifdef __i386__
 727 #define PMAP_DEACTIVATE_MAP(map, thread)                                \
 728         if (vm_map_pmap(map)->pm_task_map == TASK_MAP_64BIT_SHARED)     \
 729                 pmap_load_kernel_cr3();
 730 #else
 731 #define PMAP_DEACTIVATE_MAP(map, my_cpu)
 732 #endif
 733
 734 #if   defined(__i386__)
 735
 736 #define PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) {                   \
 737         spl_t           spl;                                            \
 738         pt_entry_t      *kpdp;                                          \
 739         pt_entry_t      *updp;                                          \
 740         int             i;                                              \
 741         int             need_flush;                                     \
 742                                                                         \
 743         need_flush = 0;                                                 \
 744         spl = splhigh();                                                \
 745         if ((old_th->map != new_th->map) || (new_th->task != old_th->task)) {   \
 746                 PMAP_DEACTIVATE_MAP(old_th->map, old_th);               \
 747                 PMAP_ACTIVATE_MAP(new_th->map, new_th);                 \
 748         }                                                               \
 749         kpdp = current_cpu_datap()->cpu_copywindow_pdp;                 \
 750         for (i = 0; i < NCOPY_WINDOWS; i++) {                           \
 751                 if (new_th->machine.copy_window[i].user_base != (user_addr_t)-1) {      \
 752                         updp = pmap_pde(new_th->map->pmap,              \
 753                               new_th->machine.copy_window[i].user_base);\
 754                         pmap_store_pte(kpdp, updp ? *updp : 0);         \
 755                 }                                                       \
 756                 kpdp++;                                                 \
 757         }                                                               \
 758         splx(spl);                                                      \
 759         if (new_th->machine.copyio_state == WINDOWS_OPENED)             \
 760                 need_flush = 1;                                         \
 761         else                                                            \
 762                 new_th->machine.copyio_state = WINDOWS_DIRTY;           \
 763         if (new_th->machine.physwindow_pte) {                           \
 764           pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep),    \
 765                                new_th->machine.physwindow_pte);         \
 766                 if (need_flush == 0)                                    \
 767                         invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);\
 768         }                                                               \
 769         if (need_flush)                                                 \
 770                 flush_tlb();                                            \
 771 }
 772
 773 #else /* __x86_64__ */
 774 #define PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) {                   \
 775         spl_t           spl;                                            \
 776                                                                         \
 777         spl = splhigh();                                                \
 778         if (old_th->map != new_th->map) {                               \
 779                 PMAP_DEACTIVATE_MAP(old_th->map, old_th);               \
 780                 PMAP_ACTIVATE_MAP(new_th->map, new_th);                 \
 781         }                                                               \
 782         splx(spl);                                                      \
 783 }
 784 #endif /* __i386__ */
 785
 786 #ifdef __i386__
 787 #define PMAP_SWITCH_USER(th, new_map, my_cpu) {                         \
 788         spl_t           spl;                                            \
 789                                                                         \
 790         spl = splhigh();                                                \
 791         PMAP_DEACTIVATE_MAP(th->map, th);                               \
 792         th->map = new_map;                                              \
 793         PMAP_ACTIVATE_MAP(th->map, th);                                 \
 794         splx(spl);                                                      \
 795         inval_copy_windows(th);                                         \
 796 }
 797 #else
 798 #define PMAP_SWITCH_USER(th, new_map, my_cpu) {                         \
 799         spl_t           spl;                                            \
 800                                                                         \
 801         spl = splhigh();                                                \
 802         PMAP_DEACTIVATE_MAP(th->map, th);                               \
 803         th->map = new_map;                                              \
 804         PMAP_ACTIVATE_MAP(th->map, th);                                 \
 805         splx(spl);                                                      \
 806 }
 807 #endif
 808
 809 /*
 810  * Marking the current cpu's cr3 inactive is achieved by setting its lsb.
 811  * Marking the current cpu's cr3 active once more involves clearng this bit.
 812  * Note that valid page tables are page-aligned and so the bottom 12 bits
 813  * are noramlly zero.
 814  * We can only mark the current cpu active/inactive but we can test any cpu.
 815  */
 816 #define CPU_CR3_MARK_INACTIVE()                                         \
 817         current_cpu_datap()->cpu_active_cr3 |= 1
 818
 819 #define CPU_CR3_MARK_ACTIVE()                                           \
 820         current_cpu_datap()->cpu_active_cr3 &= ~1
 821
 822 #define CPU_CR3_IS_ACTIVE(cpu)                                          \
 823         ((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0)
 824
 825 #define CPU_GET_ACTIVE_CR3(cpu)                                         \
 826         (cpu_datap(cpu)->cpu_active_cr3 & ~1)
 827
 828 #define CPU_GET_TASK_CR3(cpu)                                           \
 829         (cpu_datap(cpu)->cpu_task_cr3)
 830
 831 /*
 832  *      Mark this cpu idle, and remove it from the active set,
 833  *      since it is not actively using any pmap.  Signal_cpus
 834  *      will notice that it is idle, and avoid signaling it,
 835  *      but will queue the update request for when the cpu
 836  *      becomes active.
 837  */
 838 #if   defined(__x86_64__)
 839 #define MARK_CPU_IDLE(my_cpu)   {                                       \
 840         int     s = splhigh();                                          \
 841         CPU_CR3_MARK_INACTIVE();                                        \
 842         __asm__ volatile("mfence");                                     \
 843         splx(s);                                                        \
 844 }
 845 #else /* __i386__ native */
 846 #define MARK_CPU_IDLE(my_cpu)   {                                       \
 847         /*                                                              \
 848          *      Mark this cpu idle, and remove it from the active set,  \
 849          *      since it is not actively using any pmap.  Signal_cpus   \
 850          *      will notice that it is idle, and avoid signaling it,    \
 851          *      but will queue the update request for when the cpu      \
 852          *      becomes active.                                         \
 853          */                                                             \
 854         int     s = splhigh();                                          \
 855         if (!cpu_mode_is64bit() || no_shared_cr3)                       \
 856                 process_pmap_updates();                                 \
 857         else                                                            \
 858                 pmap_load_kernel_cr3();                                 \
 859         CPU_CR3_MARK_INACTIVE();                                        \
 860         __asm__ volatile("mfence");                                     \
 861         splx(s);                                                        \
 862 }
 863 #endif /* __i386__ */
 864
 865 #define MARK_CPU_ACTIVE(my_cpu) {                                       \
 866                                                                         \
 867         int     s = splhigh();                                          \
 868         /*                                                              \
 869          *      If a kernel_pmap update was requested while this cpu    \
 870          *      was idle, process it as if we got the interrupt.        \
 871          *      Before doing so, remove this cpu from the idle set.     \
 872          *      Since we do not grab any pmap locks while we flush      \
 873          *      our TLB, another cpu may start an update operation      \
 874          *      before we finish.  Removing this cpu from the idle      \
 875          *      set assures that we will receive another update         \
 876          *      interrupt if this happens.                              \
 877          */                                                             \
 878         CPU_CR3_MARK_ACTIVE();                                          \
 879         __asm__ volatile("mfence");                                     \
 880                                                                         \
 881         if (current_cpu_datap()->cpu_tlb_invalid)                       \
 882             process_pmap_updates();                                     \
 883         splx(s);                                                        \
 884 }
 885
 886 #define PMAP_CONTEXT(pmap, thread)
 887
 888 #define pmap_kernel_va(VA)      \
 889         ((((vm_offset_t) (VA)) >= vm_min_kernel_address) &&     \
 890          (((vm_offset_t) (VA)) <= vm_max_kernel_address))
 891
 892
 893 #define pmap_resident_count(pmap)       ((pmap)->stats.resident_count)
 894 #define pmap_resident_max(pmap)         ((pmap)->stats.resident_max)
 895 #define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr)
 896 #define pmap_attribute(pmap,addr,size,attr,value) \
 897                                         (KERN_INVALID_ADDRESS)
 898 #define pmap_attribute_cache_sync(addr,size,attr,value) \
 899                                         (KERN_INVALID_ADDRESS)
 900
 901 #define MACHINE_PMAP_IS_EMPTY 1
 902 extern boolean_t pmap_is_empty(pmap_t           pmap,
 903                                vm_map_offset_t  start,
 904                                vm_map_offset_t  end);
 905
 906
 907 #endif  /* ASSEMBLER */
 908
 909
 910 #endif  /* _PMAP_MACHINE_ */
 911
 912
 913 #endif  /* KERNEL_PRIVATE */