osfmk/i386/pmap.h

   1
   2 /*
   3  * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
   4  *
   5  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License. The rights granted to you under the License
  11  * may not be used to create, or enable the creation or redistribution of,
  12  * unlawful or unlicensed copies of an Apple operating system, or to
  13  * circumvent, violate, or enable the circumvention or violation of, any
  14  * terms of an Apple operating system software license agreement.
  15  *
  16  * Please obtain a copy of the License at
  17  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  18  *
  19  * The Original Code and all software distributed under the License are
  20  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  21  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  22  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  23  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  24  * Please see the License for the specific language governing rights and
  25  * limitations under the License.
  26  *
  27  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  28  */
  29 /*
  30  * @OSF_COPYRIGHT@
  31  */
  32 /*
  33  * Mach Operating System
  34  * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
  35  * All Rights Reserved.
  36  *
  37  * Permission to use, copy, modify and distribute this software and its
  38  * documentation is hereby granted, provided that both the copyright
  39  * notice and this permission notice appear in all copies of the
  40  * software, derivative works or modified versions, and any portions
  41  * thereof, and that both notices appear in supporting documentation.
  42  *
  43  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  44  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  45  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  46  *
  47  * Carnegie Mellon requests users of this software to return to
  48  *
  49  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  50  *  School of Computer Science
  51  *  Carnegie Mellon University
  52  *  Pittsburgh PA 15213-3890
  53  *
  54  * any improvements or extensions that they make and grant Carnegie Mellon
  55  * the rights to redistribute these changes.
  56  */
  57 /*
  58  */
  59
  60 /*
  61  *      File:   pmap.h
  62  *
  63  *      Authors:  Avadis Tevanian, Jr., Michael Wayne Young
  64  *      Date:   1985
  65  *
  66  *      Machine-dependent structures for the physical map module.
  67  */
  68 #ifdef KERNEL_PRIVATE
  69 #ifndef _PMAP_MACHINE_
  70 #define _PMAP_MACHINE_  1
  71
  72 #ifndef ASSEMBLER
  73
  74 #include <platforms.h>
  75
  76 #include <mach/kern_return.h>
  77 #include <mach/machine/vm_types.h>
  78 #include <mach/vm_prot.h>
  79 #include <mach/vm_statistics.h>
  80 #include <mach/machine/vm_param.h>
  81 #include <kern/kern_types.h>
  82 #include <kern/thread.h>
  83 #include <kern/lock.h>
  84
  85 #include <i386/mp.h>
  86 #include <i386/proc_reg.h>
  87
  88 /*
  89  *      Define the generic in terms of the specific
  90  */
  91
  92 #define INTEL_PGBYTES           I386_PGBYTES
  93 #define INTEL_PGSHIFT           I386_PGSHIFT
  94 #define intel_btop(x)           i386_btop(x)
  95 #define intel_ptob(x)           i386_ptob(x)
  96 #define intel_round_page(x)     i386_round_page(x)
  97 #define intel_trunc_page(x)     i386_trunc_page(x)
  98 #define trunc_intel_to_vm(x)    trunc_i386_to_vm(x)
  99 #define round_intel_to_vm(x)    round_i386_to_vm(x)
 100 #define vm_to_intel(x)          vm_to_i386(x)
 101
 102 /*
 103  *      i386/i486/i860 Page Table Entry
 104  */
 105
 106 #endif  /* ASSEMBLER */
 107
 108 #define NPGPTD          4
 109 #define PDESHIFT        21
 110 #define PTEMASK         0x1ff
 111 #define PTEINDX         3
 112
 113 #define PTESHIFT        12
 114
 115 #define PDESIZE         sizeof(pd_entry_t) /* for assembly files */
 116 #define PTESIZE         sizeof(pt_entry_t) /* for assembly files */
 117
 118 #define INTEL_OFFMASK   (I386_PGBYTES - 1)
 119 #define PG_FRAME        0x000FFFFFFFFFF000ULL
 120 #define NPTEPG          (PAGE_SIZE/(sizeof (pt_entry_t)))
 121 #define NPTDPG          (PAGE_SIZE/(sizeof (pd_entry_t)))
 122
 123 #define NBPTD           (NPGPTD << PAGE_SHIFT)
 124 #define NPDEPTD         (NBPTD / (sizeof (pd_entry_t)))
 125 #define NPDEPG          (PAGE_SIZE/(sizeof (pd_entry_t)))
 126 #define NBPDE           (1 << PDESHIFT)
 127 #define PDEMASK         (NBPDE - 1)
 128
 129  /* cleanly define parameters for all the page table levels */
 130 typedef uint64_t        pml4_entry_t;
 131 #define NPML4PG         (PAGE_SIZE/(sizeof (pml4_entry_t)))
 132 #define PML4SHIFT       39
 133 #define PML4PGSHIFT     9
 134 #define NBPML4          (1ULL << PML4SHIFT)
 135 #define PML4MASK        (NBPML4-1)
 136 #define PML4_ENTRY_NULL ((pml4_entry_t *) 0)
 137
 138 typedef uint64_t        pdpt_entry_t;
 139 #define NPDPTPG         (PAGE_SIZE/(sizeof (pdpt_entry_t)))
 140 #define PDPTSHIFT       30
 141 #define PDPTPGSHIFT     9
 142 #define NBPDPT          (1 << PDPTSHIFT)
 143 #define PDPTMASK        (NBPDPT-1)
 144 #define PDPT_ENTRY_NULL ((pdpt_entry_t *) 0)
 145
 146 typedef uint64_t        pd_entry_t;
 147 #define NPDPG           (PAGE_SIZE/(sizeof (pd_entry_t)))
 148 #define PDSHIFT         21
 149 #define PDPGSHIFT       9
 150 #define NBPD            (1 << PDSHIFT)
 151 #define PDMASK          (NBPD-1)
 152 #define PD_ENTRY_NULL   ((pd_entry_t *) 0)
 153
 154 typedef uint64_t        pt_entry_t;
 155 #define NPTPG           (PAGE_SIZE/(sizeof (pt_entry_t)))
 156 #define PTSHIFT         12
 157 #define PTPGSHIFT       9
 158 #define NBPT            (1 << PTSHIFT)
 159 #define PTMASK          (NBPT-1)
 160 #define PT_ENTRY_NULL   ((pt_entry_t *) 0)
 161
 162 typedef uint64_t  pmap_paddr_t;
 163
 164 /*
 165  * Atomic 64-bit store of a page table entry.
 166  */
 167 static inline void
 168 pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
 169 {
 170         /*
 171          * Load the new value into %ecx:%ebx
 172          * Load the old value into %edx:%eax
 173          * Compare-exchange-8bytes at address entryp (loaded in %edi)
 174          * If the compare succeeds, the new value will have been stored.
 175          * Otherwise, the old value changed and reloaded, so try again.
 176          */
 177         asm volatile(
 178                 "       movl    (%0), %%eax     \n\t"
 179                 "       movl    4(%0), %%edx    \n\t"
 180                 "1:                             \n\t"
 181                 "       cmpxchg8b (%0)          \n\t"
 182                 "       jnz 1b"
 183                 :
 184                 : "D" (entryp),
 185                   "b" ((uint32_t)value),
 186                   "c" ((uint32_t)(value >> 32))
 187                 : "eax", "edx", "memory");
 188 }
 189
 190 /* in 64 bit spaces, the number of each type of page in the page tables */
 191 #define NPML4PGS        (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
 192 #define NPDPTPGS        (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
 193 #define NPDEPGS         (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t))))
 194 #define NPTEPGS         (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
 195
 196 /*
 197  * The 64-bit kernel is remapped in uber-space which is at the base
 198  * the highest 4th-level directory (KERNEL_UBER_PML4_INDEX). That is,
 199  * 512GB from the top of virtual space (or zero).
 200  */
 201 #define KERNEL_UBER_PML4_INDEX  511
 202 #define KERNEL_UBER_BASE        (0ULL - NBPML4)
 203 #define KERNEL_UBER_BASE_HI32   ((uint32_t)(KERNEL_UBER_BASE >> 32))
 204
 205 #define VM_WIMG_COPYBACK        VM_MEM_COHERENT
 206 #define VM_WIMG_DEFAULT         VM_MEM_COHERENT
 207 /* ?? intel ?? */
 208 #define VM_WIMG_IO              (VM_MEM_COHERENT |      \
 209                                 VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED)
 210 #define VM_WIMG_WTHRU           (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED)
 211 /* write combining mode, aka store gather */
 212 #define VM_WIMG_WCOMB           (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT)
 213
 214 /*
 215  * Pte related macros
 216  */
 217 #define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDESHIFT)|((pti)<<PTESHIFT)))
 218 #define VADDR64(pmi, pdi, pti) ((vm_offset_t)(((pmi)<<PLM4SHIFT))((pdi)<<PDESHIFT)|((pti)<<PTESHIFT))
 219
 220 /*
 221  * Size of Kernel address space.  This is the number of page table pages
 222  * (4MB each) to use for the kernel.  256 pages == 1 Gigabyte.
 223  * This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc).
 224  */
 225 #ifndef KVA_PAGES
 226 #define KVA_PAGES       1024
 227 #endif
 228
 229 #ifndef NKPT
 230 #define NKPT            500     /* actual number of kernel page tables */
 231 #endif
 232 #ifndef NKPDE
 233 #define NKPDE   (KVA_PAGES - 1) /* addressable number of page tables/pde's */
 234 #endif
 235
 236
 237 enum high_cpu_types {
 238   HIGH_CPU_ISS0,
 239   HIGH_CPU_ISS1,
 240   HIGH_CPU_DESC,
 241   HIGH_CPU_LDT_BEGIN,
 242   HIGH_CPU_LDT_END = HIGH_CPU_LDT_BEGIN + (LDTSZ / 512) - 1,
 243   HIGH_CPU_END
 244 };
 245
 246 enum  high_fixed_addresses {
 247   HIGH_FIXED_TRAMPS,  /* must be first */
 248   HIGH_FIXED_TRAMPS_END,
 249   HIGH_FIXED_GDT,
 250   HIGH_FIXED_IDT,
 251   HIGH_FIXED_LDT_BEGIN,
 252   HIGH_FIXED_LDT_END = HIGH_FIXED_LDT_BEGIN + (LDTSZ / 512) - 1,
 253   HIGH_FIXED_KTSS,
 254   HIGH_FIXED_DFTSS,
 255   HIGH_FIXED_DBTSS,
 256   HIGH_FIXED_CPUS_BEGIN,
 257   HIGH_FIXED_CPUS_END = HIGH_FIXED_CPUS_BEGIN + (HIGH_CPU_END * MAX_CPUS) - 1,
 258 };
 259
 260
 261 /* XXX64  below PTDI values need cleanup */
 262 /*
 263  * The *PTDI values control the layout of virtual memory
 264  *
 265  */
 266 #define        KPTDI           (0x000)/* start of kernel virtual pde's */
 267 #define        PTDPTDI         (0x7F4) /* ptd entry that points to ptd! */
 268 #define        APTDPTDI        (0x7F8) /* alt ptd entry that points to APTD */
 269 #define        UMAXPTDI        (0x7F8) /* ptd entry for user space end */
 270 #define UMAXPTEOFF      (NPTEPG)        /* pte entry for user space end */
 271
 272 #define KERNBASE       VADDR(KPTDI,0)
 273
 274 /*
 275  *      Convert address offset to directory address
 276  *      containing the page table pointer - legacy
 277  */
 278 /*#define pmap_pde(m,v) (&((m)->dirbase[(vm_offset_t)(v) >> PDESHIFT]))*/
 279
 280 #define HIGH_MEM_BASE  ((uint32_t)( -NBPDE) )  /* shared gdt etc seg addr */ /* XXX64 ?? */
 281 #define pmap_index_to_virt(x)  (HIGH_MEM_BASE | ((unsigned)(x) << PAGE_SHIFT))
 282
 283 /*
 284  *      Convert address offset to page descriptor index
 285  */
 286 #define pdenum(pmap, a) (((vm_offset_t)(a) >> PDESHIFT) & PDEMASK)
 287
 288 #define pdeidx(pmap, a)    (((a) >> PDSHIFT)   & ((1ULL<<(48 - PDSHIFT)) -1))
 289 #define pdptidx(pmap, a)   (((a) >> PDPTSHIFT) & ((1ULL<<(48 - PDPTSHIFT)) -1))
 290 #define pml4idx(pmap, a)   (((a) >> PML4SHIFT) & ((1ULL<<(48 - PML4SHIFT)) -1))
 291
 292 /*
 293  *      Convert page descriptor index to user virtual address
 294  */
 295 #define pdetova(a)      ((vm_offset_t)(a) << PDESHIFT)
 296
 297 /*
 298  *      Convert address offset to page table index
 299  */
 300 #define ptenum(a)       (((vm_offset_t)(a) >> PTESHIFT) & PTEMASK)
 301
 302 /*
 303  *      Hardware pte bit definitions (to be used directly on the ptes
 304  *      without using the bit fields).
 305  */
 306
 307 #define INTEL_PTE_VALID         0x00000001
 308 #define INTEL_PTE_WRITE         0x00000002
 309 #define INTEL_PTE_RW            0x00000002
 310 #define INTEL_PTE_USER          0x00000004
 311 #define INTEL_PTE_WTHRU         0x00000008
 312 #define INTEL_PTE_NCACHE        0x00000010
 313 #define INTEL_PTE_REF           0x00000020
 314 #define INTEL_PTE_MOD           0x00000040
 315 #define INTEL_PTE_PS            0x00000080
 316 #define INTEL_PTE_GLOBAL        0x00000100
 317 #define INTEL_PTE_WIRED         0x00000200
 318 #define INTEL_PTE_PFN           PG_FRAME
 319 #define INTEL_PTE_PTA           0x00000080
 320
 321 #define INTEL_PTE_NX            (1ULL << 63)
 322
 323 #define INTEL_PTE_INVALID       0
 324
 325 #define pa_to_pte(a)            ((a) & INTEL_PTE_PFN) /* XXX */
 326 #define pte_to_pa(p)            ((p) & INTEL_PTE_PFN) /* XXX */
 327 #define pte_increment_pa(p)     ((p) += INTEL_OFFMASK+1)
 328
 329 #define pte_kernel_rw(p)          ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_RW))
 330 #define pte_kernel_ro(p)          ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID))
 331 #define pte_user_rw(p)            ((pt_entry)t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER|INTEL_PTE_RW))
 332 #define pte_user_ro(p)            ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER))
 333
 334 #define PMAP_DEFAULT_CACHE      0
 335 #define PMAP_INHIBIT_CACHE      1
 336 #define PMAP_GUARDED_CACHE      2
 337 #define PMAP_ACTIVATE_CACHE     4
 338 #define PMAP_NO_GUARD_CACHE     8
 339
 340
 341 #ifndef ASSEMBLER
 342
 343 #include <sys/queue.h>
 344
 345 /*
 346  * Address of current and alternate address space page table maps
 347  * and directories.
 348  */
 349
 350 extern pt_entry_t PTmap[], APTmap[], Upte;
 351 extern pd_entry_t PTD[], APTD[], PTDpde[], APTDpde[], Upde;
 352
 353 extern pd_entry_t *IdlePTD;     /* physical address of "Idle" state directory */
 354 extern pdpt_entry_t *IdlePDPT;
 355
 356 extern pmap_paddr_t lo_kernel_cr3;
 357
 358 extern pml4_entry_t *IdlePML4;
 359 extern pdpt_entry_t *IdlePDPT64;
 360 extern addr64_t     kernel64_cr3;
 361 extern boolean_t    no_shared_cr3;
 362
 363 /*
 364  * virtual address to page table entry and
 365  * to physical address. Likewise for alternate address space.
 366  * Note: these work recursively, thus vtopte of a pte will give
 367  * the corresponding pde that in turn maps it.
 368  */
 369 #define vtopte(va)      (PTmap + i386_btop((vm_offset_t)va))
 370
 371
 372 typedef volatile long   cpu_set;        /* set of CPUs - must be <= 32 */
 373                                         /* changed by other processors */
 374 struct md_page {
 375   int pv_list_count;
 376   TAILQ_HEAD(,pv_entry)  pv_list;
 377 };
 378
 379 #include <vm/vm_page.h>
 380
 381 /*
 382  *      For each vm_page_t, there is a list of all currently
 383  *      valid virtual mappings of that page.  An entry is
 384  *      a pv_entry_t; the list is the pv_table.
 385  */
 386
 387 struct pmap {
 388         pd_entry_t      *dirbase;        /* page directory pointer */
 389         pmap_paddr_t    pdirbase;        /* phys. address of dirbase */
 390         vm_object_t     pm_obj;         /* object to hold pde's */
 391         int             ref_count;      /* reference count */
 392         int             nx_enabled;
 393         boolean_t       pm_64bit;
 394         boolean_t       pm_kernel_cr3;
 395         boolean_t       pm_shared;
 396         decl_simple_lock_data(,lock)    /* lock on map */
 397         struct pmap_statistics  stats;  /* map statistics */
 398         vm_offset_t     pm_hold;        /* true pdpt zalloc addr */
 399         pmap_paddr_t    pm_cr3;         /* physical addr */
 400         pdpt_entry_t    *pm_pdpt;       /* KVA of 3rd level page */
 401         pml4_entry_t    *pm_pml4;       /* VKA of top level */
 402         vm_object_t     pm_obj_pdpt;    /* holds pdpt pages */
 403         vm_object_t     pm_obj_pml4;    /* holds pml4 pages */
 404         vm_object_t     pm_obj_top;     /* holds single top level page */
 405 };
 406
 407
 408 #define PMAP_PDPT_FIRST_WINDOW 0
 409 #define PMAP_PDPT_NWINDOWS 4
 410 #define PMAP_PDE_FIRST_WINDOW (PMAP_PDPT_NWINDOWS)
 411 #define PMAP_PDE_NWINDOWS 4
 412 #define PMAP_PTE_FIRST_WINDOW (PMAP_PDE_FIRST_WINDOW + PMAP_PDE_NWINDOWS)
 413 #define PMAP_PTE_NWINDOWS 4
 414
 415 #define PMAP_NWINDOWS_FIRSTFREE (PMAP_PTE_FIRST_WINDOW + PMAP_PTE_NWINDOWS)
 416 #define PMAP_WINDOW_SIZE 8
 417 #define PMAP_NWINDOWS (PMAP_NWINDOWS_FIRSTFREE + PMAP_WINDOW_SIZE)
 418
 419 typedef struct {
 420         pt_entry_t      *prv_CMAP;
 421         caddr_t         prv_CADDR;
 422 } mapwindow_t;
 423
 424 typedef struct cpu_pmap {
 425         int                     pdpt_window_index;
 426         int                     pde_window_index;
 427         int                     pte_window_index;
 428         mapwindow_t             mapwindow[PMAP_NWINDOWS];
 429 } cpu_pmap_t;
 430
 431
 432 extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry);
 433
 434 typedef struct pmap_memory_regions {
 435   ppnum_t base;
 436   ppnum_t end;
 437   ppnum_t alloc;
 438   uint32_t type;
 439 } pmap_memory_region_t;
 440
 441 unsigned pmap_memory_region_count;
 442 unsigned pmap_memory_region_current;
 443
 444 #define PMAP_MEMORY_REGIONS_SIZE 128
 445
 446 extern pmap_memory_region_t pmap_memory_regions[];
 447
 448 static inline void set_dirbase(pmap_t tpmap, __unused int tcpu) {
 449         current_cpu_datap()->cpu_task_cr3 = (pmap_paddr_t)((tpmap)->pm_cr3);
 450         current_cpu_datap()->cpu_task_map = tpmap->pm_64bit ? TASK_MAP_64BIT : TASK_MAP_32BIT;
 451 }
 452
 453 /*
 454  *      External declarations for PMAP_ACTIVATE.
 455  */
 456
 457 extern void             process_pmap_updates(void);
 458 extern void             pmap_update_interrupt(void);
 459
 460 /*
 461  *      Machine dependent routines that are used only for i386/i486/i860.
 462  */
 463
 464 extern addr64_t         (kvtophys)(
 465                                 vm_offset_t     addr);
 466
 467 extern pt_entry_t       *pmap_pte(
 468                                 struct pmap     *pmap,
 469                                 vm_map_offset_t addr);
 470
 471 extern pd_entry_t       *pmap_pde(
 472                                 struct pmap     *pmap,
 473                                 vm_map_offset_t addr);
 474
 475 extern pd_entry_t       *pmap64_pde(
 476                                 struct pmap     *pmap,
 477                                 vm_map_offset_t addr);
 478
 479 extern pdpt_entry_t     *pmap64_pdpt(
 480                                 struct pmap     *pmap,
 481                                 vm_map_offset_t addr);
 482
 483 extern vm_offset_t      pmap_map(
 484                                 vm_offset_t     virt,
 485                                 vm_map_offset_t start,
 486                                 vm_map_offset_t end,
 487                                 vm_prot_t       prot,
 488                                 unsigned int    flags);
 489
 490 extern vm_offset_t      pmap_map_bd(
 491                                 vm_offset_t     virt,
 492                                 vm_map_offset_t start,
 493                                 vm_map_offset_t end,
 494                                 vm_prot_t       prot,
 495                                 unsigned int    flags);
 496
 497 extern void             pmap_bootstrap(
 498                                 vm_offset_t     load_start,
 499                                 boolean_t       IA32e);
 500
 501 extern boolean_t        pmap_valid_page(
 502                                 ppnum_t pn);
 503
 504 extern int              pmap_list_resident_pages(
 505                                 struct pmap     *pmap,
 506                                 vm_offset_t     *listp,
 507                                 int             space);
 508
 509 extern void             pmap_commpage32_init(
 510                                            vm_offset_t kernel,
 511                                            vm_offset_t user,
 512                                            int count);
 513 extern void             pmap_commpage64_init(
 514                                            vm_offset_t  kernel,
 515                                            vm_map_offset_t user,
 516                                            int count);
 517
 518 extern struct cpu_pmap  *pmap_cpu_alloc(
 519                                 boolean_t       is_boot_cpu);
 520 extern void             pmap_cpu_free(
 521                                 struct cpu_pmap *cp);
 522
 523 extern void             pmap_map_block(
 524                                 pmap_t pmap,
 525                                 addr64_t va,
 526                                 ppnum_t pa,
 527                                 uint32_t size,
 528                                 vm_prot_t prot,
 529                                 int attr,
 530                                 unsigned int flags);
 531
 532 extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
 533 extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
 534 extern ppnum_t          pmap_find_phys(pmap_t map, addr64_t va);
 535 extern void pmap_sync_page_data_phys(ppnum_t pa);
 536 extern void pmap_sync_page_attributes_phys(ppnum_t pa);
 537
 538 extern kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t nstart, uint64_t size);
 539 extern kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr);
 540 extern void pmap_map_sharedpage(task_t task, pmap_t pmap);
 541 extern void pmap_unmap_sharedpage(pmap_t pmap);
 542 extern void pmap_disable_NX(pmap_t pmap);
 543 extern void pmap_set_4GB_pagezero(pmap_t pmap);
 544 extern void pmap_clear_4GB_pagezero(pmap_t pmap);
 545 extern void pmap_load_kernel_cr3(void);
 546 extern vm_offset_t pmap_cpu_high_map_vaddr(int, enum high_cpu_types);
 547 extern vm_offset_t pmap_high_map_vaddr(enum high_cpu_types);
 548 extern vm_offset_t pmap_high_map(pt_entry_t, enum high_cpu_types);
 549 extern vm_offset_t pmap_cpu_high_shared_remap(int, enum high_cpu_types, vm_offset_t, int);
 550 extern vm_offset_t pmap_high_shared_remap(enum high_fixed_addresses, vm_offset_t, int);
 551
 552 extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, int *, int *);
 553
 554
 555
 556 /*
 557  *      Macros for speed.
 558  */
 559
 560
 561 #include <kern/spl.h>
 562
 563 #if defined(PMAP_ACTIVATE_KERNEL)
 564 #undef PMAP_ACTIVATE_KERNEL
 565 #undef PMAP_DEACTIVATE_KERNEL
 566 #undef PMAP_ACTIVATE_USER
 567 #undef PMAP_DEACTIVATE_USER
 568 #endif
 569
 570
 571 #define PMAP_ACTIVATE_KERNEL(my_cpu)  {                                 \
 572         spl_t           spl;                                            \
 573                                                                         \
 574         spl = splhigh();                                                \
 575         if (current_cpu_datap()->cpu_tlb_invalid)                       \
 576                 process_pmap_updates();                                 \
 577         splx(spl);                                                      \
 578 }
 579
 580 #define PMAP_DEACTIVATE_KERNEL(my_cpu) {                                \
 581         spl_t           spl;                                            \
 582                                                                         \
 583         spl = splhigh();                                                \
 584         process_pmap_updates();                                         \
 585         splx(spl);                                                      \
 586 }
 587
 588
 589 #define PMAP_ACTIVATE_MAP(map, my_cpu)  {                               \
 590         register pmap_t         tpmap;                                  \
 591                                                                         \
 592         tpmap = vm_map_pmap(map);                                       \
 593         set_dirbase(tpmap, my_cpu);                                     \
 594 }
 595
 596 #define PMAP_DEACTIVATE_MAP(map, my_cpu)                                \
 597         if (current_cpu_datap()->cpu_task_map == TASK_MAP_64BIT_SHARED) \
 598             pmap_load_kernel_cr3();
 599
 600 #define PMAP_ACTIVATE_USER(th, my_cpu)  {                               \
 601         spl_t           spl;                                            \
 602                                                                         \
 603         spl = splhigh();                                                \
 604         PMAP_ACTIVATE_MAP(th->map, my_cpu)                              \
 605         splx(spl);                                                      \
 606 }
 607
 608 #define PMAP_DEACTIVATE_USER(th, my_cpu)
 609
 610
 611 #define PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) {                   \
 612         spl_t           spl;                                            \
 613         pt_entry_t      *kpdp;                                          \
 614         pt_entry_t      *updp;                                          \
 615         int             i;                                              \
 616         int             need_flush;                                     \
 617                                                                         \
 618         need_flush = 0;                                                 \
 619         spl = splhigh();                                                \
 620         if (old_th->map != new_th->map) {                               \
 621                 PMAP_DEACTIVATE_MAP(old_th->map, my_cpu);               \
 622                 PMAP_ACTIVATE_MAP(new_th->map, my_cpu);                 \
 623         }                                                               \
 624         kpdp = current_cpu_datap()->cpu_copywindow_pdp;                 \
 625         for (i = 0; i < NCOPY_WINDOWS; i++) {                           \
 626                 if (new_th->machine.copy_window[i].user_base != (user_addr_t)-1) {      \
 627                         updp = pmap_pde(new_th->map->pmap,              \
 628                               new_th->machine.copy_window[i].user_base);\
 629                         pmap_store_pte(kpdp,  updp ? *updp : 0);        \
 630                 }                                                       \
 631                 kpdp++;                                                 \
 632         }                                                               \
 633         splx(spl);                                                      \
 634         if (new_th->machine.copyio_state == WINDOWS_OPENED)             \
 635                 need_flush = 1;                                         \
 636         else                                                            \
 637                 new_th->machine.copyio_state = WINDOWS_DIRTY;           \
 638         if (new_th->machine.physwindow_pte) {                           \
 639                 pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep),      \
 640                                  new_th->machine.physwindow_pte);       \
 641                 if (need_flush == 0)                                    \
 642                         invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);\
 643         }                                                               \
 644         if (need_flush)                                                 \
 645                 flush_tlb();                                            \
 646 }
 647
 648 #define PMAP_SWITCH_USER(th, new_map, my_cpu) {                         \
 649         spl_t           spl;                                            \
 650                                                                         \
 651         spl = splhigh();                                                \
 652         PMAP_DEACTIVATE_MAP(th->map, my_cpu);                           \
 653         th->map = new_map;                                              \
 654         PMAP_ACTIVATE_MAP(th->map, my_cpu);                             \
 655         splx(spl);                                                      \
 656         inval_copy_windows(th);                                         \
 657 }
 658
 659 /*
 660  * Marking the current cpu's cr3 inactive is achieved by setting its lsb.
 661  * Marking the current cpu's cr3 active once more involves clearng this bit.
 662  * Note that valid page tables are page-aligned and so the bottom 12 bits
 663  * are noramlly zero.
 664  * We can only mark the current cpu active/inactive but we can test any cpu.
 665  */
 666 #define CPU_CR3_MARK_INACTIVE()                                         \
 667         current_cpu_datap()->cpu_active_cr3 |= 1
 668
 669 #define CPU_CR3_MARK_ACTIVE()                                           \
 670         current_cpu_datap()->cpu_active_cr3 &= ~1
 671
 672 #define CPU_CR3_IS_ACTIVE(cpu)                                          \
 673         ((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0)
 674
 675 #define CPU_GET_ACTIVE_CR3(cpu)                                         \
 676         (cpu_datap(cpu)->cpu_active_cr3 & ~1)
 677
 678 #define MARK_CPU_IDLE(my_cpu)   {                                       \
 679         /*                                                              \
 680          *      Mark this cpu idle, and remove it from the active set,  \
 681          *      since it is not actively using any pmap.  Signal_cpus   \
 682          *      will notice that it is idle, and avoid signaling it,    \
 683          *      but will queue the update request for when the cpu      \
 684          *      becomes active.                                         \
 685          */                                                             \
 686         int     s = splhigh();                                          \
 687         if (!cpu_mode_is64bit() || no_shared_cr3)                       \
 688                 process_pmap_updates();                                 \
 689         else                                                            \
 690                 pmap_load_kernel_cr3();                                 \
 691         CPU_CR3_MARK_INACTIVE();                                        \
 692         __asm__ volatile("mfence");                                     \
 693         splx(s);                                                        \
 694 }
 695
 696 #define MARK_CPU_ACTIVE(my_cpu) {                                       \
 697                                                                         \
 698         int     s = splhigh();                                          \
 699         /*                                                              \
 700          *      If a kernel_pmap update was requested while this cpu    \
 701          *      was idle, process it as if we got the interrupt.        \
 702          *      Before doing so, remove this cpu from the idle set.     \
 703          *      Since we do not grab any pmap locks while we flush      \
 704          *      our TLB, another cpu may start an update operation      \
 705          *      before we finish.  Removing this cpu from the idle      \
 706          *      set assures that we will receive another update         \
 707          *      interrupt if this happens.                              \
 708          */                                                             \
 709         CPU_CR3_MARK_ACTIVE();                                          \
 710         __asm__ volatile("mfence");                                     \
 711                                                                         \
 712         if (current_cpu_datap()->cpu_tlb_invalid)                       \
 713             process_pmap_updates();                                     \
 714         splx(s);                                                        \
 715 }
 716
 717 #define PMAP_CONTEXT(pmap, thread)
 718
 719 #define pmap_kernel_va(VA)      \
 720         ((((vm_offset_t) (VA)) >= vm_min_kernel_address) &&     \
 721          (((vm_offset_t) (VA)) <= vm_max_kernel_address))
 722
 723
 724 #define pmap_resident_count(pmap)       ((pmap)->stats.resident_count)
 725 #define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr)
 726 #define pmap_attribute(pmap,addr,size,attr,value) \
 727                                         (KERN_INVALID_ADDRESS)
 728 #define pmap_attribute_cache_sync(addr,size,attr,value) \
 729                                         (KERN_INVALID_ADDRESS)
 730
 731 #endif  /* ASSEMBLER */
 732
 733
 734 #endif  /* _PMAP_MACHINE_ */
 735
 736
 737 #endif  /* KERNEL_PRIVATE */