X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/55e303ae13a4cf49d70f2294092726f2fffb9ef2..4452a7af2eac33dbad800bcc91f2399d62c18f53:/osfmk/ppc/mappings.c?ds=sidebyside diff --git a/osfmk/ppc/mappings.c b/osfmk/ppc/mappings.c index 89672e4da..add0e0eb4 100644 --- a/osfmk/ppc/mappings.c +++ b/osfmk/ppc/mappings.c @@ -1,16 +1,19 @@ /* - * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ - * - * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved. + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER @@ -20,7 +23,7 @@ * Please see the License for the specific language governing rights and * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * This file is used to maintain the virtual to real mappings for a PowerPC machine. @@ -33,31 +36,31 @@ * */ -#include #include #include #include #include -#include -#include +#include #include #include + +#include +#include +#include +#include + #include #include #include #include -#include +#include -#include #include #include #include - -#include #include #include - #include #include #include @@ -77,21 +80,37 @@ int ppc_max_adrsp; /* Maximum address spaces */ addr64_t *mapdebug; /* (BRINGUP) */ extern unsigned int DebugWork; /* (BRINGUP) */ -extern unsigned int hash_table_size; - void mapping_verify(void); void mapping_phys_unused(ppnum_t pa); +int nx_enabled = 0; /* enable no-execute protection */ + /* - * ppc_prot translates from the mach representation of protections to the PPC version. - * We also allow for a direct setting of the protection bits. This extends the mach - * concepts to allow the greater control we need for Virtual Machines (VMM). - * Calculation of it like this saves a memory reference - and maybe a couple of microseconds. - * It eliminates the used of this table. - * unsigned char ppc_prot[16] = { 0, 3, 2, 2, 3, 3, 2, 2, 0, 1, 2, 3, 0, 1, 2, 3 }; + * ppc_prot translates Mach's representation of protections to that of the PPC hardware. + * For Virtual Machines (VMM), we also provide translation entries where the output is + * the same as the input, allowing direct specification of PPC protections. Mach's + * representations are always in the range 0..7, so they always fall into the first + * 8 table entries; direct translations are placed in the range 8..16, so they fall into + * the second half of the table. + * */ + +unsigned char ppc_prot[16] = { 4, 7, 6, 6, 3, 3, 2, 2, /* Mach -> PPC translations */ + 0, 1, 2, 3, 4, 5, 6, 7 }; /* VMM direct translations */ + + + +vm_prot_t getProtPPC(int key, boolean_t disable_NX) { + vm_prot_t prot; + + prot = ppc_prot[key & 0xF]; + + if (key <= 7 && disable_NX == TRUE) + prot &= ~mpN; + + return (prot); +} -#define ppc_prot(p) ((0xE4E4AFAC >> (p << 1)) & 3) /* * About PPC VSID generation: @@ -158,15 +177,15 @@ void mapping_init(void) { ppc_max_adrsp = maxAdrSp; /* Set maximum address spaces */ maxeff = 32; /* Assume 32-bit */ - if(per_proc_info[0].pf.Available & pf64Bit) maxeff = 64; /* Is this a 64-bit machine? */ + if(PerProcTable[0].ppe_vaddr->pf.Available & pf64Bit) maxeff = 64; /* Is this a 64-bit machine? */ - rwidth = per_proc_info[0].pf.pfMaxVAddr - maxAdrSpb; /* Reduce address width by width of address space ID */ + rwidth = PerProcTable[0].ppe_vaddr->pf.pfMaxVAddr - maxAdrSpb; /* Reduce address width by width of address space ID */ if(rwidth > maxeff) rwidth = maxeff; /* If we still have more virtual than effective, clamp at effective */ vm_max_address = 0xFFFFFFFFFFFFFFFFULL >> (64 - rwidth); /* Get maximum effective address supported */ - vm_max_physical = 0xFFFFFFFFFFFFFFFFULL >> (64 - per_proc_info[0].pf.pfMaxPAddr); /* Get maximum physical address supported */ + vm_max_physical = 0xFFFFFFFFFFFFFFFFULL >> (64 - PerProcTable[0].ppe_vaddr->pf.pfMaxPAddr); /* Get maximum physical address supported */ - if(per_proc_info[0].pf.Available & pf64Bit) { /* Are we 64 bit? */ + if(PerProcTable[0].ppe_vaddr->pf.Available & pf64Bit) { /* Are we 64 bit? */ tmp = 12; /* Size of hash space */ } else { @@ -200,27 +219,55 @@ void mapping_init(void) { addr64_t mapping_remove(pmap_t pmap, addr64_t va) { /* Remove a single mapping for this VADDR Returns TRUE if a mapping was found to remove */ - mapping *mp; + mapping_t *mp; addr64_t nextva; + ppnum_t pgaddr; - disable_preemption(); /* Don't change threads */ - - while(1) { /* Keep trying until we truely fail */ + va &= ~PAGE_MASK; /* Scrub noise bits */ + + do { /* Keep trying until we truely fail */ mp = hw_rem_map(pmap, va, &nextva); /* Remove a mapping from this pmap */ - if(((unsigned int)mp & mapRetCode) != mapRtRemove) break; /* If it is gone, we are done */ - } - - enable_preemption(); /* Thread change ok */ - - if(!mp) return (nextva | 1); /* Nothing found to unmap */ - - if((unsigned int)mp & mapRetCode) { /* Was there a failure? */ + } while (mapRtRemove == ((unsigned int)mp & mapRetCode)); - panic("mapping_remove: hw_rem_map failed - pmap = %08X, va = %016llX, code = %08X\n", - pmap, va, mp); + switch ((unsigned int)mp & mapRetCode) { + case mapRtOK: + break; /* Mapping removed */ + case mapRtNotFnd: + return (nextva | 1); /* Nothing found to unmap */ + default: + panic("mapping_remove: hw_rem_map failed - pmap = %08X, va = %016llX, code = %08X\n", + pmap, va, mp); + break; } + + pgaddr = mp->mpPAddr; /* Get page number from mapping */ mapping_free(mp); /* Add mapping to the free list */ + + if ((pmap->pmapFlags & pmapVMhost) && pmap->pmapVmmExt) { + /* If this is an assisted host, scrub any guest mappings */ + unsigned int idx; + phys_entry_t *physent = mapping_phys_lookup(pgaddr, &idx); + /* Get physent for our physical page */ + if (!physent) { /* No physent, could be in I/O area, so exit */ + return (nextva); + } + + do { /* Iterate 'till all guest mappings are gone */ + mp = hw_scrub_guest(physent, pmap); /* Attempt to scrub a guest mapping */ + switch ((unsigned int)mp & mapRetCode) { + case mapRtGuest: /* Found a guest mapping */ + case mapRtNotFnd: /* Mapping was there, but disappeared, must retry */ + case mapRtEmpty: /* No guest mappings left to scrub */ + break; + default: + panic("mapping_remove: hw_scrub_guest failed - physent = %08X, code = %08X\n", + physent, mp); /* Cry havoc, cry wrack, + at least we die with harness on our backs */ + break; + } + } while (mapRtEmpty != ((unsigned int)mp & mapRetCode)); + } return nextva; /* Tell them we did it */ } @@ -243,7 +290,7 @@ addr64_t mapping_remove(pmap_t pmap, addr64_t va) { /* Remove a single mapping * perm Mapping is permanent * cache inhibited Cache inhibited (used if use attribute or block set ) * guarded Guarded access (used if use attribute or block set ) - * size size of block (not used if not block) + * size size of block in pages - 1 (not used if not block) * prot VM protection bits * attr Cachability/Guardedness * @@ -262,18 +309,35 @@ addr64_t mapping_remove(pmap_t pmap, addr64_t va) { /* Remove a single mapping addr64_t mapping_make(pmap_t pmap, addr64_t va, ppnum_t pa, unsigned int flags, unsigned int size, vm_prot_t prot) { /* Make an address mapping */ - register mapping *mp; - addr64_t colladdr; - unsigned int pindex, mflags, pattr, wimg; - phys_entry *physent; - int i, nlists; - - disable_preemption(); /* Don't change threads */ + register mapping_t *mp; + addr64_t colladdr, psmask; + unsigned int pindex, mflags, pattr, wimg, rc; + phys_entry_t *physent; + int nlists, pcf; + boolean_t disable_NX = FALSE; pindex = 0; mflags = 0x01000000; /* Start building mpFlags field (busy count = 1) */ + + pcf = (flags & mmFlgPcfg) >> 24; /* Get the physical page config index */ + if(!(pPcfg[pcf].pcfFlags)) { /* Validate requested physical page configuration */ + panic("mapping_make: invalid physical page configuration request - pmap = %08X, va = %016llX, cfg = %d\n", + pmap, va, pcf); + } + + psmask = (1ULL << pPcfg[pcf].pcfPSize) - 1; /* Mask to isolate any offset into a page */ + if(va & psmask) { /* Make sure we are page aligned on virtual */ + panic("mapping_make: attempt to map unaligned vaddr - pmap = %08X, va = %016llX, cfg = %d\n", + pmap, va, pcf); + } + if(((addr64_t)pa << 12) & psmask) { /* Make sure we are page aligned on physical */ + panic("mapping_make: attempt to map unaligned paddr - pmap = %08X, pa = %016llX, cfg = %d\n", + pmap, pa, pcf); + } + mflags |= (pcf << (31-mpPcfgb)); /* Insert physical page configuration index */ + if(!(flags & mmFlgBlock)) { /* Is this a block map? */ size = 1; /* Set size to 1 page if not block */ @@ -281,11 +345,10 @@ addr64_t mapping_make(pmap_t pmap, addr64_t va, ppnum_t pa, unsigned int flags, physent = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ if(!physent) { /* Did we find the physical page? */ mflags |= mpBlock; /* Force this to a block if no physent */ - size = 1; /* Force size to 1 page */ pattr = 0; /* Assume normal, non-I/O memory */ if((pa & 0xFFF80000) == 0x00080000) pattr = mmFlgCInhib | mmFlgGuarded; /* If this page is in I/O range, set I/O attributes */ } - else pattr = ((physent->ppLink & (ppI | ppG)) >> 4); /* Get the default attributes from physent */ + else pattr = ((physent->ppLink & (ppI | ppG)) >> 60); /* Get the default attributes from physent */ if(flags & mmFlgUseAttr) pattr = flags & (mmFlgCInhib | mmFlgGuarded); /* Use requested attributes */ } @@ -293,6 +356,12 @@ addr64_t mapping_make(pmap_t pmap, addr64_t va, ppnum_t pa, unsigned int flags, pattr = flags & (mmFlgCInhib | mmFlgGuarded); /* Use requested attributes */ mflags |= mpBlock; /* Show that this is a block */ + + if(size > pmapSmallBlock) { /* Is it one? */ + if(size & 0x00001FFF) return mapRtBadSz; /* Fail if bigger than 256MB and not a 32MB multiple */ + size = size >> 13; /* Convert to 32MB chunks */ + mflags = mflags | mpBSu; /* Show 32MB basic size unit */ + } } wimg = 0x2; /* Set basic PPC wimg to 0b0010 - Coherent */ @@ -304,7 +373,7 @@ addr64_t mapping_make(pmap_t pmap, addr64_t va, ppnum_t pa, unsigned int flags, if(flags & mmFlgPerm) mflags |= mpPerm; /* Set permanent mapping */ size = size - 1; /* Change size to offset */ - if(size > 0xFFFF) return 1; /* Leave if size is too big */ + if(size > 0xFFFF) return mapRtBadSz; /* Leave if size is too big */ nlists = mapSetLists(pmap); /* Set number of lists this will be on */ @@ -313,40 +382,44 @@ addr64_t mapping_make(pmap_t pmap, addr64_t va, ppnum_t pa, unsigned int flags, /* the mapping is zero except that the mpLists field is set */ mp->mpFlags |= mflags; /* Add in the rest of the flags to mpLists */ mp->mpSpace = pmap->space; /* Set the address space/pmap lookup ID */ - mp->mpBSize = size; /* Set the size */ + mp->u.mpBSize = size; /* Set the size */ mp->mpPte = 0; /* Set the PTE invalid */ mp->mpPAddr = pa; /* Set the physical page number */ - mp->mpVAddr = (va & ~mpHWFlags) | (wimg << 3) | ppc_prot(prot); /* Add the protection and attributes to the field */ - + + if ( !nx_enabled || (pmap->pmapFlags & pmapNXdisabled) ) + disable_NX = TRUE; + + mp->mpVAddr = (va & ~mpHWFlags) | (wimg << 3) | getProtPPC(prot, disable_NX); /* Add the protection and attributes to the field */ + while(1) { /* Keep trying... */ colladdr = hw_add_map(pmap, mp); /* Go add the mapping to the pmap */ - if(!colladdr) { /* All is ok... */ - enable_preemption(); /* Ok to switch around here */ - return 0; /* Return... */ - } + rc = colladdr & mapRetCode; /* Separate return code */ + colladdr &= ~mapRetCode; /* Clean up collision effective address */ - if((colladdr & mapRetCode) == mapRtRemove) { /* Is our target being removed? */ - (void)mapping_remove(pmap, colladdr); /* Yes, go help out */ - continue; /* Try to add it now */ - } - - if((colladdr & mapRetCode) == mapRtMapDup) { /* Is our target already mapped (collision mapping must be identical)? */ - mapping_free(mp); /* Return mapping to the free list */ - enable_preemption(); /* Ok to switch around here */ - return 0; /* Normal return */ + switch (rc) { + case mapRtOK: + return mapRtOK; /* Mapping added successfully */ + + case mapRtRemove: /* Remove in progress */ + (void)mapping_remove(pmap, colladdr); /* Lend a helping hand to another CPU doing block removal */ + continue; /* Retry mapping add */ + + case mapRtMapDup: /* Identical mapping already present */ + mapping_free(mp); /* Free duplicate mapping */ + return mapRtOK; /* Return success */ + + case mapRtSmash: /* Mapping already present but does not match new mapping */ + mapping_free(mp); /* Free duplicate mapping */ + return (colladdr | mapRtSmash); /* Return colliding address, with some dirt added to avoid + confusion if effective address is 0 */ + default: + panic("mapping_make: hw_add_map failed - collision addr = %016llX, code = %02X, pmap = %08X, va = %016llX, mapping = %08X\n", + colladdr, rc, pmap, va, mp); /* Die dead */ } - if(colladdr != mapRtBadLk) { /* Did it collide? */ - mapping_free(mp); /* Yeah, toss the pending mapping */ - enable_preemption(); /* Ok to switch around here */ - return colladdr; /* Pass back the overlapping address */ - } - - panic("mapping_make: hw_add_map failed - code = %08X, pmap = %08X, va = %016llX, mapping = %08X\n", - colladdr, pmap, va, mp); /* Die dead */ } - return 1; /* Leave... */ + return 1; /* Unreachable, but pleases compiler */ } @@ -367,16 +440,16 @@ addr64_t mapping_make(pmap_t pmap, addr64_t va, ppnum_t pa, unsigned int flags, * */ -mapping *mapping_find(pmap_t pmap, addr64_t va, addr64_t *nextva, int full) { /* Make an address mapping */ +mapping_t *mapping_find(pmap_t pmap, addr64_t va, addr64_t *nextva, int full) { /* Make an address mapping */ - register mapping *mp; + register mapping_t *mp; addr64_t curva; pmap_t curpmap; int nestdepth; curpmap = pmap; /* Remember entry */ nestdepth = 0; /* Set nest depth */ - curva = (addr64_t)va; /* Set current va */ + curva = (addr64_t)va; /* Set current va */ while(1) { @@ -385,9 +458,10 @@ mapping *mapping_find(pmap_t pmap, addr64_t va, addr64_t *nextva, int full) { /* panic("mapping_find: pmap lock failure - rc = %08X, pmap = %08X\n", mp, curpmap); /* Die... */ } - if(!mp || !(mp->mpFlags & mpNest) || !full) break; /* Are we a nest or are we only going one deep? */ + if(!mp || ((mp->mpFlags & mpType) < mpMinSpecial) || !full) break; /* Are we done looking? */ - if(mp->mpFlags & mpSpecial) { /* Don't chain through a special mapping */ + if((mp->mpFlags & mpType) != mpNest) { /* Don't chain through anything other than a nested pmap */ + mapping_drop_busy(mp); /* We have everything we need from the mapping */ mp = 0; /* Set not found */ break; } @@ -407,7 +481,7 @@ mapping *mapping_find(pmap_t pmap, addr64_t va, addr64_t *nextva, int full) { /* } /* - * kern_return_t mapping_protect(pmap_t pmap, addt_t va, vm_prot_t prot, addr64_t *nextva) - change the protection of a virtual page + * void mapping_protect(pmap_t pmap, addt_t va, vm_prot_t prot, addr64_t *nextva) - change the protection of a virtual page * * This routine takes a pmap and virtual address and changes * the protection. If there are PTEs associated with the mappings, they will be invalidated before @@ -419,22 +493,23 @@ mapping *mapping_find(pmap_t pmap, addr64_t va, addr64_t *nextva, int full) { /* * */ -int mapping_protect(pmap_t pmap, addr64_t va, vm_prot_t prot, addr64_t *nextva) { /* Change protection of a virtual page */ +void +mapping_protect(pmap_t pmap, addr64_t va, vm_prot_t prot, addr64_t *nextva) { /* Change protection of a virtual page */ int ret; - - ret = hw_protect(pmap, va, ppc_prot(prot), nextva); /* Try to change the protect here */ + boolean_t disable_NX = FALSE; + + if ( !nx_enabled || (pmap->pmapFlags & pmapNXdisabled) ) + disable_NX = TRUE; + + ret = hw_protect(pmap, va, getProtPPC(prot, disable_NX), nextva); /* Try to change the protect here */ switch (ret) { /* Decode return code */ case mapRtOK: /* Changed */ case mapRtNotFnd: /* Didn't find it */ - return mapRtOK; /* Ok, return... */ - break; - case mapRtBlock: /* Block map, just ignore request */ case mapRtNest: /* Nested pmap, just ignore request */ - return ret; /* Pass back return code */ break; default: @@ -449,8 +524,8 @@ int mapping_protect(pmap_t pmap, addr64_t va, vm_prot_t prot, addr64_t *nextva) * * This routine takes a physical entry and runs through all mappings attached to it and changes * the protection. If there are PTEs associated with the mappings, they will be invalidated before - * the protection is changed. There is no limitation on changes, e.g., - * higher to lower, lower to higher. + * the protection is changed. There is no limitation on changes, e.g., higher to lower, lower to + * higher; however, changes to execute protection are ignored. * * Any mapping that is marked permanent is not changed * @@ -460,16 +535,17 @@ int mapping_protect(pmap_t pmap, addr64_t va, vm_prot_t prot, addr64_t *nextva) void mapping_protect_phys(ppnum_t pa, vm_prot_t prot) { /* Change protection of all mappings to page */ unsigned int pindex; - phys_entry *physent; - + phys_entry_t *physent; + physent = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ if(!physent) { /* Did we find the physical page? */ panic("mapping_protect_phys: invalid physical page %08X\n", pa); } - hw_walk_phys(physent, hwpSPrtPhy, hwpSPrtMap, hwpNoop, ppc_prot(prot)); /* Set the new protection for page and mappings */ + hw_walk_phys(physent, hwpNoop, hwpSPrtMap, hwpNoop, + getProtPPC(prot, FALSE), hwpPurgePTE); /* Set the new protection for page and mappings */ - return; /* Leave... */ + return; /* Leave... */ } @@ -483,14 +559,15 @@ void mapping_protect_phys(ppnum_t pa, vm_prot_t prot) { /* Change protection of void mapping_clr_mod(ppnum_t pa) { /* Clears the change bit of a physical page */ unsigned int pindex; - phys_entry *physent; + phys_entry_t *physent; physent = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ if(!physent) { /* Did we find the physical page? */ panic("mapping_clr_mod: invalid physical page %08X\n", pa); } - hw_walk_phys(physent, hwpNoop, hwpCCngMap, hwpCCngPhy, 0); /* Clear change for page and mappings */ + hw_walk_phys(physent, hwpNoop, hwpCCngMap, hwpCCngPhy, + 0, hwpPurgePTE); /* Clear change for page and mappings */ return; /* Leave... */ } @@ -505,14 +582,15 @@ void mapping_clr_mod(ppnum_t pa) { /* Clears the change bit of a physical void mapping_set_mod(ppnum_t pa) { /* Sets the change bit of a physical page */ unsigned int pindex; - phys_entry *physent; + phys_entry_t *physent; physent = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ if(!physent) { /* Did we find the physical page? */ panic("mapping_set_mod: invalid physical page %08X\n", pa); } - hw_walk_phys(physent, hwpNoop, hwpSCngMap, hwpSCngPhy, 0); /* Set change for page and mappings */ + hw_walk_phys(physent, hwpNoop, hwpSCngMap, hwpSCngPhy, + 0, hwpNoopPTE); /* Set change for page and mappings */ return; /* Leave... */ } @@ -527,14 +605,15 @@ void mapping_set_mod(ppnum_t pa) { /* Sets the change bit of a physical p void mapping_clr_ref(ppnum_t pa) { /* Clears the reference bit of a physical page */ unsigned int pindex; - phys_entry *physent; + phys_entry_t *physent; physent = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ if(!physent) { /* Did we find the physical page? */ panic("mapping_clr_ref: invalid physical page %08X\n", pa); } - hw_walk_phys(physent, hwpNoop, hwpCRefMap, hwpCRefPhy, 0); /* Clear reference for page and mappings */ + hw_walk_phys(physent, hwpNoop, hwpCRefMap, hwpCRefPhy, + 0, hwpPurgePTE); /* Clear reference for page and mappings */ return; /* Leave... */ } @@ -549,20 +628,21 @@ void mapping_clr_ref(ppnum_t pa) { /* Clears the reference bit of a physi void mapping_set_ref(ppnum_t pa) { /* Sets the reference bit of a physical page */ unsigned int pindex; - phys_entry *physent; + phys_entry_t *physent; physent = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ if(!physent) { /* Did we find the physical page? */ panic("mapping_set_ref: invalid physical page %08X\n", pa); } - hw_walk_phys(physent, hwpNoop, hwpSRefMap, hwpSRefPhy, 0); /* Set reference for page and mappings */ + hw_walk_phys(physent, hwpNoop, hwpSRefMap, hwpSRefPhy, + 0, hwpNoopPTE); /* Set reference for page and mappings */ return; /* Leave... */ } /* - * void mapping_tst_mod(ppnum_t pa) - test the change bit of a physical page + * boolean_t mapping_tst_mod(ppnum_t pa) - test the change bit of a physical page * * This routine takes a physical entry and runs through all mappings attached to it and tests * the changed bit. @@ -571,20 +651,21 @@ void mapping_set_ref(ppnum_t pa) { /* Sets the reference bit of a physica boolean_t mapping_tst_mod(ppnum_t pa) { /* Tests the change bit of a physical page */ unsigned int pindex, rc; - phys_entry *physent; + phys_entry_t *physent; physent = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ if(!physent) { /* Did we find the physical page? */ panic("mapping_tst_mod: invalid physical page %08X\n", pa); } - rc = hw_walk_phys(physent, hwpTCngPhy, hwpTCngMap, hwpNoop, 0); /* Set change for page and mappings */ + rc = hw_walk_phys(physent, hwpTCngPhy, hwpTCngMap, hwpNoop, + 0, hwpMergePTE); /* Set change for page and mappings */ return ((rc & (unsigned long)ppC) != 0); /* Leave with change bit */ } /* - * void mapping_tst_ref(ppnum_t pa) - tests the reference bit of a physical page + * boolean_t mapping_tst_ref(ppnum_t pa) - tests the reference bit of a physical page * * This routine takes a physical entry and runs through all mappings attached to it and tests * the reference bit. @@ -593,18 +674,72 @@ boolean_t mapping_tst_mod(ppnum_t pa) { /* Tests the change bit of a physi boolean_t mapping_tst_ref(ppnum_t pa) { /* Tests the reference bit of a physical page */ unsigned int pindex, rc; - phys_entry *physent; + phys_entry_t *physent; physent = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ if(!physent) { /* Did we find the physical page? */ panic("mapping_tst_ref: invalid physical page %08X\n", pa); } - rc = hw_walk_phys(physent, hwpTRefPhy, hwpTRefMap, hwpNoop, 0); /* Test reference for page and mappings */ + rc = hw_walk_phys(physent, hwpTRefPhy, hwpTRefMap, hwpNoop, + 0, hwpMergePTE); /* Test reference for page and mappings */ return ((rc & (unsigned long)ppR) != 0); /* Leave with reference bit */ } +/* + * unsigned int mapping_tst_refmod(ppnum_t pa) - tests the reference and change bits of a physical page + * + * This routine takes a physical entry and runs through all mappings attached to it and tests + * their reference and changed bits. + */ + +unsigned int mapping_tst_refmod(ppnum_t pa) { /* Tests the reference and change bits of a physical page */ + + unsigned int pindex, rc; + phys_entry_t *physent; + + physent = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ + if (!physent) { /* Did we find the physical page? */ + panic("mapping_tst_refmod: invalid physical page %08X\n", pa); + } + + rc = hw_walk_phys(physent, hwpTRefCngPhy, hwpTRefCngMap, hwpNoop, + 0, hwpMergePTE); /* Test reference and change bits in page and mappings */ + return (((rc & ppC)? VM_MEM_MODIFIED : 0) | ((rc & ppR)? VM_MEM_REFERENCED : 0)); + /* Convert bits to generic format and return */ + +} + + +/* + * void mapping_clr_refmod(ppnum_t pa, unsigned int mask) - clears the reference and change bits specified + * by mask of a physical page + * + * This routine takes a physical entry and runs through all mappings attached to it and turns + * off all the reference and change bits. + */ + +void mapping_clr_refmod(ppnum_t pa, unsigned int mask) { /* Clears the reference and change bits of a physical page */ + + unsigned int pindex; + phys_entry_t *physent; + unsigned int ppcMask; + + physent = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ + if(!physent) { /* Did we find the physical page? */ + panic("mapping_clr_refmod: invalid physical page %08X\n", pa); + } + + ppcMask = (((mask & VM_MEM_MODIFIED)? ppC : 0) | ((mask & VM_MEM_REFERENCED)? ppR : 0)); + /* Convert mask bits to PPC-specific format */ + hw_walk_phys(physent, hwpNoop, hwpCRefCngMap, hwpCRefCngPhy, + ppcMask, hwpPurgePTE); /* Clear reference and change bits for page and mappings */ + return; /* Leave... */ +} + + + /* * phys_ent *mapping_phys_lookup(ppnum_t pp, unsigned int *pindex) - tests the reference bit of a physical page * @@ -613,9 +748,8 @@ boolean_t mapping_tst_ref(ppnum_t pa) { /* Tests the reference bit of a ph * the reference bit. */ -phys_entry *mapping_phys_lookup(ppnum_t pp, unsigned int *pindex) { /* Finds the physical entry for the page */ +phys_entry_t *mapping_phys_lookup(ppnum_t pp, unsigned int *pindex) { /* Finds the physical entry for the page */ - phys_entry *physent; int i; for(i = 0; i < pmap_mem_regions_count; i++) { /* Walk through the list */ @@ -627,7 +761,7 @@ phys_entry *mapping_phys_lookup(ppnum_t pp, unsigned int *pindex) { /* Finds the return &pmap_mem_regions[i].mrPhysTab[pp - pmap_mem_regions[i].mrStart]; /* Return the physent pointer */ } - return (phys_entry *)0; /* Shucks, can't find it... */ + return (phys_entry_t *)0; /* Shucks, can't find it... */ } @@ -652,10 +786,9 @@ static thread_call_data_t mapping_adjust_call_data; void mapping_adjust(void) { /* Adjust free mappings */ kern_return_t retr = KERN_SUCCESS; - mappingblok *mb, *mbn; + mappingblok_t *mb, *mbn; spl_t s; - int allocsize, i; - extern int vm_page_free_count; + int allocsize; if(mapCtl.mapcmin <= MAPPERBLOK) { mapCtl.mapcmin = (sane_size / PAGE_SIZE) / 16; @@ -715,7 +848,7 @@ void mapping_adjust(void) { /* Adjust free mappings */ break; /* Fail to alocate, bail out... */ for(; allocsize > 0; allocsize -= MAPPERBLOK) { /* Release one block at a time */ mapping_free_init((vm_offset_t)mbn, 0, 1); /* Initialize a non-permanent block */ - mbn = (mappingblok *)((unsigned int)mbn + PAGE_SIZE); /* Point to the next slot */ + mbn = (mappingblok_t *)((unsigned int)mbn + PAGE_SIZE); /* Point to the next slot */ } if ((mapCtl.mapcinuse + mapCtl.mapcfree + (mapCtl.mapcreln * (MAPPERBLOK + 1))) > mapCtl.mapcmaxalloc) @@ -761,12 +894,12 @@ void mapping_adjust(void) { /* Adjust free mappings */ void mapping_free(struct mapping *mp) { /* Release a mapping */ - mappingblok *mb, *mbn; + mappingblok_t *mb, *mbn; spl_t s; unsigned int full, mindx, lists; mindx = ((unsigned int)mp & (PAGE_SIZE - 1)) >> 6; /* Get index to mapping */ - mb = (mappingblok *)((unsigned int)mp & -PAGE_SIZE); /* Point to the mapping block */ + mb = (mappingblok_t *)((unsigned int)mp & -PAGE_SIZE); /* Point to the mapping block */ lists = (mp->mpFlags & mpLists); /* get #lists */ if ((lists == 0) || (lists > kSkipListMaxLists)) /* panic if out of range */ panic("mapping_free: mpLists invalid\n"); @@ -774,7 +907,7 @@ void mapping_free(struct mapping *mp) { /* Release a mapping */ #if 0 mp->mpFlags = 0x99999999; /* (BRINGUP) */ mp->mpSpace = 0x9999; /* (BRINGUP) */ - mp->mpBSize = 0x9999; /* (BRINGUP) */ + mp->u.mpBSize = 0x9999; /* (BRINGUP) */ mp->mpPte = 0x99999998; /* (BRINGUP) */ mp->mpPAddr = 0x99999999; /* (BRINGUP) */ mp->mpVAddr = 0x9999999999999999ULL; /* (BRINGUP) */ @@ -881,22 +1014,19 @@ void mapping_free(struct mapping *mp) { /* Release a mapping */ * we allocate a new block. * */ +decl_simple_lock_data(extern,free_pmap_lock) -mapping *mapping_alloc(int lists) { /* Obtain a mapping */ +mapping_t * +mapping_alloc(int lists) { /* Obtain a mapping */ - register mapping *mp; - mappingblok *mb, *mbn; + register mapping_t *mp; + mappingblok_t *mb, *mbn; spl_t s; int mindx; - kern_return_t retr; int big = (lists > mpBasicLists); /* set flag if big block req'd */ pmap_t refpmap, ckpmap; unsigned int space, i; - int ref_count; addr64_t va, nextva; - extern pmap_t free_pmap_list; - extern int free_pmap_count; - decl_simple_lock_data(extern,free_pmap_lock) boolean_t found_mapping; boolean_t do_rescan; @@ -912,7 +1042,7 @@ mapping *mapping_alloc(int lists) { /* Obtain a mapping */ * list. If so, rescue one. Otherwise, try to steal a couple blocks worth. */ - if(mbn = mapCtl.mapcrel) { /* Try to rescue a block from impending doom */ + if((mbn = mapCtl.mapcrel) != 0) { /* Try to rescue a block from impending doom */ mapCtl.mapcrel = mbn->nextblok; /* Pop the queue */ mapCtl.mapcreln--; /* Back off the count */ mapping_free_init((vm_offset_t)mbn, 0, 1); /* Initialize a non-permanent block */ @@ -946,24 +1076,32 @@ mapping *mapping_alloc(int lists) { /* Obtain a mapping */ ckpmap = (pmap_t)ckpmap->pmap_link.next; - if ((ckpmap->stats.resident_count != 0) && (ckpmap != kernel_pmap)) { + /* We don't steal mappings from the kernel pmap, a VMM host pmap, or a VMM guest pmap with guest + shadow assist active. + */ + if ((ckpmap->stats.resident_count != 0) && (ckpmap != kernel_pmap) + && !(ckpmap->pmapFlags & (pmapVMgsaa|pmapVMhost))) { do_rescan = TRUE; for (i=0;i<8;i++) { mp = hw_purge_map(ckpmap, va, &nextva); - if((unsigned int)mp & mapRetCode) { - panic("mapping_alloc: hw_purge_map failed - pmap = %08X, va = %16llX, code = %08X\n", ckpmap, va, mp); + switch ((unsigned int)mp & mapRetCode) { + case mapRtOK: + mapping_free(mp); + found_mapping = TRUE; + break; + case mapRtNotFnd: + break; + default: + panic("mapping_alloc: hw_purge_map failed - pmap = %08X, va = %16llX, code = %08X\n", ckpmap, va, mp); + break; } - if(!mp) { + if (mapRtNotFnd == ((unsigned int)mp & mapRetCode)) if (do_rescan) do_rescan = FALSE; else break; - } else { - mapping_free(mp); - found_mapping = TRUE; - } va = nextva; } @@ -1051,7 +1189,7 @@ rescued: */ if(mapCtl.mapcfree < mapCtl.mapcmin) { /* See if we need to replenish */ - if(mbn = mapCtl.mapcrel) { /* Try to rescue a block from impending doom */ + if((mbn = mapCtl.mapcrel) != 0) { /* Try to rescue a block from impending doom */ mapCtl.mapcrel = mbn->nextblok; /* Pop the queue */ mapCtl.mapcreln--; /* Back off the count */ mapping_free_init((vm_offset_t)mbn, 0, 1); /* Initialize a non-permanent block */ @@ -1068,7 +1206,7 @@ rescued: hw_lock_unlock((hw_lock_t)&mapCtl.mapclock); /* Unlock our stuff */ splx(s); /* Restore 'rupts */ - mp = &((mapping *)mb)[mindx]; /* Point to the allocated mapping */ + mp = &((mapping_t *)mb)[mindx]; /* Point to the allocated mapping */ mp->mpFlags = lists; /* set the list count */ @@ -1077,7 +1215,7 @@ rescued: void -consider_mapping_adjust() +consider_mapping_adjust(void) { spl_t s; @@ -1139,13 +1277,12 @@ void mapping_free_init(vm_offset_t mbl, int perm, boolean_t locked) { or goes straight to the release queue . locked indicates if the lock is held already */ - mappingblok *mb; + mappingblok_t *mb; spl_t s; - int i; addr64_t raddr; ppnum_t pp; - mb = (mappingblok *)mbl; /* Start of area */ + mb = (mappingblok_t *)mbl; /* Start of area */ if(perm >= 0) { /* See if we need to initialize the block */ if(perm) { @@ -1220,7 +1357,7 @@ void mapping_prealloc(unsigned int size) { /* Preallocates mapppings for lar int nmapb, i; kern_return_t retr; - mappingblok *mbn; + mappingblok_t *mbn; spl_t s; s = splhigh(); /* Don't bother from now on */ @@ -1298,11 +1435,11 @@ void mapping_free_prime(void) { /* Primes the mapping block release list int nmapb, i; kern_return_t retr; - mappingblok *mbn; + mappingblok_t *mbn; vm_offset_t mapping_min; retr = kmem_suballoc(kernel_map, &mapping_min, sane_size / 16, - FALSE, TRUE, &mapping_map); + FALSE, VM_FLAGS_ANYWHERE, &mapping_map); if (retr != KERN_SUCCESS) panic("mapping_free_prime: kmem_suballoc failed"); @@ -1328,7 +1465,7 @@ void mapping_free_prime(void) { /* Primes the mapping block release list } - +void mapping_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, vm_size_t *alloc_size, int *collectable, int *exhaustable) { @@ -1355,9 +1492,9 @@ mapping_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_ addr64_t mapping_p2v(pmap_t pmap, ppnum_t pa) { /* Finds first virtual mapping of a physical page in a space */ spl_t s; - mapping *mp; + mapping_t *mp; unsigned int pindex; - phys_entry *physent; + phys_entry_t *physent; addr64_t va; physent = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ @@ -1367,7 +1504,7 @@ addr64_t mapping_p2v(pmap_t pmap, ppnum_t pa) { /* Finds first virtual mappin s = splhigh(); /* Make sure interruptions are disabled */ - mp = (mapping *) hw_find_space(physent, pmap->space); /* Go find the first mapping to the page from the requested pmap */ + mp = hw_find_space(physent, pmap->space); /* Go find the first mapping to the page from the requested pmap */ if(mp) { /* Did we find one? */ va = mp->mpVAddr & -4096; /* If so, get the cleaned up vaddr */ @@ -1381,34 +1518,13 @@ addr64_t mapping_p2v(pmap_t pmap, ppnum_t pa) { /* Finds first virtual mappin } -/* - * phystokv(addr) - * - * Convert a physical address to a kernel virtual address if - * there is a mapping, otherwise return NULL - */ - -vm_offset_t phystokv(vm_offset_t pa) { - - addr64_t va; - ppnum_t pp; - - pp = pa >> 12; /* Convert to a page number */ - - if(!(va = mapping_p2v(kernel_pmap, pp))) { - return 0; /* Can't find it, return 0... */ - } - - return (va | (pa & (PAGE_SIZE - 1))); /* Build and return VADDR... */ - -} /* * kvtophys(addr) * * Convert a kernel virtual address to a physical address */ -vm_offset_t kvtophys(vm_offset_t va) { +addr64_t kvtophys(vm_offset_t va) { return pmap_extract(kernel_pmap, va); /* Find mapping and lock the physical entry for this mapping */ @@ -1425,21 +1541,28 @@ vm_offset_t kvtophys(vm_offset_t va) { void ignore_zero_fault(boolean_t type) { /* Sets up to ignore or honor any fault on page 0 access for the current thread */ - if(type) current_act()->mact.specFlags |= ignoreZeroFault; /* Ignore faults on page 0 */ - else current_act()->mact.specFlags &= ~ignoreZeroFault; /* Honor faults on page 0 */ + if(type) current_thread()->machine.specFlags |= ignoreZeroFault; /* Ignore faults on page 0 */ + else current_thread()->machine.specFlags &= ~ignoreZeroFault; /* Honor faults on page 0 */ return; /* Return the result or 0... */ } +/* + * nop in current ppc implementation + */ +void inval_copy_windows(__unused thread_t t) +{ +} + /* * Copies data between a physical page and a virtual page, or 2 physical. This is used to * move data from the kernel to user state. Note that the "which" parm * says which of the parameters is physical and if we need to flush sink/source. - * Note that both addresses may be physicical but only one may be virtual + * Note that both addresses may be physical, but only one may be virtual. * * The rules are that the size can be anything. Either address can be on any boundary - * and span pages. The physical data must be congiguous as must the virtual. + * and span pages. The physical data must be contiguous as must the virtual. * * We can block when we try to resolve the virtual address at each page boundary. * We don't check protection on the physical page. @@ -1449,17 +1572,17 @@ void ignore_zero_fault(boolean_t type) { /* Sets up to ignore or honor any fa * */ -kern_return_t copypv(addr64_t source, addr64_t sink, unsigned int size, int which) { +kern_return_t hw_copypv_32(addr64_t source, addr64_t sink, unsigned int size, int which) { vm_map_t map; kern_return_t ret; - addr64_t pa, nextva, vaddr, paddr; - register mapping *mp; + addr64_t nextva, vaddr, paddr; + register mapping_t *mp; spl_t s; - unsigned int sz, left, lop, csize; + unsigned int lop, csize; int needtran, bothphys; unsigned int pindex; - phys_entry *physent; + phys_entry_t *physent; vm_prot_t prot; int orig_which; @@ -1473,11 +1596,11 @@ kern_return_t copypv(addr64_t source, addr64_t sink, unsigned int size, int whic bothphys = 1; /* Assume both are physical */ - if(!(which & cppvPsnk)) { /* Is there a virtual page here? */ + if(!(which & cppvPsnk)) { /* Is sink page virtual? */ vaddr = sink; /* Sink side is virtual */ bothphys = 0; /* Show both aren't physical */ prot = VM_PROT_READ | VM_PROT_WRITE; /* Sink always must be read/write */ - } else if(!(which & cppvPsrc)) { /* Source side is virtual */ + } else if (!(which & cppvPsrc)) { /* Is source page virtual? */ vaddr = source; /* Source side is virtual */ bothphys = 0; /* Show both aren't physical */ prot = VM_PROT_READ; /* Virtual source is always read only */ @@ -1497,11 +1620,11 @@ kern_return_t copypv(addr64_t source, addr64_t sink, unsigned int size, int whic while(1) { mp = mapping_find(map->pmap, vaddr, &nextva, 1); /* Find and busy the mapping */ if(!mp) { /* Was it there? */ - if(per_proc_info[cpu_number()].istackptr == 0) + if(getPerProc()->istackptr == 0) panic("copypv: No vaild mapping on memory %s %x", "RD", vaddr); splx(s); /* Restore the interrupt level */ - ret = vm_fault(map, trunc_page_32((vm_offset_t)vaddr), prot, FALSE, FALSE, NULL, 0); /* Didn't find it, try to fault it in... */ + ret = vm_fault(map, vm_map_trunc_page(vaddr), prot, FALSE, THREAD_UNINT, NULL, 0); /* Didn't find it, try to fault it in... */ if(ret != KERN_SUCCESS)return KERN_FAILURE; /* Didn't find any, return no good... */ @@ -1523,11 +1646,11 @@ kern_return_t copypv(addr64_t source, addr64_t sink, unsigned int size, int whic if((which & cppvPsnk) || !(mp->mpVAddr & 1)) break; /* We got it mapped R/W or the source is not virtual, leave... */ mapping_drop_busy(mp); /* Go ahead and release the mapping for now */ - if(per_proc_info[cpu_number()].istackptr == 0) + if(getPerProc()->istackptr == 0) panic("copypv: No vaild mapping on memory %s %x", "RDWR", vaddr); splx(s); /* Restore the interrupt level */ - ret = vm_fault(map, trunc_page_32((vm_offset_t)vaddr), VM_PROT_READ | VM_PROT_WRITE, FALSE, FALSE, NULL, 0); /* check for a COW area */ + ret = vm_fault(map, vm_map_trunc_page(vaddr), VM_PROT_READ | VM_PROT_WRITE, FALSE, THREAD_UNINT, NULL, 0); /* check for a COW area */ if (ret != KERN_SUCCESS) return KERN_FAILURE; /* We couldn't get it R/W, leave in disgrace... */ s = splhigh(); /* Don't bother me */ } @@ -1546,7 +1669,7 @@ kern_return_t copypv(addr64_t source, addr64_t sink, unsigned int size, int whic if(which & cppvFsrc) flush_dcache64(source, csize, 1); /* If requested, flush source before move */ if(which & cppvFsnk) flush_dcache64(sink, csize, 1); /* If requested, flush sink before move */ - bcopy_physvir(source, sink, csize); /* Do a physical copy, virtually */ + bcopy_physvir_32(source, sink, csize); /* Do a physical copy, virtually */ if(which & cppvFsrc) flush_dcache64(source, csize, 1); /* If requested, flush source after move */ if(which & cppvFsnk) flush_dcache64(sink, csize, 1); /* If requested, flush sink after move */ @@ -1584,8 +1707,8 @@ kern_return_t copypv(addr64_t source, addr64_t sink, unsigned int size, int whic void mapping_verify(void) { spl_t s; - mappingblok *mb, *mbn; - int relncnt; + mappingblok_t *mb, *mbn; + unsigned int relncnt; unsigned int dumbodude; dumbodude = 0; @@ -1594,7 +1717,7 @@ void mapping_verify(void) { mbn = 0; /* Start with none */ for(mb = mapCtl.mapcnext; mb; mb = mb->nextblok) { /* Walk the free chain */ - if((mappingblok *)(mb->mapblokflags & 0x7FFFFFFF) != mb) { /* Is tag ok? */ + if((mappingblok_t *)(mb->mapblokflags & 0x7FFFFFFF) != mb) { /* Is tag ok? */ panic("mapping_verify: flags tag bad, free chain; mb = %08X, tag = %08X\n", mb, mb->mapblokflags); } mbn = mb; /* Remember the last one */ @@ -1622,23 +1745,36 @@ void mapping_verify(void) { void mapping_phys_unused(ppnum_t pa) { unsigned int pindex; - phys_entry *physent; + phys_entry_t *physent; physent = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ if(!physent) return; /* Did we find the physical page? */ - if(!(physent->ppLink & ~(ppLock | ppN | ppFlags))) return; /* No one else is here */ + if(!(physent->ppLink & ~(ppLock | ppFlags))) return; /* No one else is here */ panic("mapping_phys_unused: physical page (%08X) in use, physent = %08X\n", pa, physent); } - - - - - - - - - +void mapping_hibernate_flush(void) +{ + int bank; + unsigned int page; + struct phys_entry * entry; + + for (bank = 0; bank < pmap_mem_regions_count; bank++) + { + entry = (struct phys_entry *) pmap_mem_regions[bank].mrPhysTab; + for (page = pmap_mem_regions[bank].mrStart; page <= pmap_mem_regions[bank].mrEnd; page++) + { + hw_walk_phys(entry, hwpNoop, hwpNoop, hwpNoop, 0, hwpPurgePTE); + entry++; + } + } +} + + + + + +