From 483a1d1004b64bbaaef2c64c17c6b999009a54d2 Mon Sep 17 00:00:00 2001 From: Apple Date: Tue, 10 Aug 2004 20:04:38 +0000 Subject: [PATCH] xnu-517.7.21.tar.gz --- bsd/conf/version.minor | 2 +- bsd/hfs/hfs_link.c | 2 +- bsd/kern/sysctl_init.c | 2 + bsd/netinet/ip_input.c | 130 +- bsd/netinet/ip_var.h | 3 +- bsd/netinet6/frag6.c | 2 +- bsd/nfs/nfs_bio.c | 418 +- bsd/nfs/nfs_socket.c | 11 +- bsd/nfs/nfs_subs.c | 2 + bsd/nfs/nfs_vnops.c | 292 +- bsd/nfs/nfsm_subs.h | 24 +- bsd/nfs/nfsnode.h | 18 +- bsd/vfs/vfs_journal.c | 30 +- config/IOKit.exports | 1 + config/System.kext/Info.plist | 8 +- .../PlugIns/AppleNMI.kext/Info.plist | 6 +- .../ApplePlatformFamily.kext/Info.plist | 6 +- .../PlugIns/BSDKernel.kext/Info.plist | 6 +- .../System.kext/PlugIns/IOKit.kext/Info.plist | 6 +- .../PlugIns/IONVRAMFamily.kext/Info.plist | 6 +- .../IOSystemManagement.kext/Info.plist | 6 +- .../PlugIns/Libkern.kext/Info.plist | 6 +- .../System.kext/PlugIns/Mach.kext/Info.plist | 6 +- config/System6.0.exports | 2 + iokit/IOKit/IOUserClient.h | 9 + iokit/IOKit/pwr_mgt/RootDomain.h | 11 +- iokit/Kernel/IOLib.c | 2 +- iokit/Kernel/IOPMrootDomain.cpp | 48 +- iokit/Kernel/IOUserClient.cpp | 27 + iokit/KernelConfigTables.cpp | 18 +- iokit/conf/version.minor | 2 +- libkern/conf/version.minor | 2 +- libsa/conf/version.minor | 2 +- osfmk/conf/kernelversion.minor | 2 +- osfmk/conf/version.minor | 2 +- osfmk/ppc/Diagnostics.c | 9 +- osfmk/ppc/asm.h | 6 + osfmk/ppc/chud/chud_spr.h | 2 +- osfmk/ppc/cpu.c | 7 +- osfmk/ppc/exception.h | 22 +- osfmk/ppc/genassym.c | 13 +- osfmk/ppc/interrupt.c | 45 +- osfmk/ppc/lowmem_vectors.s.orig | 3403 +++++++++++++++++ osfmk/ppc/machine_routines.c | 5 + osfmk/ppc/machine_routines.h | 4 + osfmk/ppc/machine_routines_asm.s | 306 +- osfmk/ppc/start.s | 148 +- osfmk/vm/task_working_set.c | 175 +- osfmk/vm/task_working_set.h | 2 +- pexpert/conf/version.minor | 2 +- 50 files changed, 4408 insertions(+), 861 deletions(-) create mode 100644 osfmk/ppc/lowmem_vectors.s.orig diff --git a/bsd/conf/version.minor b/bsd/conf/version.minor index b8626c4cf..7ed6ff82d 100644 --- a/bsd/conf/version.minor +++ b/bsd/conf/version.minor @@ -1 +1 @@ -4 +5 diff --git a/bsd/hfs/hfs_link.c b/bsd/hfs/hfs_link.c index 9f112e375..97dfde7bb 100644 --- a/bsd/hfs/hfs_link.c +++ b/bsd/hfs/hfs_link.c @@ -332,7 +332,7 @@ hfs_link(ap) } // XXXdbg - need to do this here as well because cp could have changed - error = VOP_UPDATE(vp, &tv, &tv, 1); + (void) VOP_UPDATE(vp, &tv, &tv, 1); if (hfsmp->jnl) { diff --git a/bsd/kern/sysctl_init.c b/bsd/kern/sysctl_init.c index 684f105f3..3500efd8a 100644 --- a/bsd/kern/sysctl_init.c +++ b/bsd/kern/sysctl_init.c @@ -124,6 +124,7 @@ extern struct sysctl_oid sysctl__net_inet_ip_sourceroute; extern struct sysctl_oid sysctl__net_inet_ip_subnets_are_local; extern struct sysctl_oid sysctl__net_inet_ip_keepfaith; extern struct sysctl_oid sysctl__net_inet_ip_maxfragpackets; +extern struct sysctl_oid sysctl__net_inet_ip_maxfragsperpacket; extern struct sysctl_oid sysctl__net_inet_ip_check_interface; extern struct sysctl_oid sysctl__net_inet_ip_check_route_selfref; extern struct sysctl_oid sysctl__net_inet_ip_use_route_genid; @@ -525,6 +526,7 @@ struct sysctl_oid *newsysctl_list[] = ,&sysctl__net_inet_ip_subnets_are_local ,&sysctl__net_inet_ip_keepfaith ,&sysctl__net_inet_ip_maxfragpackets + ,&sysctl__net_inet_ip_maxfragsperpacket ,&sysctl__net_inet_ip_check_interface ,&sysctl__net_inet_ip_check_route_selfref ,&sysctl__net_inet_ip_use_route_genid diff --git a/bsd/netinet/ip_input.c b/bsd/netinet/ip_input.c index 9ce6d6f1c..5ac01c9e1 100644 --- a/bsd/netinet/ip_input.c +++ b/bsd/netinet/ip_input.c @@ -148,12 +148,17 @@ SYSCTL_INT(_net_inet_ip, IPCTL_KEEPFAITH, keepfaith, CTLFLAG_RW, &ip_keepfaith, 0, "Enable packet capture for FAITH IPv4->IPv6 translater daemon"); -static int ip_nfragpackets = 0; -static int ip_maxfragpackets; /* initialized in ip_init() */ +static int nipq = 0; /* total # of reass queues */ +static int maxnipq = 0; SYSCTL_INT(_net_inet_ip, OID_AUTO, maxfragpackets, CTLFLAG_RW, - &ip_maxfragpackets, 0, + &maxnipq, 0, "Maximum number of IPv4 fragment reassembly queue entries"); +static int maxfragsperpacket; +SYSCTL_INT(_net_inet_ip, OID_AUTO, maxfragsperpacket, CTLFLAG_RW, + &maxfragsperpacket, 0, + "Maximum number of IPv4 fragments allowed per packet"); + /* * XXX - Setting ip_checkinterface mostly implements the receive side of * the Strong ES model described in RFC 1122, but since the routing table @@ -198,8 +203,6 @@ SYSCTL_STRUCT(_net_inet_ip, IPCTL_STATS, stats, CTLFLAG_RD, (((((x) & 0xF) | ((((x) >> 8) & 0xF) << 4)) ^ (y)) & IPREASS_HMASK) static struct ipq ipq[IPREASS_NHASH]; -static int nipq = 0; /* total # of reass queues */ -static int maxnipq; const int ipintrq_present = 1; #if IPCTL_DEFMTU @@ -314,8 +317,8 @@ ip_init() for (i = 0; i < IPREASS_NHASH; i++) ipq[i].next = ipq[i].prev = &ipq[i]; - maxnipq = nmbclusters / 4; - ip_maxfragpackets = nmbclusters / 4; + maxnipq = nmbclusters / 32; + maxfragsperpacket = 16; #if RANDOM_IP_ID ip_id = time_second & 0xffff; @@ -770,21 +773,13 @@ ours: */ if (ip->ip_off & (IP_MF | IP_OFFMASK | IP_RF)) { -#if 0 /* - * Reassembly should be able to treat a mbuf cluster, for later - * operation of contiguous protocol headers on the cluster. (KAME) - */ - if (m->m_flags & M_EXT) { /* XXX */ - if ((m = m_pullup(m, hlen)) == 0) { - ipstat.ips_toosmall++; -#if IPFIREWALL_FORWARD - ip_fw_fwd_addr = NULL; -#endif - return; + /* If maxnipq is 0, never accept fragments. */ + if (maxnipq == 0) { + ipstat.ips_fragments++; + ipstat.ips_fragdropped++; + goto bad; } - ip = mtod(m, struct ip *); - } -#endif + sum = IPREASS_HASH(ip->ip_src.s_addr, ip->ip_id); /* * Look for queue of fragments @@ -799,8 +794,12 @@ ours: fp = 0; - /* check if there's a place for the new queue */ - if (nipq > maxnipq) { + /* + * Enforce upper bound on number of fragmented packets + * for which we attempt reassembly; + * If maxnipq is -1, accept all fragments without limitation. + */ + if ((nipq > maxnipq) && (maxnipq > 0)) { /* * drop something from the tail of the current queue * before proceeding further @@ -808,22 +807,24 @@ ours: if (ipq[sum].prev == &ipq[sum]) { /* gak */ for (i = 0; i < IPREASS_NHASH; i++) { if (ipq[i].prev != &ipq[i]) { + ipstat.ips_fragtimeout += + ipq[i].prev->ipq_nfrags; ip_freef(ipq[i].prev); break; } } - } else + } else { + ipstat.ips_fragtimeout += ipq[sum].prev->ipq_nfrags; ip_freef(ipq[sum].prev); } + } found: /* * Adjust ip_len to not reflect header, - * set ip_mff if more fragments are expected, * convert offset of this to bytes. */ ip->ip_len -= hlen; - mff = (ip->ip_off & IP_MF) != 0; - if (mff) { + if (ip->ip_off & IP_MF) { /* * Make sure that fragments have a data length * that's a non-zero multiple of 8 bytes. @@ -833,18 +834,15 @@ found: goto bad; } m->m_flags |= M_FRAG; - } else { - /* Clear the flag in case packet comes from loopback */ + } else m->m_flags &= ~M_FRAG; - } ip->ip_off <<= 3; /* - * If datagram marked as having more fragments - * or if this is not the first fragment, - * attempt reassembly; if it succeeds, proceed. + * Attempt reassembly; if it succeeds, proceed. + * ip_reass() will return a different mbuf, and update + * the divert info in divert_info and args.divert_rule. */ - if (mff || ip->ip_off) { ipstat.ips_fragments++; m->m_pkthdr.header = ip; #if IPDIVERT @@ -877,9 +875,6 @@ found: } #endif } else - if (fp) - ip_freef(fp); - } else ip->ip_len -= hlen; #if IPDIVERT @@ -1021,20 +1016,12 @@ ip_reass(m, fp, where) * If first fragment to arrive, create a reassembly queue. */ if (fp == 0) { - /* - * Enforce upper bound on number of fragmented packets - * for which we attempt reassembly; - * If maxfrag is 0, never accept fragments. - * If maxfrag is -1, accept all fragments without limitation. - */ - if ((ip_maxfragpackets >= 0) && (ip_nfragpackets >= ip_maxfragpackets)) - goto dropfrag; - ip_nfragpackets++; if ((t = m_get(M_DONTWAIT, MT_FTABLE)) == NULL) goto dropfrag; fp = mtod(t, struct ipq *); insque((void*)fp, (void*)where); nipq++; + fp->ipq_nfrags = 1; fp->ipq_ttl = IPFRAGTTL; fp->ipq_p = ip->ip_p; fp->ipq_id = ip->ip_id; @@ -1051,6 +1038,8 @@ ip_reass(m, fp, where) fp->ipq_div_cookie = 0; #endif goto inserted; + } else { + fp->ipq_nfrags++; } #define GETIP(m) ((struct ip*)((m)->m_pkthdr.header)) @@ -1105,6 +1094,8 @@ ip_reass(m, fp, where) } nq = q->m_nextpkt; m->m_nextpkt = nq; + ipstat.ips_fragdropped++; + fp->ipq_nfrags--; m_freem(q); } @@ -1113,30 +1104,49 @@ inserted: #if IPDIVERT /* * Transfer firewall instructions to the fragment structure. - * Any fragment diverting causes the whole packet to divert. + * Only trust info in the fragment at offset 0. */ + if (ip->ip_off == 0) { #ifdef IPDIVERT_44 fp->ipq_div_info = *divinfo; #else fp->ipq_divert = *divinfo; #endif fp->ipq_div_cookie = *divcookie; + } *divinfo = 0; *divcookie = 0; #endif /* - * Check for complete reassembly. + * Check for complete reassembly and perform frag per packet + * limiting. + * + * Frag limiting is performed here so that the nth frag has + * a chance to complete the packet before we drop the packet. + * As a result, n+1 frags are actually allowed per packet, but + * only n will ever be stored. (n = maxfragsperpacket.) + * */ next = 0; for (p = NULL, q = fp->ipq_frags; q; p = q, q = q->m_nextpkt) { - if (GETIP(q)->ip_off != next) + if (GETIP(q)->ip_off != next) { + if (fp->ipq_nfrags > maxfragsperpacket) { + ipstat.ips_fragdropped += fp->ipq_nfrags; + ip_freef(fp); + } return (0); + } next += GETIP(q)->ip_len; } /* Make sure the last packet didn't have the IP_MF flag */ - if (p->m_flags & M_FRAG) + if (p->m_flags & M_FRAG) { + if (fp->ipq_nfrags > maxfragsperpacket) { + ipstat.ips_fragdropped += fp->ipq_nfrags; + ip_freef(fp); + } return (0); + } /* * Reassembly is complete. Make sure the packet is a sane size. @@ -1145,6 +1155,7 @@ inserted: ip = GETIP(q); if (next + (IP_VHL_HL(ip->ip_vhl) << 2) > IP_MAXPACKET) { ipstat.ips_toolong++; + ipstat.ips_fragdropped += fp->ipq_nfrags; ip_freef(fp); return (0); } @@ -1161,12 +1172,8 @@ inserted: for (q = nq; q != NULL; q = nq) { nq = q->m_nextpkt; q->m_nextpkt = NULL; - if (q->m_pkthdr.csum_flags & CSUM_TCP_SUM16) - m->m_pkthdr.csum_flags = 0; - else { m->m_pkthdr.csum_flags &= q->m_pkthdr.csum_flags; m->m_pkthdr.csum_data += q->m_pkthdr.csum_data; - } m_cat(m, q); } @@ -1194,7 +1201,6 @@ inserted: remque((void*)fp); nipq--; (void) m_free(dtom(fp)); - ip_nfragpackets--; m->m_len += (IP_VHL_HL(ip->ip_vhl) << 2); m->m_data -= (IP_VHL_HL(ip->ip_vhl) << 2); /* some debugging cruft by sklower, below, will go away soon */ @@ -1212,6 +1218,8 @@ dropfrag: *divcookie = 0; #endif ipstat.ips_fragdropped++; + if (fp != 0) + fp->ipq_nfrags--; m_freem(m); return (0); @@ -1235,7 +1243,6 @@ ip_freef(fp) } remque((void*)fp); (void) m_free(dtom(fp)); - ip_nfragpackets--; nipq--; } @@ -1259,7 +1266,7 @@ ip_slowtimo() --fp->ipq_ttl; fp = fp->next; if (fp->prev->ipq_ttl == 0) { - ipstat.ips_fragtimeout++; + ipstat.ips_fragtimeout += fp->prev->ipq_nfrags; ip_freef(fp->prev); } } @@ -1269,11 +1276,12 @@ ip_slowtimo() * (due to the limit being lowered), drain off * enough to get down to the new limit. */ + if (maxnipq >= 0 && nipq > maxnipq) { for (i = 0; i < IPREASS_NHASH; i++) { - if (ip_maxfragpackets >= 0) { - while ((ip_nfragpackets > ip_maxfragpackets) && + while (nipq > maxnipq && (ipq[i].next != &ipq[i])) { - ipstat.ips_fragdropped++; + ipstat.ips_fragdropped += + ipq[i].next->ipq_nfrags; ip_freef(ipq[i].next); } } @@ -1292,7 +1300,7 @@ ip_drain() for (i = 0; i < IPREASS_NHASH; i++) { while (ipq[i].next != &ipq[i]) { - ipstat.ips_fragdropped++; + ipstat.ips_fragdropped += ipq[i].next->ipq_nfrags; ip_freef(ipq[i].next); } } diff --git a/bsd/netinet/ip_var.h b/bsd/netinet/ip_var.h index b591a22bc..d5c0f22f6 100644 --- a/bsd/netinet/ip_var.h +++ b/bsd/netinet/ip_var.h @@ -83,7 +83,8 @@ struct ipq { u_short ipq_id; /* sequence id for reassembly */ struct mbuf *ipq_frags; /* to ip headers of fragments */ struct in_addr ipq_src,ipq_dst; - u_long reserved[4]; /* for future use */ + u_long ipq_nfrags; + u_long reserved[3]; /* for future use */ #if IPDIVERT #ifdef IPDIVERT_44 u_int32_t ipq_div_info; /* ipfw divert port & flags */ diff --git a/bsd/netinet6/frag6.c b/bsd/netinet6/frag6.c index 01888c6cb..f686d01cd 100644 --- a/bsd/netinet6/frag6.c +++ b/bsd/netinet6/frag6.c @@ -84,7 +84,7 @@ frag6_init() { struct timeval tv; - ip6_maxfragpackets = nmbclusters / 4; + ip6_maxfragpackets = nmbclusters / 32; /* * in many cases, random() here does NOT return random number diff --git a/bsd/nfs/nfs_bio.c b/bsd/nfs/nfs_bio.c index 8478a7da4..c756be676 100644 --- a/bsd/nfs/nfs_bio.c +++ b/bsd/nfs/nfs_bio.c @@ -103,13 +103,40 @@ extern struct nfsstats nfsstats; #define NFSBUFHASH(dvp, lbn) \ (&nfsbufhashtbl[((long)(dvp) / sizeof(*(dvp)) + (int)(lbn)) & nfsbufhash]) LIST_HEAD(nfsbufhashhead, nfsbuf) *nfsbufhashtbl; -struct nfsbuffreehead nfsbuffree, nfsbufdelwri; +struct nfsbuffreehead nfsbuffree, nfsbuffreemeta, nfsbufdelwri; u_long nfsbufhash; int nfsbufhashlock, nfsbufcnt, nfsbufmin, nfsbufmax; -int nfsbuffreecnt, nfsbufdelwricnt, nfsneedbuffer; +int nfsbuffreecnt, nfsbuffreemetacnt, nfsbufdelwricnt, nfsneedbuffer; int nfs_nbdwrite; +time_t nfsbuffreeuptimestamp; + +#define NFSBUFWRITE_THROTTLE 9 +#define NFSBUF_LRU_STALE 120 +#define NFSBUF_META_STALE 240 + +/* number of nfsbufs nfs_buf_freeup() should attempt to free from nfsbuffree list */ +#define LRU_TO_FREEUP 6 +/* number of nfsbufs nfs_buf_freeup() should attempt to free from nfsbuffreemeta list */ +#define META_TO_FREEUP 3 +/* total number of nfsbufs nfs_buf_freeup() should attempt to free */ +#define TOTAL_TO_FREEUP (LRU_TO_FREEUP+META_TO_FREEUP) +/* fraction of nfsbufs nfs_buf_freeup() should attempt to free from nfsbuffree list when called from nfs_timer() */ +#define LRU_FREEUP_FRAC_ON_TIMER 8 +/* fraction of nfsbufs nfs_buf_freeup() should attempt to free from nfsbuffreemeta list when called from nfs_timer() */ +#define META_FREEUP_FRAC_ON_TIMER 16 +/* fraction of total nfsbufs that nfsbuffreecnt should exceed before bothering to call nfs_buf_freeup() */ +#define LRU_FREEUP_MIN_FRAC 4 +/* fraction of total nfsbufs that nfsbuffreemetacnt should exceed before bothering to call nfs_buf_freeup() */ +#define META_FREEUP_MIN_FRAC 2 -#define NFSBUFWRITE_THROTTLE 9 +#define NFS_BUF_FREEUP() \ + do { \ + /* only call nfs_buf_freeup() if it has work to do: */ \ + if (((nfsbuffreecnt > nfsbufcnt/LRU_FREEUP_MIN_FRAC) || \ + (nfsbuffreemetacnt > nfsbufcnt/META_FREEUP_MIN_FRAC)) && \ + ((nfsbufcnt - TOTAL_TO_FREEUP) > nfsbufmin)) \ + nfs_buf_freeup(0); \ + } while (0) /* * Initialize nfsbuf lists @@ -120,42 +147,37 @@ nfs_nbinit(void) nfsbufhashlock = 0; nfsbufhashtbl = hashinit(nbuf, M_TEMP, &nfsbufhash); TAILQ_INIT(&nfsbuffree); + TAILQ_INIT(&nfsbuffreemeta); TAILQ_INIT(&nfsbufdelwri); - nfsbufcnt = nfsbuffreecnt = nfsbufdelwricnt = 0; + nfsbufcnt = nfsbuffreecnt = nfsbuffreemetacnt = nfsbufdelwricnt = 0; nfsbufmin = 128; // XXX tune me! nfsbufmax = 8192; // XXX tune me! nfsneedbuffer = 0; nfs_nbdwrite = 0; + nfsbuffreeuptimestamp = 0; } /* * try to free up some excess, unused nfsbufs */ -static void -nfs_buf_freeup(void) +void +nfs_buf_freeup(int timer) { struct nfsbuf *fbp; - int cnt; + struct timeval now; + int count; -#define NFS_BUF_FREEUP() \ - do { \ - /* only call nfs_buf_freeup() if it has work to do */ \ - if ((nfsbuffreecnt > nfsbufcnt/4) && \ - (nfsbufcnt-nfsbuffreecnt/8 > nfsbufmin)) \ - nfs_buf_freeup(); \ - } while (0) + microuptime(&now); + nfsbuffreeuptimestamp = now.tv_sec; - if (nfsbuffreecnt < nfsbufcnt/4) - return; - cnt = nfsbuffreecnt/8; - if (nfsbufcnt-cnt < nfsbufmin) - return; - - FSDBG(320, -1, nfsbufcnt, nfsbuffreecnt, cnt); - while (cnt-- > 0) { + FSDBG(320, nfsbufcnt, nfsbuffreecnt, nfsbuffreemetacnt, count); + count = timer ? nfsbuffreecnt/LRU_FREEUP_FRAC_ON_TIMER : LRU_TO_FREEUP; + while ((nfsbufcnt > nfsbufmin) && (count-- > 0)) { fbp = TAILQ_FIRST(&nfsbuffree); if (!fbp) break; + if ((fbp->nb_timestamp + (2*NFSBUF_LRU_STALE)) > now.tv_sec) + break; nfs_buf_remfree(fbp); /* disassociate buffer from any vnode */ if (fbp->nb_vp) { @@ -181,7 +203,40 @@ nfs_buf_freeup(void) FREE(fbp, M_TEMP); nfsbufcnt--; } - FSDBG(320, -1, nfsbufcnt, nfsbuffreecnt, cnt); + + count = timer ? nfsbuffreemetacnt/META_FREEUP_FRAC_ON_TIMER : META_TO_FREEUP; + while ((nfsbufcnt > nfsbufmin) && (count-- > 0)) { + fbp = TAILQ_FIRST(&nfsbuffreemeta); + if (!fbp) + break; + if ((fbp->nb_timestamp + (2*NFSBUF_META_STALE)) > now.tv_sec) + break; + nfs_buf_remfree(fbp); + /* disassociate buffer from any vnode */ + if (fbp->nb_vp) { + struct vnode *oldvp; + if (fbp->nb_vnbufs.le_next != NFSNOLIST) { + LIST_REMOVE(fbp, nb_vnbufs); + fbp->nb_vnbufs.le_next = NFSNOLIST; + } + oldvp = fbp->nb_vp; + fbp->nb_vp = NULL; + HOLDRELE(oldvp); + } + LIST_REMOVE(fbp, nb_hash); + /* nuke any creds */ + if (fbp->nb_rcred != NOCRED) + crfree(fbp->nb_rcred); + if (fbp->nb_wcred != NOCRED) + crfree(fbp->nb_wcred); + /* if buf was NB_META, dump buffer */ + if (ISSET(fbp->nb_flags, NB_META) && fbp->nb_data) { + FREE(fbp->nb_data, M_TEMP); + } + FREE(fbp, M_TEMP); + nfsbufcnt--; + } + FSDBG(320, nfsbufcnt, nfsbuffreecnt, nfsbuffreemetacnt, count); } void @@ -192,6 +247,9 @@ nfs_buf_remfree(struct nfsbuf *bp) if (ISSET(bp->nb_flags, NB_DELWRI)) { nfsbufdelwricnt--; TAILQ_REMOVE(&nfsbufdelwri, bp, nb_free); + } else if (ISSET(bp->nb_flags, NB_META) && !ISSET(bp->nb_flags, NB_INVAL)) { + nfsbuffreemetacnt--; + TAILQ_REMOVE(&nfsbuffreemeta, bp, nb_free); } else { nfsbuffreecnt--; TAILQ_REMOVE(&nfsbuffree, bp, nb_free); @@ -209,11 +267,12 @@ nfs_buf_incore(struct vnode *vp, daddr_t blkno) /* Search hash chain */ struct nfsbuf * bp = NFSBUFHASH(vp, blkno)->lh_first; for (; bp != NULL; bp = bp->nb_hash.le_next) - if (bp->nb_lblkno == blkno && bp->nb_vp == vp && - !ISSET(bp->nb_flags, NB_INVAL)) { - FSDBG(547, bp, blkno, bp->nb_flags, bp->nb_vp); - return (bp); - } + if (bp->nb_lblkno == blkno && bp->nb_vp == vp) { + if (!ISSET(bp->nb_flags, NB_INVAL)) { + FSDBG(547, bp, blkno, bp->nb_flags, bp->nb_vp); + return (bp); + } + } return (NULL); } @@ -541,85 +600,125 @@ loop: /* * where to get a free buffer: * - alloc new if we haven't reached min bufs - * - free list + * - if free lists are NOT empty + * - if free list is stale, use it + * - else if freemeta list is stale, use it + * - else if max bufs allocated, use least-time-to-stale * - alloc new if we haven't reached max allowed * - start clearing out delwri list and try again */ - if ((nfsbufcnt > nfsbufmin) && !TAILQ_EMPTY(&nfsbuffree)) { - /* pull an nfsbuf off the free list */ - bp = TAILQ_FIRST(&nfsbuffree); - FSDBG(544, vp, blkno, bp, bp->nb_flags); - nfs_buf_remfree(bp); - if (ISSET(bp->nb_flags, NB_DELWRI)) - panic("nfs_buf_get: delwri"); - SET(bp->nb_flags, NB_BUSY); - /* disassociate buffer from previous vnode */ - if (bp->nb_vp) { - struct vnode *oldvp; - if (bp->nb_vnbufs.le_next != NFSNOLIST) { - LIST_REMOVE(bp, nb_vnbufs); - bp->nb_vnbufs.le_next = NFSNOLIST; + if ((nfsbufcnt > nfsbufmin) && + (!TAILQ_EMPTY(&nfsbuffree) || !TAILQ_EMPTY(&nfsbuffreemeta))) { + /* try to pull an nfsbuf off a free list */ + struct nfsbuf *lrubp, *metabp; + struct timeval now; + microuptime(&now); + + /* if the next LRU or META buffer is stale, use it */ + lrubp = TAILQ_FIRST(&nfsbuffree); + if (lrubp && ((lrubp->nb_timestamp + NFSBUF_LRU_STALE) < now.tv_sec)) + bp = lrubp; + metabp = TAILQ_FIRST(&nfsbuffreemeta); + if (!bp && metabp && ((metabp->nb_timestamp + NFSBUF_META_STALE) < now.tv_sec)) + bp = metabp; + + if (!bp && (nfsbufcnt >= nfsbufmax)) { + /* we've already allocated all bufs, so */ + /* choose the buffer that'll go stale first */ + if (!metabp) + bp = lrubp; + else if (!lrubp) + bp = metabp; + else { + int32_t lru_stale_time, meta_stale_time; + lru_stale_time = lrubp->nb_timestamp + NFSBUF_LRU_STALE; + meta_stale_time = metabp->nb_timestamp + NFSBUF_META_STALE; + if (lru_stale_time <= meta_stale_time) + bp = lrubp; + else + bp = metabp; } - oldvp = bp->nb_vp; - bp->nb_vp = NULL; - HOLDRELE(oldvp); } - LIST_REMOVE(bp, nb_hash); - /* nuke any creds we're holding */ - cred = bp->nb_rcred; - if (cred != NOCRED) { - bp->nb_rcred = NOCRED; - crfree(cred); - } - cred = bp->nb_wcred; - if (cred != NOCRED) { - bp->nb_wcred = NOCRED; - crfree(cred); - } - /* if buf will no longer be NB_META, dump old buffer */ - if ((operation != BLK_META) && - ISSET(bp->nb_flags, NB_META) && bp->nb_data) { - FREE(bp->nb_data, M_TEMP); - bp->nb_data = NULL; + + if (bp) { + /* we have a buffer to reuse */ + FSDBG(544, vp, blkno, bp, bp->nb_flags); + nfs_buf_remfree(bp); + if (ISSET(bp->nb_flags, NB_DELWRI)) + panic("nfs_buf_get: delwri"); + SET(bp->nb_flags, NB_BUSY); + /* disassociate buffer from previous vnode */ + if (bp->nb_vp) { + struct vnode *oldvp; + if (bp->nb_vnbufs.le_next != NFSNOLIST) { + LIST_REMOVE(bp, nb_vnbufs); + bp->nb_vnbufs.le_next = NFSNOLIST; + } + oldvp = bp->nb_vp; + bp->nb_vp = NULL; + HOLDRELE(oldvp); + } + LIST_REMOVE(bp, nb_hash); + /* nuke any creds we're holding */ + cred = bp->nb_rcred; + if (cred != NOCRED) { + bp->nb_rcred = NOCRED; + crfree(cred); + } + cred = bp->nb_wcred; + if (cred != NOCRED) { + bp->nb_wcred = NOCRED; + crfree(cred); + } + /* if buf will no longer be NB_META, dump old buffer */ + if ((operation != BLK_META) && + ISSET(bp->nb_flags, NB_META) && bp->nb_data) { + FREE(bp->nb_data, M_TEMP); + bp->nb_data = NULL; + } + /* re-init buf fields */ + bp->nb_error = 0; + bp->nb_validoff = bp->nb_validend = -1; + bp->nb_dirtyoff = bp->nb_dirtyend = 0; + bp->nb_valid = 0; + bp->nb_dirty = 0; } - /* re-init buf fields */ - bp->nb_error = 0; - bp->nb_validoff = bp->nb_validend = -1; - bp->nb_dirtyoff = bp->nb_dirtyend = 0; - bp->nb_valid = 0; - bp->nb_dirty = 0; - } else if (nfsbufcnt < nfsbufmax) { - /* just alloc a new one */ - MALLOC(bp, struct nfsbuf *, sizeof(struct nfsbuf), M_TEMP, M_WAITOK); - nfsbufcnt++; - NFSBUFCNTCHK(); - /* init nfsbuf */ - bzero(bp, sizeof(*bp)); - bp->nb_free.tqe_next = NFSNOLIST; - bp->nb_validoff = bp->nb_validend = -1; - FSDBG(545, vp, blkno, bp, 0); - } else { - /* too many bufs... wait for buffers to free up */ - FSDBG_TOP(546, vp, blkno, nfsbufcnt, nfsbufmax); - /* unlock hash */ - if (nfsbufhashlock < 0) { - nfsbufhashlock = 0; - wakeup(&nfsbufhashlock); - } else - nfsbufhashlock = 0; + } - /* poke the delwri list */ - nfs_buf_delwri_push(); + if (!bp) { + if (nfsbufcnt < nfsbufmax) { + /* just alloc a new one */ + MALLOC(bp, struct nfsbuf *, sizeof(struct nfsbuf), M_TEMP, M_WAITOK); + nfsbufcnt++; + NFSBUFCNTCHK(); + /* init nfsbuf */ + bzero(bp, sizeof(*bp)); + bp->nb_free.tqe_next = NFSNOLIST; + bp->nb_validoff = bp->nb_validend = -1; + FSDBG(545, vp, blkno, bp, 0); + } else { + /* too many bufs... wait for buffers to free up */ + FSDBG_TOP(546, vp, blkno, nfsbufcnt, nfsbufmax); + /* unlock hash */ + if (nfsbufhashlock < 0) { + nfsbufhashlock = 0; + wakeup(&nfsbufhashlock); + } else + nfsbufhashlock = 0; + + /* poke the delwri list */ + nfs_buf_delwri_push(); - nfsneedbuffer = 1; - tsleep(&nfsneedbuffer, PCATCH, "nfsbufget", 0); - FSDBG_BOT(546, vp, blkno, nfsbufcnt, nfsbufmax); - if (nfs_sigintr(VFSTONFS(vp->v_mount), NULL, p)) { - FSDBG_BOT(541, vp, blkno, 0, EINTR); - return (NULL); + nfsneedbuffer = 1; + tsleep(&nfsneedbuffer, PCATCH, "nfsbufget", 0); + FSDBG_BOT(546, vp, blkno, nfsbufcnt, nfsbufmax); + if (nfs_sigintr(VFSTONFS(vp->v_mount), NULL, p)) { + FSDBG_BOT(541, vp, blkno, 0, EINTR); + return (NULL); + } + goto loop; } - goto loop; } setup_nfsbuf: @@ -671,6 +770,8 @@ buffer_setup: LIST_REMOVE(bp, nb_vnbufs); bp->nb_vnbufs.le_next = NFSNOLIST; bp->nb_vp = NULL; + /* clear usage timestamp to allow immediate freeing */ + bp->nb_timestamp = 0; HOLDRELE(vp); if (bp->nb_free.tqe_next != NFSNOLIST) panic("nfsbuf on freelist"); @@ -700,9 +801,10 @@ buffer_setup: } void -nfs_buf_release(struct nfsbuf *bp) +nfs_buf_release(struct nfsbuf *bp, int freeup) { struct vnode *vp = bp->nb_vp; + struct timeval now; FSDBG_TOP(548, bp, NBOFF(bp), bp->nb_flags, bp->nb_data); FSDBG(548, bp->nb_validoff, bp->nb_validend, bp->nb_dirtyoff, bp->nb_dirtyend); @@ -800,12 +902,16 @@ pagelist_cleanup_done: NFSBUFCNTCHK(); wakeup((caddr_t)&nfs_nbdwrite); } + /* clear usage timestamp to allow immediate freeing */ + bp->nb_timestamp = 0; /* put buffer at head of free list */ if (bp->nb_free.tqe_next != NFSNOLIST) panic("nfsbuf on freelist"); + SET(bp->nb_flags, NB_INVAL); TAILQ_INSERT_HEAD(&nfsbuffree, bp, nb_free); nfsbuffreecnt++; - NFS_BUF_FREEUP(); + if (freeup) + NFS_BUF_FREEUP(); } else if (ISSET(bp->nb_flags, NB_DELWRI)) { /* put buffer at end of delwri list */ if (bp->nb_free.tqe_next != NFSNOLIST) @@ -813,12 +919,21 @@ pagelist_cleanup_done: TAILQ_INSERT_TAIL(&nfsbufdelwri, bp, nb_free); nfsbufdelwricnt++; } else { + /* update usage timestamp */ + microuptime(&now); + bp->nb_timestamp = now.tv_sec; /* put buffer at end of free list */ if (bp->nb_free.tqe_next != NFSNOLIST) panic("nfsbuf on freelist"); - TAILQ_INSERT_TAIL(&nfsbuffree, bp, nb_free); - nfsbuffreecnt++; - NFS_BUF_FREEUP(); + if (ISSET(bp->nb_flags, NB_META)) { + TAILQ_INSERT_TAIL(&nfsbuffreemeta, bp, nb_free); + nfsbuffreemetacnt++; + } else { + TAILQ_INSERT_TAIL(&nfsbuffree, bp, nb_free); + nfsbuffreecnt++; + } + if (freeup) + NFS_BUF_FREEUP(); } NFSBUFCNTCHK(); @@ -885,7 +1000,7 @@ nfs_buf_iodone(struct nfsbuf *bp) } if (ISSET(bp->nb_flags, NB_ASYNC)) /* if async, release it */ - nfs_buf_release(bp); + nfs_buf_release(bp, 1); else { /* or just wakeup the buffer */ CLR(bp->nb_flags, NB_WANTED); wakeup(bp); @@ -948,7 +1063,7 @@ nfs_buf_write_delayed(struct nfsbuf *bp) /* Otherwise, the "write" is done, so mark and release the buffer. */ SET(bp->nb_flags, NB_DONE); - nfs_buf_release(bp); + nfs_buf_release(bp, 1); FSDBG_BOT(551, bp, NBOFF(bp), bp->nb_flags, 0); return; } @@ -1034,6 +1149,12 @@ nfs_bioread(vp, uio, ioflag, cred, getpages) FSDBG_BOT(514, vp, 0xd1e0004, 0, error); return (error); } + if (vp->v_type == VDIR) { + /* if directory changed, purge any name cache entries */ + if (np->n_ncmtime != vattr.va_mtime.tv_sec) + cache_purge(vp); + np->n_ncmtime = vattr.va_mtime.tv_sec; + } np->n_mtime = vattr.va_mtime.tv_sec; } else { error = VOP_GETATTR(vp, &vattr, cred, p); @@ -1045,13 +1166,16 @@ nfs_bioread(vp, uio, ioflag, cred, getpages) if (vp->v_type == VDIR) { nfs_invaldir(vp); /* purge name cache entries */ - cache_purge(vp); + if (np->n_ncmtime != vattr.va_mtime.tv_sec) + cache_purge(vp); } error = nfs_vinvalbuf(vp, V_SAVE, cred, p, 1); if (error) { FSDBG_BOT(514, vp, 0xd1e0006, 0, error); return (error); } + if (vp->v_type == VDIR) + np->n_ncmtime = vattr.va_mtime.tv_sec; np->n_mtime = vattr.va_mtime.tv_sec; } } @@ -1172,10 +1296,10 @@ nfs_bioread(vp, uio, ioflag, cred, getpages) if (nfs_asyncio(rabp, cred)) { SET(rabp->nb_flags, (NB_INVAL|NB_ERROR)); rabp->nb_error = EIO; - nfs_buf_release(rabp); + nfs_buf_release(rabp, 1); } } else - nfs_buf_release(rabp); + nfs_buf_release(rabp, 1); } } @@ -1272,7 +1396,7 @@ again: iov.iov_len = auio.uio_resid; error = nfs_readrpc(vp, &auio, cred); if (error) { - nfs_buf_release(bp); + nfs_buf_release(bp, 1); FSDBG_BOT(514, vp, 0xd1e000e, 0, error); return (error); } @@ -1296,7 +1420,7 @@ again: CLR(bp->nb_flags, (NB_DONE | NB_ERROR | NB_INVAL)); error = nfs_doio(bp, cred, p); if (error) { - nfs_buf_release(bp); + nfs_buf_release(bp, 1); FSDBG_BOT(514, vp, 0xd1e000f, 0, error); return (error); } @@ -1324,7 +1448,7 @@ buffer_ready: error = nfs_doio(bp, cred, p); if (error) { SET(bp->nb_flags, NB_ERROR); - nfs_buf_release(bp); + nfs_buf_release(bp, 1); FSDBG_BOT(514, vp, 0xd1e0011, 0, error); return (error); } @@ -1349,7 +1473,7 @@ buffer_ready: SET(bp->nb_flags, NB_READ); error = nfs_doio(bp, cred, p); if (error) { - nfs_buf_release(bp); + nfs_buf_release(bp, 1); } while (error == NFSERR_BAD_COOKIE) { nfs_invaldir(vp); @@ -1388,7 +1512,7 @@ buffer_ready: * block and go for the next one via the for loop. */ if (error || i < lbn) - nfs_buf_release(bp); + nfs_buf_release(bp, 1); } } /* @@ -1420,10 +1544,10 @@ buffer_ready: if (nfs_asyncio(rabp, cred)) { SET(rabp->nb_flags, (NB_INVAL|NB_ERROR)); rabp->nb_error = EIO; - nfs_buf_release(rabp); + nfs_buf_release(rabp, 1); } } else { - nfs_buf_release(rabp); + nfs_buf_release(rabp, 1); } } } @@ -1478,7 +1602,7 @@ buffer_ready: SET(bp->nb_flags, NB_INVAL); break; } - nfs_buf_release(bp); + nfs_buf_release(bp, 1); } while (error == 0 && uio->uio_resid > 0 && n > 0); FSDBG_BOT(514, vp, uio->uio_offset, uio->uio_resid, error); return (error); @@ -1509,7 +1633,7 @@ nfs_write(ap) daddr_t lbn; int biosize, bufsize, writeop; int n, on, error = 0, iomode, must_commit; - off_t boff, start, end; + off_t boff, start, end, cureof; struct iovec iov; struct uio auio; @@ -1710,6 +1834,7 @@ again: * If there was a partial buf at the old eof, validate * and zero the new bytes. */ + cureof = (off_t)np->n_size; if (uio->uio_offset + n > np->n_size) { struct nfsbuf *eofbp = NULL; daddr_t eofbn = np->n_size / biosize; @@ -1783,7 +1908,7 @@ again: eofoff += PAGE_SIZE - poff; i++; } - nfs_buf_release(eofbp); + nfs_buf_release(eofbp, 1); } } /* @@ -1835,14 +1960,6 @@ again: if (end > start) { /* need to read the data in range: start...end-1 */ - /* - * XXX: If we know any of these reads are beyond the - * current EOF (what np->n_size was before we possibly - * just modified it above), we could short-circuit the - * reads and just zero buffer. No need to make a trip - * across the network to read nothing. - */ - /* first, check for dirty pages in between */ /* if there are, we'll have to do two reads because */ /* we don't want to overwrite the dirty pages. */ @@ -1907,19 +2024,31 @@ again: break; } - /* now we'll read the (rest of the) data */ - auio.uio_offset = boff + start; - auio.uio_resid = iov.iov_len = end - start; - iov.iov_base = bp->nb_data + start; - error = nfs_readrpc(vp, &auio, cred); - if (error) { - bp->nb_error = error; - SET(bp->nb_flags, NB_ERROR); - printf("nfs_write: readrpc %d", error); - } - if (auio.uio_resid > 0) { - FSDBG(516, bp, iov.iov_base - bp->nb_data, auio.uio_resid, 0xd00dee02); - bzero(iov.iov_base, auio.uio_resid); + if (((boff+start) >= cureof) || ((start >= on) && ((boff + on + n) >= cureof))) { + /* + * Either this entire read is beyond the current EOF + * or the range that we won't be modifying (on+n...end) + * is all beyond the current EOF. + * No need to make a trip across the network to + * read nothing. So, just zero the buffer instead. + */ + FSDBG(516, bp, start, end - start, 0xd00dee00); + bzero(bp->nb_data + start, end - start); + } else { + /* now we'll read the (rest of the) data */ + auio.uio_offset = boff + start; + auio.uio_resid = iov.iov_len = end - start; + iov.iov_base = bp->nb_data + start; + error = nfs_readrpc(vp, &auio, cred); + if (error) { + bp->nb_error = error; + SET(bp->nb_flags, NB_ERROR); + printf("nfs_write: readrpc %d", error); + } + if (auio.uio_resid > 0) { + FSDBG(516, bp, iov.iov_base - bp->nb_data, auio.uio_resid, 0xd00dee02); + bzero(iov.iov_base, auio.uio_resid); + } } /* update validoff/validend if necessary */ if ((bp->nb_validoff < 0) || (bp->nb_validoff > start)) @@ -1940,7 +2069,7 @@ again: if (ISSET(bp->nb_flags, NB_ERROR)) { error = bp->nb_error; - nfs_buf_release(bp); + nfs_buf_release(bp, 1); FSDBG_BOT(515, vp, uio->uio_offset, uio->uio_resid, error); return (error); } @@ -1957,13 +2086,13 @@ again: error = nqnfs_getlease(vp, ND_WRITE, cred, p); } while (error == NQNFS_EXPIRED); if (error) { - nfs_buf_release(bp); + nfs_buf_release(bp, 1); FSDBG_BOT(515, vp, uio->uio_offset, 0x11220001, error); return (error); } if (np->n_lrev != np->n_brev || (np->n_flag & NQNFSNONCACHE)) { - nfs_buf_release(bp); + nfs_buf_release(bp, 1); error = nfs_vinvalbuf(vp, V_SAVE, cred, p, 1); if (error) { FSDBG_BOT(515, vp, uio->uio_offset, 0x11220002, error); @@ -1977,7 +2106,7 @@ again: error = uiomove((char *)bp->nb_data + on, n, uio); if (error) { SET(bp->nb_flags, NB_ERROR); - nfs_buf_release(bp); + nfs_buf_release(bp, 1); FSDBG_BOT(515, vp, uio->uio_offset, uio->uio_resid, error); return (error); } @@ -2164,9 +2293,12 @@ nfs_vinvalbuf_internal(vp, flags, cred, p, slpflag, slptimeo) } } SET(bp->nb_flags, NB_INVAL); - nfs_buf_release(bp); + // Note: We don't want to do FREEUPs here because + // that may modify the buffer chain we're iterating! + nfs_buf_release(bp, 0); } } + NFS_BUF_FREEUP(); if (np->n_dirtyblkhd.lh_first || np->n_cleanblkhd.lh_first) panic("nfs_vinvalbuf: flush failed"); return (0); diff --git a/bsd/nfs/nfs_socket.c b/bsd/nfs/nfs_socket.c index 157bd8cb7..047714c00 100644 --- a/bsd/nfs/nfs_socket.c +++ b/bsd/nfs/nfs_socket.c @@ -2172,11 +2172,11 @@ nfs_timer(arg) rep->r_rtt = 0; } } + microuptime(&now); #ifndef NFS_NOSERVER /* * Call the nqnfs server timer once a second to handle leases. */ - microuptime(&now); if (lasttime != now.tv_sec) { lasttime = now.tv_sec; nqnfs_serverd(); @@ -2194,6 +2194,15 @@ nfs_timer(arg) } #endif /* NFS_NOSERVER */ splx(s); + + if (nfsbuffreeuptimestamp + 30 <= now.tv_sec) { + /* + * We haven't called nfs_buf_freeup() in a little while. + * So, see if we can free up any stale/unused bufs now. + */ + nfs_buf_freeup(1); + } + timeout(nfs_timer_funnel, (void *)0, nfs_ticks); } diff --git a/bsd/nfs/nfs_subs.c b/bsd/nfs/nfs_subs.c index fd32619f7..836b85f0f 100644 --- a/bsd/nfs/nfs_subs.c +++ b/bsd/nfs/nfs_subs.c @@ -1398,6 +1398,8 @@ nfs_loadattrcache(vpp, mdp, dposp, vaper, dontshrink, xidp) } } np->n_mtime = mtime.tv_sec; + if (vp->v_type == VDIR) + np->n_ncmtime = mtime.tv_sec; FSDBG(527, vp, np->n_mtime, 0, 0); } np->n_xid = *xidp; diff --git a/bsd/nfs/nfs_vnops.c b/bsd/nfs/nfs_vnops.c index 0780ecc9a..9ca4502e2 100644 --- a/bsd/nfs/nfs_vnops.c +++ b/bsd/nfs/nfs_vnops.c @@ -142,7 +142,7 @@ static int nfsfifo_close __P((struct vop_close_args *)); #define nfs_poll vop_nopoll static int nfs_ioctl __P((struct vop_ioctl_args *)); static int nfs_select __P((struct vop_select_args *)); -static int nfs_flush __P((struct vnode *,struct ucred *,int,struct proc *,int)); +static int nfs_flush __P((struct vnode *,struct ucred *,int,struct proc *)); static int nfs_setattrrpc __P((struct vnode *,struct vattr *,struct ucred *,struct proc *)); static int nfs_lookup __P((struct vop_lookup_args *)); static int nfs_create __P((struct vop_create_args *)); @@ -716,10 +716,12 @@ nfs_open(ap) error = VOP_GETATTR(vp, &vattr, ap->a_cred, ap->a_p); if (error) return (error); - /* if directory changed, purge any name cache entries */ - if ((vp->v_type == VDIR) && - (np->n_mtime != vattr.va_mtime.tv_sec)) - cache_purge(vp); + if (vp->v_type == VDIR) { + /* if directory changed, purge any name cache entries */ + if (np->n_ncmtime != vattr.va_mtime.tv_sec) + cache_purge(vp); + np->n_ncmtime = vattr.va_mtime.tv_sec; + } np->n_mtime = vattr.va_mtime.tv_sec; } else { error = VOP_GETATTR(vp, &vattr, ap->a_cred, ap->a_p); @@ -730,11 +732,14 @@ nfs_open(ap) np->n_direofoffset = 0; nfs_invaldir(vp); /* purge name cache entries */ - cache_purge(vp); + if (np->n_ncmtime != vattr.va_mtime.tv_sec) + cache_purge(vp); } if ((error = nfs_vinvalbuf(vp, V_SAVE, ap->a_cred, ap->a_p, 1)) == EINTR) return (error); + if (vp->v_type == VDIR) + np->n_ncmtime = vattr.va_mtime.tv_sec; np->n_mtime = vattr.va_mtime.tv_sec; } } @@ -764,12 +769,7 @@ nfs_open(ap) * The current code does the following: * for NFS Version 2 - play it safe and flush/invalidate all dirty buffers * for NFS Version 3 - flush dirty buffers to the server but don't invalidate - * or commit them (this satisfies 1 and 2 except for the - * case where the server crashes after this close but - * before the commit RPC, which is felt to be "good - * enough". Changing the last argument to nfs_flush() to - * a 1 would force a commit operation, if it is felt a - * commit is necessary now. + * them. * for NQNFS - do nothing now, since 2 is dealt with via leases and * 1 should be dealt with via an fsync() system call for * cases where write errors are important. @@ -814,7 +814,7 @@ nfs_close(ap) return (error); } if (NFS_ISV3(vp)) { - error = nfs_flush(vp, ap->a_cred, MNT_WAIT, ap->a_p, 1); + error = nfs_flush(vp, ap->a_cred, MNT_WAIT, ap->a_p); /* * We cannot clear the NMODIFIED bit in np->n_flag due to * potential races with other processes @@ -921,12 +921,15 @@ tryagain: if (vp->v_type == VDIR) { nfs_invaldir(vp); /* purge name cache entries */ - cache_purge(vp); + if (np->n_ncmtime != ap->a_vap->va_mtime.tv_sec) + cache_purge(vp); } error = nfs_vinvalbuf(vp, V_SAVE, ap->a_cred, ap->a_p, 1); FSDBG(513, -1, np, -2, error); if (!error) { + if (vp->v_type == VDIR) + np->n_ncmtime = ap->a_vap->va_mtime.tv_sec; np->n_mtime = ap->a_vap->va_mtime.tv_sec; } } @@ -1066,7 +1069,7 @@ nfs_setattr(ap) if (bp) { FSDBG(512, bp, bp->nb_flags, 0, obn); SET(bp->nb_flags, NB_INVAL); - nfs_buf_release(bp); + nfs_buf_release(bp, 1); } } } @@ -1113,7 +1116,7 @@ nfs_setattrrpc(vp, vap, cred, procp) register long t1, t2; caddr_t bpos, dpos, cp2; u_long *tl; - int error = 0, wccflag = NFSV3_WCCRATTR; + int error = 0, wccpostattr = 0; struct mbuf *mreq, *mrep, *md, *mb, *mb2; int v3; u_int64_t xid; @@ -1208,10 +1211,19 @@ nfs_setattrrpc(vp, vap, cred, procp) } nfsm_request(vp, NFSPROC_SETATTR, procp, cred, &xid); if (v3) { + time_t premtime = 0; if (mrep) { - nfsm_wcc_data(vp, wccflag, &xid); + nfsm_wcc_data(vp, premtime, wccpostattr, &xid); + } + /* if file hadn't changed, update cached mtime */ + if (VTONFS(vp)->n_mtime == premtime) { + VTONFS(vp)->n_mtime = VTONFS(vp)->n_vattr.va_mtime.tv_sec; + } + /* if directory hadn't changed, update namecache mtime */ + if ((vp->v_type == VDIR) && (VTONFS(vp)->n_ncmtime == premtime)) { + VTONFS(vp)->n_ncmtime = VTONFS(vp)->n_vattr.va_mtime.tv_sec; } - if (!wccflag) + if (!wccpostattr) VTONFS(vp)->n_xid = 0; } else { if (mrep) { @@ -1267,10 +1279,15 @@ nfs_lookup(ap) wantparent = flags & (LOCKPARENT|WANTPARENT); np = VTONFS(dvp); - /* if directory has changed, purge any name cache entries */ if (!VOP_GETATTR(dvp, &vattr, cnp->cn_cred, p) && - (np->n_mtime != vattr.va_mtime.tv_sec)) + (np->n_ncmtime != vattr.va_mtime.tv_sec)) { + /* + * This directory has changed on us. + * Purge any name cache entries. + */ cache_purge(dvp); + np->n_ncmtime = vattr.va_mtime.tv_sec; + } if ((error = cache_lookup(dvp, vpp, cnp)) && error != ENOENT) { int vpid; @@ -1317,15 +1334,11 @@ nfs_lookup(ap) VOP_UNLOCK(dvp, 0, p); if (vpid == newvp->v_id) { - if (!VOP_GETATTR(newvp, &vattr, cnp->cn_cred, p) - && vattr.va_ctime.tv_sec == VTONFS(newvp)->n_ctime) { - nfsstats.lookupcache_hits++; - if (cnp->cn_nameiop != LOOKUP && (flags & ISLASTCN)) - cnp->cn_flags |= SAVENAME; - error = 0; /* ignore any from VOP_GETATTR */ - goto error_return; - } - cache_purge(newvp); + nfsstats.lookupcache_hits++; + if (cnp->cn_nameiop != LOOKUP && (flags & ISLASTCN)) + cnp->cn_flags |= SAVENAME; + error = 0; /* ignore any from VOP_GETATTR */ + goto error_return; } vput(newvp); if ((dvp != newvp) && lockparent && (flags & ISLASTCN)) @@ -1453,7 +1466,6 @@ cache_lookup_out: cnp->cn_flags |= SAVENAME; if ((cnp->cn_flags & MAKEENTRY) && (cnp->cn_nameiop != DELETE || !(flags & ISLASTCN))) { - np->n_ctime = np->n_vattr.va_ctime.tv_sec; cache_enter(dvp, newvp, cnp); } *vpp = newvp; @@ -1667,7 +1679,7 @@ nfs_writerpc(vp, uiop, cred, iomode, must_commit) caddr_t bpos, dpos, cp2; struct mbuf *mreq, *mrep, *md, *mb, *mb2; struct nfsmount *nmp; - int error = 0, len, tsiz, wccflag = NFSV3_WCCRATTR, rlen, commit; + int error = 0, len, tsiz, updatemtime = 0, wccpostattr = 0, rlen, commit; int v3, committed = NFSV3WRITE_FILESYNC; u_int64_t xid; @@ -1716,9 +1728,11 @@ nfs_writerpc(vp, uiop, cred, iomode, must_commit) if (!nmp) error = ENXIO; if (v3) { - wccflag = NFSV3_WCCCHK; if (mrep) { - nfsm_wcc_data(vp, wccflag, &xid); + time_t premtime; + nfsm_wcc_data(vp, premtime, wccpostattr, &xid); + if (VTONFS(vp)->n_mtime == premtime) + updatemtime = 1; } if (!error) { nfsm_dissect(tl, u_long *, 2 * NFSX_UNSIGNED + @@ -1763,8 +1777,8 @@ nfs_writerpc(vp, uiop, cred, iomode, must_commit) } } - if (wccflag) - VTONFS(vp)->n_mtime = VTONFS(vp)->n_vattr.va_mtime.tv_sec; + if (updatemtime) + VTONFS(vp)->n_mtime = VTONFS(vp)->n_vattr.va_mtime.tv_sec; m_freem(mrep); /* * we seem to have a case where we end up looping on shutdown @@ -1809,7 +1823,8 @@ nfs_mknodrpc(dvp, vpp, cnp, vap) struct vattr vattr; char *cp2; caddr_t bpos, dpos; - int error = 0, wccflag = NFSV3_WCCRATTR, gotvp = 0; + int error = 0, wccpostattr = 0, gotvp = 0; + time_t premtime = 0; struct mbuf *mreq, *mrep, *md, *mb, *mb2; u_long rdev; u_int64_t xid; @@ -1868,7 +1883,7 @@ nfs_mknodrpc(dvp, vpp, cnp, vap) } } if (v3 && mrep) - nfsm_wcc_data(dvp, wccflag, &xid); + nfsm_wcc_data(dvp, premtime, wccpostattr, &xid); nfsm_reqdone; if (error) { if (newvp) @@ -1879,7 +1894,10 @@ nfs_mknodrpc(dvp, vpp, cnp, vap) *vpp = newvp; } VTONFS(dvp)->n_flag |= NMODIFIED; - if (!wccflag) + /* if directory hadn't changed, update namecache mtime */ + if (VTONFS(dvp)->n_ncmtime == premtime) + VTONFS(dvp)->n_ncmtime = VTONFS(dvp)->n_vattr.va_mtime.tv_sec; + if (!wccpostattr) VTONFS(dvp)->n_xid = 0; vput(dvp); NFS_FREE_PNBUF(cnp); @@ -1934,7 +1952,8 @@ nfs_create(ap) struct nfsnode *np = (struct nfsnode *)0; struct vnode *newvp = (struct vnode *)0; caddr_t bpos, dpos, cp2; - int error = 0, wccflag = NFSV3_WCCRATTR, gotvp = 0, fmode = 0; + int error = 0, wccpostattr = 0, gotvp = 0, fmode = 0; + time_t premtime = 0; struct mbuf *mreq, *mrep, *md, *mb, *mb2; struct vattr vattr; int v3 = NFS_ISV3(dvp); @@ -1999,7 +2018,7 @@ again: } } if (v3 && mrep) - nfsm_wcc_data(dvp, wccflag, &xid); + nfsm_wcc_data(dvp, premtime, wccpostattr, &xid); nfsm_reqdone; if (error) { if (v3 && (fmode & O_EXCL) && error == NFSERR_NOTSUPP) { @@ -2016,7 +2035,10 @@ again: *ap->a_vpp = newvp; } VTONFS(dvp)->n_flag |= NMODIFIED; - if (!wccflag) + /* if directory hadn't changed, update namecache mtime */ + if (VTONFS(dvp)->n_ncmtime == premtime) + VTONFS(dvp)->n_ncmtime = VTONFS(dvp)->n_vattr.va_mtime.tv_sec; + if (!wccpostattr) VTONFS(dvp)->n_xid = 0; vput(dvp); NFS_FREE_PNBUF(cnp); @@ -2086,9 +2108,7 @@ nfs_remove(ap) /* * Purge the name cache so that the chance of a lookup for * the name succeeding while the remove is in progress is - * minimized. Without node locking it can still happen, such - * that an I/O op returns ESTALE, but since you get this if - * another host removes the file.. + * minimized. */ cache_purge(vp); /* @@ -2160,7 +2180,8 @@ nfs_removerpc(dvp, name, namelen, cred, proc) register caddr_t cp; register long t1, t2; caddr_t bpos, dpos, cp2; - int error = 0, wccflag = NFSV3_WCCRATTR; + int error = 0, wccpostattr = 0; + time_t premtime = 0; struct mbuf *mreq, *mrep, *md, *mb, *mb2; int v3; u_int64_t xid; @@ -2176,10 +2197,13 @@ nfs_removerpc(dvp, name, namelen, cred, proc) nfsm_strtom(name, namelen, NFS_MAXNAMLEN); nfsm_request(dvp, NFSPROC_REMOVE, proc, cred, &xid); if (v3 && mrep) - nfsm_wcc_data(dvp, wccflag, &xid); + nfsm_wcc_data(dvp, premtime, wccpostattr, &xid); nfsm_reqdone; VTONFS(dvp)->n_flag |= NMODIFIED; - if (!wccflag) + /* if directory hadn't changed, update namecache mtime */ + if (VTONFS(dvp)->n_ncmtime == premtime) + VTONFS(dvp)->n_ncmtime = VTONFS(dvp)->n_vattr.va_mtime.tv_sec; + if (!wccpostattr) VTONFS(dvp)->n_xid = 0; return (error); } @@ -2204,7 +2228,7 @@ nfs_rename(ap) register struct vnode *tdvp = ap->a_tdvp; register struct componentname *tcnp = ap->a_tcnp; register struct componentname *fcnp = ap->a_fcnp; - int error, purged=0, inuse=0; + int error, inuse=0; #if DIAGNOSTIC if ((tcnp->cn_flags & HASBUF) == 0 || @@ -2271,20 +2295,10 @@ nfs_rename(ap) LIST_REMOVE(VTONFS(tvp), n_hash); VTONFS(tvp)->n_flag &= ~NHASHED; } - - if (fvp->v_type == VDIR) { - if (tvp != NULL && tvp->v_type == VDIR) { - cache_purge(tdvp); - if (tvp == tdvp) - purged = 1; - } - cache_purge(fdvp); - } cache_purge(fvp); if (tvp) { - if (!purged) - cache_purge(tvp); + cache_purge(tvp); VOP_UNLOCK(tvp, 0, tcnp->cn_proc); ubc_uncache(tvp); /* get the nfs turd file to disappear */ } @@ -2337,7 +2351,8 @@ nfs_renamerpc(fdvp, fnameptr, fnamelen, tdvp, tnameptr, tnamelen, cred, proc) register caddr_t cp; register long t1, t2; caddr_t bpos, dpos, cp2; - int error = 0, fwccflag = NFSV3_WCCRATTR, twccflag = NFSV3_WCCRATTR; + int error = 0, fwccpostattr = 0, twccpostattr = 0; + time_t fpremtime = 0, tpremtime = 0; struct mbuf *mreq, *mrep, *md, *mb, *mb2; int v3; u_int64_t xid; @@ -2358,15 +2373,21 @@ nfs_renamerpc(fdvp, fnameptr, fnamelen, tdvp, tnameptr, tnamelen, cred, proc) if (v3 && mrep) { u_int64_t txid = xid; - nfsm_wcc_data(fdvp, fwccflag, &xid); - nfsm_wcc_data(tdvp, twccflag, &txid); + nfsm_wcc_data(fdvp, fpremtime, fwccpostattr, &xid); + nfsm_wcc_data(tdvp, tpremtime, twccpostattr, &txid); } nfsm_reqdone; VTONFS(fdvp)->n_flag |= NMODIFIED; - if (!fwccflag) + /* if directory hadn't changed, update namecache mtime */ + if (VTONFS(fdvp)->n_ncmtime == fpremtime) + VTONFS(fdvp)->n_ncmtime = VTONFS(fdvp)->n_vattr.va_mtime.tv_sec; + if (!fwccpostattr) VTONFS(fdvp)->n_xid = 0; VTONFS(tdvp)->n_flag |= NMODIFIED; - if (!twccflag) + /* if directory hadn't changed, update namecache mtime */ + if (VTONFS(tdvp)->n_ncmtime == tpremtime) + VTONFS(tdvp)->n_ncmtime = VTONFS(tdvp)->n_vattr.va_mtime.tv_sec; + if (!twccpostattr) VTONFS(tdvp)->n_xid = 0; return (error); } @@ -2389,7 +2410,8 @@ nfs_link(ap) register caddr_t cp; register long t1, t2; caddr_t bpos, dpos, cp2; - int error = 0, wccflag = NFSV3_WCCRATTR, attrflag = 0; + int error = 0, wccpostattr = 0, attrflag = 0; + time_t premtime = 0; struct mbuf *mreq, *mrep, *md, *mb, *mb2; int v3, didhold; u_int64_t xid; @@ -2435,14 +2457,17 @@ nfs_link(ap) u_int64_t txid = xid; nfsm_postop_attr(vp, attrflag, &xid); - nfsm_wcc_data(tdvp, wccflag, &txid); + nfsm_wcc_data(tdvp, premtime, wccpostattr, &txid); } nfsm_reqdone; VTONFS(tdvp)->n_flag |= NMODIFIED; if (!attrflag) VTONFS(vp)->n_xid = 0; - if (!wccflag) + /* if directory hadn't changed, update namecache mtime */ + if (VTONFS(tdvp)->n_ncmtime == premtime) + VTONFS(tdvp)->n_ncmtime = VTONFS(tdvp)->n_vattr.va_mtime.tv_sec; + if (!wccpostattr) VTONFS(tdvp)->n_xid = 0; if (didhold) ubc_rele(vp); @@ -2478,7 +2503,8 @@ nfs_symlink(ap) register caddr_t cp; register long t1, t2; caddr_t bpos, dpos, cp2; - int slen, error = 0, wccflag = NFSV3_WCCRATTR, gotvp; + int slen, error = 0, wccpostattr = 0, gotvp; + time_t premtime = 0; struct mbuf *mreq, *mrep, *md, *mb, *mb2; struct vnode *newvp = (struct vnode *)0; int v3 = NFS_ISV3(dvp); @@ -2511,14 +2537,17 @@ nfs_symlink(ap) if (!error) nfsm_mtofh(dvp, newvp, v3, gotvp, &xid); - nfsm_wcc_data(dvp, wccflag, &dxid); + nfsm_wcc_data(dvp, premtime, wccpostattr, &dxid); } nfsm_reqdone; if (newvp) vput(newvp); VTONFS(dvp)->n_flag |= NMODIFIED; - if (!wccflag) + /* if directory hadn't changed, update namecache mtime */ + if (VTONFS(dvp)->n_ncmtime == premtime) + VTONFS(dvp)->n_ncmtime = VTONFS(dvp)->n_vattr.va_mtime.tv_sec; + if (!wccpostattr) VTONFS(dvp)->n_xid = 0; vput(dvp); NFS_FREE_PNBUF(cnp); @@ -2554,7 +2583,8 @@ nfs_mkdir(ap) struct nfsnode *np = (struct nfsnode *)0; struct vnode *newvp = (struct vnode *)0; caddr_t bpos, dpos, cp2; - int error = 0, wccflag = NFSV3_WCCRATTR; + int error = 0, wccpostattr = 0; + time_t premtime = 0; int gotvp = 0; struct mbuf *mreq, *mrep, *md, *mb, *mb2; struct vattr vattr; @@ -2589,10 +2619,13 @@ nfs_mkdir(ap) if (!error) nfsm_mtofh(dvp, newvp, v3, gotvp, &xid); if (v3 && mrep) - nfsm_wcc_data(dvp, wccflag, &dxid); + nfsm_wcc_data(dvp, premtime, wccpostattr, &dxid); nfsm_reqdone; VTONFS(dvp)->n_flag |= NMODIFIED; - if (!wccflag) + /* if directory hadn't changed, update namecache mtime */ + if (VTONFS(dvp)->n_ncmtime == premtime) + VTONFS(dvp)->n_ncmtime = VTONFS(dvp)->n_vattr.va_mtime.tv_sec; + if (!wccpostattr) VTONFS(dvp)->n_xid = 0; /* * Kludge: Map EEXIST => 0 assuming that you have a reply to a retry @@ -2614,8 +2647,11 @@ nfs_mkdir(ap) if (error) { if (newvp) vput(newvp); - } else + } else { + if (cnp->cn_flags & MAKEENTRY) + cache_enter(dvp, newvp, cnp); *ap->a_vpp = newvp; + } vput(dvp); NFS_FREE_PNBUF(cnp); return (error); @@ -2639,7 +2675,8 @@ nfs_rmdir(ap) register caddr_t cp; register long t1, t2; caddr_t bpos, dpos, cp2; - int error = 0, wccflag = NFSV3_WCCRATTR; + int error = 0, wccpostattr = 0; + time_t premtime = 0; struct mbuf *mreq, *mrep, *md, *mb, *mb2; int v3 = NFS_ISV3(dvp); u_int64_t xid; @@ -2651,21 +2688,32 @@ nfs_rmdir(ap) nfsm_strtom(cnp->cn_nameptr, cnp->cn_namelen, NFS_MAXNAMLEN); nfsm_request(dvp, NFSPROC_RMDIR, cnp->cn_proc, cnp->cn_cred, &xid); if (v3 && mrep) - nfsm_wcc_data(dvp, wccflag, &xid); + nfsm_wcc_data(dvp, premtime, wccpostattr, &xid); nfsm_reqdone; VTONFS(dvp)->n_flag |= NMODIFIED; - if (!wccflag) + /* if directory hadn't changed, update namecache mtime */ + if (VTONFS(dvp)->n_ncmtime == premtime) + VTONFS(dvp)->n_ncmtime = VTONFS(dvp)->n_vattr.va_mtime.tv_sec; + if (!wccpostattr) VTONFS(dvp)->n_xid = 0; - cache_purge(dvp); cache_purge(vp); - vput(vp); - vput(dvp); - NFS_FREE_PNBUF(cnp); /* * Kludge: Map ENOENT => 0 assuming that you have a reply to a retry. */ if (error == ENOENT) error = 0; + if (!error) { + /* + * remove nfsnode from hash now so we can't accidentally find it + * again if another object gets created with the same filehandle + * before this vnode gets reclaimed + */ + LIST_REMOVE(VTONFS(vp), n_hash); + VTONFS(vp)->n_flag &= ~NHASHED; + } + vput(vp); + vput(dvp); + NFS_FREE_PNBUF(cnp); return (error); } @@ -2703,8 +2751,10 @@ nfs_readdir(ap) nfsstats.direofcache_hits++; return (0); } - /* directory changed, purge any name cache entries */ - cache_purge(vp); + if (np->n_ncmtime != vattr.va_mtime.tv_sec) { + /* directory changed, purge any name cache entries */ + cache_purge(vp); + } } } @@ -3114,8 +3164,7 @@ nfs_readdirplusrpc(vp, uiop, cred) for (cp = cnp->cn_nameptr, i = 1; i <= len; i++, cp++) cnp->cn_hash += (unsigned char)*cp * i; - if (cnp->cn_namelen <= NCHNAMLEN) - cache_enter(ndp->ni_dvp, ndp->ni_vp, cnp); + cache_enter(ndp->ni_dvp, ndp->ni_vp, cnp); } } else { /* Just skip over the file handle */ @@ -3203,7 +3252,7 @@ nfs_sillyrename(dvp, vp, cnp) struct ucred *cred; int i, j, k; - cache_purge(dvp); + cache_purge(vp); np = VTONFS(vp); #if DIAGNOSTIC if (vp->v_type == VDIR) @@ -3376,7 +3425,8 @@ nfs_commit(vp, offset, cnt, cred, procp) register int t1, t2; register struct nfsmount *nmp = VFSTONFS(vp->v_mount); caddr_t bpos, dpos, cp2; - int error = 0, wccflag = NFSV3_WCCRATTR; + int error = 0, wccpostattr = 0; + time_t premtime = 0; struct mbuf *mreq, *mrep, *md, *mb, *mb2; u_int64_t xid; @@ -3394,7 +3444,8 @@ nfs_commit(vp, offset, cnt, cred, procp) *tl = txdr_unsigned(cnt); nfsm_request(vp, NFSPROC_COMMIT, procp, cred, &xid); if (mrep) { - nfsm_wcc_data(vp, wccflag, &xid); + nfsm_wcc_data(vp, premtime, wccpostattr, &xid); + /* XXX can we do anything useful with the wcc info? */ } if (!error) { nfsm_dissect(tl, u_long *, NFSX_V3WRITEVERF); @@ -3460,7 +3511,7 @@ nfs_mmap(ap) } /* - * fsync vnode op. Just call nfs_flush() with commit == 1. + * fsync vnode op. Just call nfs_flush(). */ /* ARGSUSED */ static int @@ -3473,7 +3524,7 @@ nfs_fsync(ap) struct proc * a_p; } */ *ap; { - return (nfs_flush(ap->a_vp, ap->a_cred, ap->a_waitfor, ap->a_p, 1)); + return (nfs_flush(ap->a_vp, ap->a_cred, ap->a_waitfor, ap->a_p)); } int @@ -3645,7 +3696,7 @@ nfs_flushcommits(struct vnode *vp, struct proc *p) CHECK_NEEDCOMMITCNT(np); if (retv) { - nfs_buf_release(bp); + nfs_buf_release(bp, 1); } else { s = splbio(); vp->v_numoutput++; @@ -3691,12 +3742,11 @@ done: * associated with the vnode. */ static int -nfs_flush(vp, cred, waitfor, p, commit) +nfs_flush(vp, cred, waitfor, p) register struct vnode *vp; struct ucred *cred; int waitfor; struct proc *p; - int commit; { struct nfsnode *np = VTONFS(vp); struct nfsbuf *bp, *nbp; @@ -3704,7 +3754,7 @@ nfs_flush(vp, cred, waitfor, p, commit) int i, s, error = 0, error2, slptimeo = 0, slpflag = 0; int passone = 1; - FSDBG_TOP(517, vp, np, waitfor, commit); + FSDBG_TOP(517, vp, np, waitfor, 0); if (!nmp) { error = ENXIO; @@ -3712,12 +3762,13 @@ nfs_flush(vp, cred, waitfor, p, commit) } if (nmp->nm_flag & NFSMNT_INT) slpflag = PCATCH; - if (!commit) - passone = 0; /* - * On the first pass, commit all the bufs that can be. - * On the second pass, nfs_buf_write() is called to do the job. + * On the first pass, start async/unstable writes on all + * delayed write buffers. Then wait for all writes to complete + * and call nfs_flushcommits() to commit any uncommitted buffers. + * On all subsequent passes, start STABLE writes on any remaining + * dirty buffers. Then wait for all writes to complete. */ again: FSDBG(518, np->n_dirtyblkhd.lh_first, np->n_flag, 0, 0); @@ -3727,13 +3778,6 @@ again: error = ENXIO; goto done; } - if (NFS_ISV3(vp) && commit) { - /* loop while it looks like there are still buffers to be */ - /* commited and nfs_flushcommits() seems to be handling them. */ - while (np->n_needcommitcnt) - if (nfs_flushcommits(vp, p)) - break; - } /* Start/do any write(s) that are required. */ loop: @@ -3764,17 +3808,17 @@ loop: } if (!ISSET(bp->nb_flags, NB_DELWRI)) panic("nfs_fsync: not dirty"); - FSDBG(525, bp, passone, commit, bp->nb_flags); - if ((passone || !commit) && ISSET(bp->nb_flags, NB_NEEDCOMMIT)) + FSDBG(525, bp, passone, 0, bp->nb_flags); + if ((passone || (waitfor != MNT_WAIT)) && ISSET(bp->nb_flags, NB_NEEDCOMMIT)) continue; nfs_buf_remfree(bp); if (ISSET(bp->nb_flags, NB_ERROR)) { np->n_error = bp->nb_error ? bp->nb_error : EIO; np->n_flag |= NWRITEERR; - nfs_buf_release(bp); + nfs_buf_release(bp, 1); continue; } - if (passone || !commit) + if (passone) SET(bp->nb_flags, NB_BUSY|NB_ASYNC); else { /* the NB_STABLE forces this to be written FILESYNC */ @@ -3786,11 +3830,6 @@ loop: } splx(s); - if (passone) { - passone = 0; - goto again; - } - if (waitfor == MNT_WAIT) { while (vp->v_numoutput) { vp->v_flag |= VBWAIT; @@ -3809,10 +3848,27 @@ loop: } } } - if (np->n_dirtyblkhd.lh_first && commit) { - goto loop; + } + + if (NFS_ISV3(vp)) { + /* loop while it looks like there are still buffers to be */ + /* commited and nfs_flushcommits() seems to be handling them. */ + while (np->n_needcommitcnt) + if (nfs_flushcommits(vp, p)) + break; + } + + if (passone) { + passone = 0; + goto again; + } + + if (waitfor == MNT_WAIT) { + if (np->n_dirtyblkhd.lh_first) { + goto again; } } + FSDBG(526, np->n_flag, np->n_error, 0, 0); if (np->n_flag & NWRITEERR) { error = np->n_error; @@ -4037,7 +4093,7 @@ nfs_buf_write(struct nfsbuf *bp) splx(s); } FSDBG_BOT(553, bp, NBOFF(bp), bp->nb_flags, rv); - nfs_buf_release(bp); + nfs_buf_release(bp, 1); return (rv); } @@ -4610,7 +4666,7 @@ nfs_pageout(ap) np->n_needcommitcnt--; CHECK_NEEDCOMMITCNT(np); } - nfs_buf_release(bp); + nfs_buf_release(bp, 1); } splx(s); } diff --git a/bsd/nfs/nfsm_subs.h b/bsd/nfs/nfsm_subs.h index 1fba34823..cc1ac71b2 100644 --- a/bsd/nfs/nfsm_subs.h +++ b/bsd/nfs/nfsm_subs.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * @@ -239,25 +239,17 @@ struct mbuf *nfsm_rpchead __P((struct ucred *cr, int nmflag, int procid, (v) = ttvp; \ } } -/* Used as (f) for nfsm_wcc_data() */ -#define NFSV3_WCCRATTR 0 -#define NFSV3_WCCCHK 1 - -#define nfsm_wcc_data(v, f, x) \ - { int ttattrf, ttretf = 0; \ +#define nfsm_wcc_data(v, premtime, newpostattr, x) \ + { \ nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \ if (*tl == nfs_true) { \ nfsm_dissect(tl, u_long *, 6 * NFSX_UNSIGNED); \ - if (f) \ - ttretf = (VTONFS(v)->n_mtime == \ - fxdr_unsigned(u_long, *(tl + 2))); \ - } \ - nfsm_postop_attr((v), ttattrf, (x)); \ - if (f) { \ - (f) = ttretf; \ + (premtime) = fxdr_unsigned(time_t, *(tl + 2)); \ } else { \ - (f) = ttattrf; \ - } } + (premtime) = 0; \ + } \ + nfsm_postop_attr((v), (newpostattr), (x)); \ + } #define nfsm_v3sattr(s, a, u, g) \ { (s)->sa_modetrue = nfs_true; \ diff --git a/bsd/nfs/nfsnode.h b/bsd/nfs/nfsnode.h index e2f8f10bd..53f821217 100644 --- a/bsd/nfs/nfsnode.h +++ b/bsd/nfs/nfsnode.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * @@ -107,6 +107,7 @@ struct nfsbuf { LIST_ENTRY(nfsbuf) nb_vnbufs; /* vnode's nfsbuf chain */ TAILQ_ENTRY(nfsbuf) nb_free; /* free list position if not active. */ volatile long nb_flags; /* NB_* flags. */ + time_t nb_timestamp; /* buffer timestamp */ long nb_bufsize; /* buffer size */ daddr_t nb_lblkno; /* logical block number. */ int nb_error; /* errno value. */ @@ -167,9 +168,10 @@ TAILQ_HEAD(nfsbuffreehead, nfsbuf); #define NFSNOLIST ((struct nfsbuf *)0xdeadbeef) extern int nfsbufhashlock, nfsbufcnt, nfsbufmin, nfsbufmax; -extern int nfsbuffreecnt, nfsbufdelwricnt, nfsneedbuffer; +extern int nfsbuffreecnt, nfsbuffreemetacnt, nfsbufdelwricnt, nfsneedbuffer; extern int nfs_nbdwrite; extern struct nfsbuffreehead nfsbuffree, nfsbufdelwri; +extern time_t nfsbuffreeuptimestamp; #define NFSBUFCNTCHK() \ do { \ @@ -178,14 +180,17 @@ extern struct nfsbuffreehead nfsbuffree, nfsbufdelwri; (nfsbuffreecnt < 0) || \ (nfsbuffreecnt > nfsbufmax) || \ (nfsbuffreecnt > nfsbufcnt) || \ + (nfsbuffreemetacnt < 0) || \ + (nfsbuffreemetacnt > nfsbufmax) || \ + (nfsbuffreemetacnt > nfsbufcnt) || \ (nfsbufdelwricnt < 0) || \ (nfsbufdelwricnt > nfsbufmax) || \ (nfsbufdelwricnt > nfsbufcnt) || \ (nfs_nbdwrite < 0) || \ (nfs_nbdwrite > nfsbufcnt) || \ 0) \ - panic("nfsbuf count error: max %d cnt %d free %d delwr %d bdw %d\n", \ - nfsbufmax, nfsbufcnt, nfsbuffreecnt, \ + panic("nfsbuf count error: max %d cnt %d free %d meta %d delwr %d bdw %d\n", \ + nfsbufmax, nfsbufcnt, nfsbuffreecnt, nfsbuffreemetacnt, \ nfsbufdelwricnt, nfs_nbdwrite); \ } while (0) @@ -215,7 +220,7 @@ struct nfsnode { uid_t n_modeuid; /* credentials having mode */ time_t n_modestamp; /* mode cache timestamp */ time_t n_mtime; /* Prev modify time. */ - time_t n_ctime; /* Prev create time. */ + time_t n_ncmtime; /* namecache modify time. */ time_t n_expiry; /* Lease expiry time */ nfsfh_t *n_fhp; /* NFS File Handle */ union { @@ -332,10 +337,11 @@ struct nfsbuf * nfs_buf_incore(struct vnode *, daddr_t); struct nfsbuf * nfs_buf_get(struct vnode *, daddr_t, int, struct proc *, int); int nfs_buf_upl_setup(struct nfsbuf *bp); void nfs_buf_upl_check(struct nfsbuf *bp); -void nfs_buf_release(struct nfsbuf *); +void nfs_buf_release(struct nfsbuf *, int); int nfs_buf_iowait(struct nfsbuf *); void nfs_buf_iodone(struct nfsbuf *); void nfs_buf_write_delayed(struct nfsbuf *); +void nfs_buf_freeup(int); #endif /* KERNEL */ diff --git a/bsd/vfs/vfs_journal.c b/bsd/vfs/vfs_journal.c index ea0867aad..19c28f39a 100644 --- a/bsd/vfs/vfs_journal.c +++ b/bsd/vfs/vfs_journal.c @@ -977,22 +977,13 @@ replay_journal(journal *jnl) goto bad_replay; } - for(i=1,max_bsize=0; i < blhdr->num_blocks; i++) { + for(i=1; i < blhdr->num_blocks; i++) { if (blhdr->binfo[i].bnum < 0 && blhdr->binfo[i].bnum != (off_t)-1) { printf("jnl: replay_journal: bogus block number 0x%llx\n", blhdr->binfo[i].bnum); goto bad_replay; } - if (blhdr->binfo[i].bsize > max_bsize) { - max_bsize = blhdr->binfo[i].bsize; - } } - // make sure it's at least one page in size. - if (max_bsize & (PAGE_SIZE - 1)) { - max_bsize = (max_bsize + PAGE_SIZE) & ~(PAGE_SIZE - 1); - } - - //printf("jnl: replay_journal: adding %d blocks in journal entry @ 0x%llx to co_buf\n", // blhdr->num_blocks-1, jnl->jhdr->start); for(i=1; i < blhdr->num_blocks; i++) { @@ -1039,6 +1030,25 @@ replay_journal(journal *jnl) //printf("jnl: replay_journal: replaying %d blocks\n", num_full); + /* + * make sure it's at least one page in size, so + * start max_bsize at PAGE_SIZE + */ + for (i = 0, max_bsize = PAGE_SIZE; i < num_full; i++) { + + if (co_buf[i].block_num == (off_t)-1) + continue; + + if (co_buf[i].block_size > max_bsize) + max_bsize = co_buf[i].block_size; + } + /* + * round max_bsize up to the nearest PAGE_SIZE multiple + */ + if (max_bsize & (PAGE_SIZE - 1)) { + max_bsize = (max_bsize + PAGE_SIZE) & ~(PAGE_SIZE - 1); + } + if (kmem_alloc(kernel_map, (vm_offset_t *)&block_ptr, max_bsize)) { goto bad_replay; } diff --git a/config/IOKit.exports b/config/IOKit.exports index 980e8ca30..d47cc85c8 100644 --- a/config/IOKit.exports +++ b/config/IOKit.exports @@ -511,6 +511,7 @@ __ZN14IOPMrootDomain22changePowerStateToPrivEm __ZN14IOPMrootDomain23requestPowerDomainStateEmP17IOPowerConnectionm __ZN14IOPMrootDomain23setQuickSpinDownTimeoutEv __ZN14IOPMrootDomain24displayWranglerPublishedEPvS0_P9IOService +__ZN14IOPMrootDomain27registerPMSettingControllerEPFiiiPvES0_ __ZN14IOPMrootDomain24receivePowerNotificationEm __ZN14IOPMrootDomain25announcePowerSourceChangeEv __ZN14IOPMrootDomain26handleSleepTimerExpirationEv diff --git a/config/System.kext/Info.plist b/config/System.kext/Info.plist index d627662c5..6db4b9b55 100644 --- a/config/System.kext/Info.plist +++ b/config/System.kext/Info.plist @@ -5,7 +5,7 @@ CFBundleDevelopmentRegion English CFBundleGetInfoString - System Resource Pseudoextension, Apple Computer Inc, 7.4.0 + System Resource Pseudoextension, Apple Computer Inc, 7.5.0 CFBundleIdentifier com.apple.kernel CFBundleInfoDictionaryVersion @@ -15,13 +15,13 @@ CFBundlePackageType KEXT CFBundleShortVersionString - 7.4.0 + 7.5.0 CFBundleSignature ???? CFBundleVersion - 7.4.0 + 7.5.0 OSBundleCompatibleVersion - 7.4.0 + 7.5.0 OSBundleRequired Root OSKernelResource diff --git a/config/System.kext/PlugIns/AppleNMI.kext/Info.plist b/config/System.kext/PlugIns/AppleNMI.kext/Info.plist index b5f491792..237229b9a 100644 --- a/config/System.kext/PlugIns/AppleNMI.kext/Info.plist +++ b/config/System.kext/PlugIns/AppleNMI.kext/Info.plist @@ -5,7 +5,7 @@ CFBundleDevelopmentRegion English CFBundleGetInfoString - AppleNMI Pseudoextension, Apple Computer Inc, 7.4.0 + AppleNMI Pseudoextension, Apple Computer Inc, 7.5.0 CFBundleIdentifier com.apple.driver.AppleNMI CFBundleInfoDictionaryVersion @@ -15,11 +15,11 @@ CFBundlePackageType KEXT CFBundleShortVersionString - 7.4.0 + 7.5.0 CFBundleSignature ???? CFBundleVersion - 7.4.0 + 7.5.0 OSBundleRequired Root OSKernelResource diff --git a/config/System.kext/PlugIns/ApplePlatformFamily.kext/Info.plist b/config/System.kext/PlugIns/ApplePlatformFamily.kext/Info.plist index 32340be92..9e272bc09 100644 --- a/config/System.kext/PlugIns/ApplePlatformFamily.kext/Info.plist +++ b/config/System.kext/PlugIns/ApplePlatformFamily.kext/Info.plist @@ -5,7 +5,7 @@ CFBundleDevelopmentRegion English CFBundleGetInfoString - Apple Platform Family Pseudoextension, Apple Computer Inc, 7.4.0 + Apple Platform Family Pseudoextension, Apple Computer Inc, 7.5.0 CFBundleIdentifier com.apple.iokit.ApplePlatformFamily CFBundleInfoDictionaryVersion @@ -15,11 +15,11 @@ CFBundlePackageType KEXT CFBundleShortVersionString - 7.4.0 + 7.5.0 CFBundleSignature ???? CFBundleVersion - 7.4.0 + 7.5.0 OSBundleCompatibleVersion 1.0 OSBundleRequired diff --git a/config/System.kext/PlugIns/BSDKernel.kext/Info.plist b/config/System.kext/PlugIns/BSDKernel.kext/Info.plist index 654c1a02c..7fbbccf2a 100644 --- a/config/System.kext/PlugIns/BSDKernel.kext/Info.plist +++ b/config/System.kext/PlugIns/BSDKernel.kext/Info.plist @@ -7,7 +7,7 @@ CFBundleExecutable BSDKernel CFBundleGetInfoString - BSD Kernel Pseudoextension, Apple Computer Inc, 7.4.0 + BSD Kernel Pseudoextension, Apple Computer Inc, 7.5.0 CFBundleIdentifier com.apple.kpi.bsd CFBundleInfoDictionaryVersion @@ -17,11 +17,11 @@ CFBundlePackageType KEXT CFBundleShortVersionString - 7.4.0 + 7.5.0 CFBundleSignature ???? CFBundleVersion - 7.4.0 + 7.5.0 OSBundleCompatibleVersion 7.0 OSBundleRequired diff --git a/config/System.kext/PlugIns/IOKit.kext/Info.plist b/config/System.kext/PlugIns/IOKit.kext/Info.plist index 7f9f169f5..03981b908 100644 --- a/config/System.kext/PlugIns/IOKit.kext/Info.plist +++ b/config/System.kext/PlugIns/IOKit.kext/Info.plist @@ -7,7 +7,7 @@ CFBundleExecutable IOKit CFBundleGetInfoString - I/O Kit Pseudoextension, Apple Computer Inc, 7.4.0 + I/O Kit Pseudoextension, Apple Computer Inc, 7.5.0 CFBundleIdentifier com.apple.kpi.iokit CFBundleInfoDictionaryVersion @@ -17,11 +17,11 @@ CFBundlePackageType KEXT CFBundleShortVersionString - 7.4.0 + 7.5.0 CFBundleSignature ???? CFBundleVersion - 7.4.0 + 7.5.0 OSBundleCompatibleVersion 7.0 OSBundleRequired diff --git a/config/System.kext/PlugIns/IONVRAMFamily.kext/Info.plist b/config/System.kext/PlugIns/IONVRAMFamily.kext/Info.plist index 0b0176c7d..7f7d87098 100644 --- a/config/System.kext/PlugIns/IONVRAMFamily.kext/Info.plist +++ b/config/System.kext/PlugIns/IONVRAMFamily.kext/Info.plist @@ -5,7 +5,7 @@ CFBundleDevelopmentRegion English CFBundleGetInfoString - AppleNMI Pseudoextension, Apple Computer Inc, 7.4.0 + AppleNMI Pseudoextension, Apple Computer Inc, 7.5.0 CFBundleIdentifier com.apple.iokit.IONVRAMFamily CFBundleInfoDictionaryVersion @@ -15,11 +15,11 @@ CFBundlePackageType KEXT CFBundleShortVersionString - 7.4.0 + 7.5.0 CFBundleSignature ???? CFBundleVersion - 7.4.0 + 7.5.0 OSBundleCompatibleVersion 1.1 OSBundleRequired diff --git a/config/System.kext/PlugIns/IOSystemManagement.kext/Info.plist b/config/System.kext/PlugIns/IOSystemManagement.kext/Info.plist index ee7d23408..c0593199a 100644 --- a/config/System.kext/PlugIns/IOSystemManagement.kext/Info.plist +++ b/config/System.kext/PlugIns/IOSystemManagement.kext/Info.plist @@ -5,7 +5,7 @@ CFBundleDevelopmentRegion English CFBundleGetInfoString - System Management Pseudoextension, Apple Computer Inc, 7.4.0 + System Management Pseudoextension, Apple Computer Inc, 7.5.0 CFBundleIdentifier com.apple.iokit.IOSystemManagementFamily CFBundleInfoDictionaryVersion @@ -15,11 +15,11 @@ CFBundlePackageType KEXT CFBundleShortVersionString - 7.4.0 + 7.5.0 CFBundleSignature ???? CFBundleVersion - 7.4.0 + 7.5.0 OSBundleCompatibleVersion 1.0.0b1 OSBundleRequired diff --git a/config/System.kext/PlugIns/Libkern.kext/Info.plist b/config/System.kext/PlugIns/Libkern.kext/Info.plist index f84633b7f..a7e925151 100644 --- a/config/System.kext/PlugIns/Libkern.kext/Info.plist +++ b/config/System.kext/PlugIns/Libkern.kext/Info.plist @@ -7,7 +7,7 @@ CFBundleExecutable Libkern CFBundleGetInfoString - Libkern Pseudoextension, Apple Computer Inc, 7.4.0 + Libkern Pseudoextension, Apple Computer Inc, 7.5.0 CFBundleIdentifier com.apple.kpi.libkern CFBundleInfoDictionaryVersion @@ -17,11 +17,11 @@ CFBundlePackageType KEXT CFBundleShortVersionString - 7.4.0 + 7.5.0 CFBundleSignature ???? CFBundleVersion - 7.4.0 + 7.5.0 OSBundleCompatibleVersion 7.0 OSBundleRequired diff --git a/config/System.kext/PlugIns/Mach.kext/Info.plist b/config/System.kext/PlugIns/Mach.kext/Info.plist index 69237f362..e1f8c0468 100644 --- a/config/System.kext/PlugIns/Mach.kext/Info.plist +++ b/config/System.kext/PlugIns/Mach.kext/Info.plist @@ -7,7 +7,7 @@ CFBundleExecutable Mach CFBundleGetInfoString - Mach Kernel Pseudoextension, Apple Computer Inc, 7.4.0 + Mach Kernel Pseudoextension, Apple Computer Inc, 7.5.0 CFBundleIdentifier com.apple.kpi.mach CFBundleInfoDictionaryVersion @@ -17,11 +17,11 @@ CFBundlePackageType KEXT CFBundleShortVersionString - 7.4.0 + 7.5.0 CFBundleSignature ???? CFBundleVersion - 7.4.0 + 7.5.0 OSBundleCompatibleVersion 7.0 OSBundleRequired diff --git a/config/System6.0.exports b/config/System6.0.exports index 3c81dc92e..44c377dba 100644 --- a/config/System6.0.exports +++ b/config/System6.0.exports @@ -635,6 +635,7 @@ __ZN12IOUserClient24getTargetAndTrapForIndexEPP9IOServicem __ZN12IOUserClient24registerNotificationPortEP8ipc_portmm __ZN12IOUserClient25getExternalMethodForIndexEm __ZN12IOUserClient26getTargetAndMethodForIndexEPP9IOServicem +__ZN12IOUserClient26removeMappingForDescriptorEP18IOMemoryDescriptor __ZN12IOUserClient30getExternalAsyncMethodForIndexEm __ZN12IOUserClient31getAsyncTargetAndMethodForIndexEPP9IOServicem __ZN12IOUserClient4freeEv @@ -948,6 +949,7 @@ __ZN14IOPMrootDomain22changePowerStateToPrivEm __ZN14IOPMrootDomain23requestPowerDomainStateEmP17IOPowerConnectionm __ZN14IOPMrootDomain23setQuickSpinDownTimeoutEv __ZN14IOPMrootDomain24displayWranglerPublishedEPvS0_P9IOService +__ZN14IOPMrootDomain27registerPMSettingControllerEPFiiiPvES0_ __ZN14IOPMrootDomain24receivePowerNotificationEm __ZN14IOPMrootDomain25announcePowerSourceChangeEv __ZN14IOPMrootDomain26handleSleepTimerExpirationEv diff --git a/iokit/IOKit/IOUserClient.h b/iokit/IOKit/IOUserClient.h index b36415fe3..e6a891c50 100644 --- a/iokit/IOKit/IOUserClient.h +++ b/iokit/IOKit/IOUserClient.h @@ -162,6 +162,15 @@ public: task_t task, IOOptionBits mapFlags = kIOMapAnywhere, IOVirtualAddress atAddress = 0 ); + + /*! + @function removeMappingForDescriptor + Remove the first mapping created from the memory descriptor returned by clientMemoryForType() from IOUserClient's list of mappings. If such a mapping exists, it is retained and the reference currently held by IOUserClient is returned to the caller. + @param memory The memory descriptor instance previously returned by the implementation of clientMemoryForType(). + @result A reference to the first IOMemoryMap instance found in the list of mappings created by IOUserClient from that passed memory descriptor is returned, or zero if none exist. The caller should release this reference. + */ + IOMemoryMap * removeMappingForDescriptor(IOMemoryDescriptor * memory); + /*! @function exportObjectToClient Make an arbitrary OSObject available to the client task. diff --git a/iokit/IOKit/pwr_mgt/RootDomain.h b/iokit/IOKit/pwr_mgt/RootDomain.h index bf56b97a4..23eca1121 100644 --- a/iokit/IOKit/pwr_mgt/RootDomain.h +++ b/iokit/IOKit/pwr_mgt/RootDomain.h @@ -125,6 +125,9 @@ private: static bool displayWranglerPublished( void * target, void * refCon, IOService * newService); + static bool batteryLocationPublished( void * target, void * refCon, + IOService * resourceService ); + void setQuickSpinDownTimeout ( void ); void adjustPowerState( void ); void restoreUserSpinDownTimeout ( void ); @@ -153,10 +156,12 @@ private: // Private helper to call PM setting controller IOReturn setPMSetting(int type, OSNumber *); - + struct ExpansionData { - PMSettingCtrl *_settingController; - thread_call_t diskSyncCalloutEntry; + PMSettingCtrl *_settingController; + thread_call_t diskSyncCalloutEntry; + IONotifier *_batteryLocationNotifier; + IONotifier *_displayWranglerNotifier; }; ExpansionData *_reserved; IOOptionBits platformSleepSupport; diff --git a/iokit/Kernel/IOLib.c b/iokit/Kernel/IOLib.c index d21406437..3b6e3e957 100644 --- a/iokit/Kernel/IOLib.c +++ b/iokit/Kernel/IOLib.c @@ -69,7 +69,7 @@ static queue_head_t gIOMallocContiguousEntries; static mutex_t * gIOMallocContiguousEntriesLock; enum { kIOMaxPageableMaps = 16 }; -enum { kIOPageableMapSize = 16 * 1024 * 1024 }; +enum { kIOPageableMapSize = 96 * 1024 * 1024 }; enum { kIOPageableMaxMapSize = 96 * 1024 * 1024 }; typedef struct { diff --git a/iokit/Kernel/IOPMrootDomain.cpp b/iokit/Kernel/IOPMrootDomain.cpp index 8f15a10b7..1d18bec18 100644 --- a/iokit/Kernel/IOPMrootDomain.cpp +++ b/iokit/Kernel/IOPMrootDomain.cpp @@ -74,6 +74,9 @@ static IOPMPowerState ourPowerStates[number_of_power_states] = { // RESERVED IOPMrootDomain class variables #define diskSyncCalloutEntry _reserved->diskSyncCalloutEntry #define _settingController _reserved->_settingController +#define _batteryLocationNotifier _reserved->_batteryLocationNotifier +#define _displayWranglerNotifier _reserved->_displayWranglerNotifier + static IOPMrootDomain * gRootDomain; static UInt32 gSleepOrShutdownPending = 0; @@ -257,7 +260,13 @@ bool IOPMrootDomain::start ( IOService * nub ) registerPrioritySleepWakeInterest( &sysPowerDownHandler, this, 0); // Register for a notification when IODisplayWrangler is published - addNotification( gIOPublishNotification, serviceMatching("IODisplayWrangler"), &displayWranglerPublished, this, 0); + _displayWranglerNotifier = addNotification( gIOPublishNotification, + serviceMatching("IODisplayWrangler"), + &displayWranglerPublished, this, 0); + + _batteryLocationNotifier = addNotification( gIOPublishNotification, + resourceMatching("battery"), + &batteryLocationPublished, this, this); const OSSymbol *ucClassName = OSSymbol::withCStringNoCopy("RootDomainUserClient"); setProperty(gIOUserClientClassKey, (OSMetaClassBase *) ucClassName); @@ -734,6 +743,18 @@ void IOPMrootDomain::unIdleDevice( IOService *theDevice, unsigned long theState void IOPMrootDomain::announcePowerSourceChange( void ) { + IORegistryEntry *_batteryRegEntry = getProperty("BatteryEntry"); + + // (if possible) re-publish power source state under IOPMrootDomain + // (only done if the battery controller publishes an IOResource defining battery location) + if(_batteryRegEntry) + { + OSArray *batt_info; + batt_info = _batteryRegEntry->getProperty(kIOBatteryInfoKey); + if(batt_info) + setProperty(kIOBatteryInfoKey, batt_info); + } + messageClients(kIOPMMessageBatteryStatusHasChanged); } @@ -1270,6 +1291,31 @@ bool IOPMrootDomain::displayWranglerPublished( void * target, void * refCon, return true; } +//********************************************************************************* +// batteryLocationPublished +// +// Notification on AppleSMU publishing location of battery data +// +//********************************************************************************* + +bool IOPMrootDomain::batteryLocationPublished( void * target, void * root_domain, + IOService * resourceService ) +{ + IORegistryEntry *battery_location; + char battery_reg_path[255]; + int path_len = 255; + + battery_location = resourceService->getProperty("battery"); + if (!battery_location || !OSDynamicCast(IORegistryEntry, battery_location)) + return (true); + + ((IOPMrootDomain *)root_domain)->setProperty("BatteryEntry", battery_location); + + ((IOPMrootDomain *)root_domain)->announcePowerSourceChange(); + return (true); +} + + //********************************************************************************* // adjustPowerState diff --git a/iokit/Kernel/IOUserClient.cpp b/iokit/Kernel/IOUserClient.cpp index 9a05dc3bb..67e63d561 100644 --- a/iokit/Kernel/IOUserClient.cpp +++ b/iokit/Kernel/IOUserClient.cpp @@ -1965,6 +1965,33 @@ kern_return_t is_io_connect_map_memory( return( err ); } +IOMemoryMap * IOUserClient::removeMappingForDescriptor(IOMemoryDescriptor * mem) +{ + OSIterator * iter; + IOMemoryMap * map = 0; + + IOLockLock(gIOObjectPortLock); + + iter = OSCollectionIterator::withCollection(mappings); + if(iter) + { + while ((map = OSDynamicCast(IOMemoryMap, iter->getNextObject()))) + { + if(mem == map->getMemoryDescriptor()) + { + map->retain(); + mappings->removeObject(map); + break; + } + } + iter->release(); + } + + IOLockUnlock(gIOObjectPortLock); + + return (map); +} + kern_return_t is_io_connect_unmap_memory( io_object_t connect, int type, diff --git a/iokit/KernelConfigTables.cpp b/iokit/KernelConfigTables.cpp index 03f3444d8..97b740218 100644 --- a/iokit/KernelConfigTables.cpp +++ b/iokit/KernelConfigTables.cpp @@ -26,15 +26,15 @@ */ const char * gIOKernelKmods = "{" -" 'com.apple.kernel' = '7.4.0';" -" 'com.apple.kpi.bsd' = '7.4.0';" -" 'com.apple.kpi.iokit' = '7.4.0';" -" 'com.apple.kpi.libkern' = '7.4.0';" -" 'com.apple.kpi.mach' = '7.4.0';" -" 'com.apple.iokit.IONVRAMFamily' = '7.4.0';" -" 'com.apple.driver.AppleNMI' = '7.4.0';" -" 'com.apple.iokit.IOSystemManagementFamily' = '7.4.0';" -" 'com.apple.iokit.ApplePlatformFamily' = '7.4.0';" +" 'com.apple.kernel' = '7.5.0';" +" 'com.apple.kpi.bsd' = '7.5.0';" +" 'com.apple.kpi.iokit' = '7.5.0';" +" 'com.apple.kpi.libkern' = '7.5.0';" +" 'com.apple.kpi.mach' = '7.5.0';" +" 'com.apple.iokit.IONVRAMFamily' = '7.5.0';" +" 'com.apple.driver.AppleNMI' = '7.5.0';" +" 'com.apple.iokit.IOSystemManagementFamily' = '7.5.0';" +" 'com.apple.iokit.ApplePlatformFamily' = '7.5.0';" " 'com.apple.kernel.6.0' = '6.9.9';" " 'com.apple.kernel.bsd' = '6.9.9';" " 'com.apple.kernel.iokit' = '6.9.9';" diff --git a/iokit/conf/version.minor b/iokit/conf/version.minor index b8626c4cf..7ed6ff82d 100644 --- a/iokit/conf/version.minor +++ b/iokit/conf/version.minor @@ -1 +1 @@ -4 +5 diff --git a/libkern/conf/version.minor b/libkern/conf/version.minor index b8626c4cf..7ed6ff82d 100644 --- a/libkern/conf/version.minor +++ b/libkern/conf/version.minor @@ -1 +1 @@ -4 +5 diff --git a/libsa/conf/version.minor b/libsa/conf/version.minor index b8626c4cf..7ed6ff82d 100644 --- a/libsa/conf/version.minor +++ b/libsa/conf/version.minor @@ -1 +1 @@ -4 +5 diff --git a/osfmk/conf/kernelversion.minor b/osfmk/conf/kernelversion.minor index b8626c4cf..7ed6ff82d 100644 --- a/osfmk/conf/kernelversion.minor +++ b/osfmk/conf/kernelversion.minor @@ -1 +1 @@ -4 +5 diff --git a/osfmk/conf/version.minor b/osfmk/conf/version.minor index b8626c4cf..7ed6ff82d 100644 --- a/osfmk/conf/version.minor +++ b/osfmk/conf/version.minor @@ -1 +1 @@ -4 +5 diff --git a/osfmk/ppc/Diagnostics.c b/osfmk/ppc/Diagnostics.c index 92d3898c1..f51a66179 100644 --- a/osfmk/ppc/Diagnostics.c +++ b/osfmk/ppc/Diagnostics.c @@ -351,7 +351,14 @@ int diagCall(struct savearea *save) { cpu = cpu_number(); /* Get us */ if((sarea.scomcpu < NCPUS) && machine_slot[sarea.scomcpu].running) { - if(sarea.scomcpu == cpu) fwSCOM(&sarea); /* Do it if it is us */ + if(sarea.scomcpu == cpu) { /* Is it us? */ + if(sarea.scomfunc) { /* Are we writing */ + sarea.scomstat = ml_scom_write(sarea.scomreg, sarea.scomdata); /* Write scom */ + } + else { + sarea.scomstat = ml_scom_read(sarea.scomreg, &sarea.scomdata); /* Read scom */ + } + } else { /* Otherwise, tell the other processor */ (void)cpu_signal(sarea.scomcpu, SIGPcpureq, CPRQscom ,(unsigned int)&sarea); /* Ask him to do this */ (void)hw_cpu_sync((unsigned long)&sarea.scomstat, LockTimeOut); /* Wait for the other processor to get its temperature */ diff --git a/osfmk/ppc/asm.h b/osfmk/ppc/asm.h index b5ff7de82..7c89f3c30 100644 --- a/osfmk/ppc/asm.h +++ b/osfmk/ppc/asm.h @@ -237,6 +237,8 @@ #define parm 0x01000000 #define sten 7 #define stenm 0x01000000 +#define dnap 7 +#define dnapm 0x01000000 #define doze 8 #define dozem 0x00800000 #define nap 9 @@ -459,6 +461,10 @@ #define GUSMstgttim 0x00000038 #define GUSMstgttoff 0x00000004 +; PowerTune +#define PowerTuneControlReg 0x0AA001 +#define PowerTuneStatusReg 0x408001 + ; HID4 #define hid4RMCI 23 #define hid4FAlgn 24 diff --git a/osfmk/ppc/chud/chud_spr.h b/osfmk/ppc/chud/chud_spr.h index de5e6ee6c..69e4bb655 100644 --- a/osfmk/ppc/chud/chud_spr.h +++ b/osfmk/ppc/chud/chud_spr.h @@ -234,7 +234,7 @@ /* PPC SPRs - 7400/7410 Specific */ #define chud_7400_msscr1 1015 - + /* PPC SPRs - 64-bit implementations */ #define chud_ppc64_accr 29 #define chud_ppc64_ctrl 152 diff --git a/osfmk/ppc/cpu.c b/osfmk/ppc/cpu.c index 3e66ab9c0..9e3c01ee1 100644 --- a/osfmk/ppc/cpu.c +++ b/osfmk/ppc/cpu.c @@ -586,7 +586,12 @@ cpu_signal_handler( return; case CPRQscom: - fwSCOM((scomcomm *)holdParm2); /* Do the function */ + if(((scomcomm *)holdParm2)->scomfunc) { /* Are we writing */ + ((scomcomm *)holdParm2)->scomstat = ml_scom_write(((scomcomm *)holdParm2)->scomreg, ((scomcomm *)holdParm2)->scomdata); /* Write scom */ + } + else { /* No, reading... */ + ((scomcomm *)holdParm2)->scomstat = ml_scom_read(((scomcomm *)holdParm2)->scomreg, &((scomcomm *)holdParm2)->scomdata); /* Read scom */ + } return; default: diff --git a/osfmk/ppc/exception.h b/osfmk/ppc/exception.h index 3bb18e5ed..96168ab05 100644 --- a/osfmk/ppc/exception.h +++ b/osfmk/ppc/exception.h @@ -63,10 +63,6 @@ struct procFeatures { #define pfCanNapb 5 #define pfCanDoze 0x02000000 #define pfCanDozeb 6 -#define pfThermal 0x01000000 -#define pfThermalb 7 -#define pfThermInt 0x00800000 -#define pfThermIntb 8 #define pfSlowNap 0x00400000 #define pfSlowNapb 9 #define pfNoMuMMCK 0x00200000 @@ -139,23 +135,14 @@ struct procFeatures { #define pmDFSb 30 #define pmDualPLL 0x00000001 #define pmDualPLLb 31 + unsigned int pfPowerTune0; /* 0x080 */ + unsigned int pfPowerTune1; /* 0x084 */ + unsigned int rsrvd88[6]; /* 0x088 */ }; #pragma pack() typedef struct procFeatures procFeatures; -#pragma pack(4) /* Make sure the structure stays as we defined it */ -struct thrmControl { - unsigned int maxTemp; /* Maximum temprature before damage */ - unsigned int throttleTemp; /* Temprature at which to throttle down */ - unsigned int lowTemp; /* Interrupt when temprature drops below */ - unsigned int highTemp; /* Interrupt when temprature exceeds this */ - unsigned int thrm3val; /* Value for thrm3 register */ - unsigned int rsvd[3]; /* Pad to cache line */ -}; -#pragma pack() - -typedef struct thrmControl thrmControl; /* * @@ -352,9 +339,6 @@ struct per_proc_info { /* PPC cache line boundary here - 0A0 */ procFeatures pf; /* Processor features */ - /* PPC cache line boundary here - 120 */ - thrmControl thrm; /* Thermal controls */ - /* PPC cache line boundary here - 140 */ unsigned int ppRsvd140[8]; /* Reserved */ diff --git a/osfmk/ppc/genassym.c b/osfmk/ppc/genassym.c index 9f7c26350..7e2f5ecd6 100644 --- a/osfmk/ppc/genassym.c +++ b/osfmk/ppc/genassym.c @@ -168,10 +168,6 @@ int main(int argc, char *argv[]) DECLARE("pfCanNapb", pfCanNapb); DECLARE("pfCanDoze", pfCanDoze); DECLARE("pfCanDozeb", pfCanDozeb); - DECLARE("pfThermal", pfThermal); - DECLARE("pfThermalb", pfThermalb); - DECLARE("pfThermInt", pfThermInt); - DECLARE("pfThermIntb", pfThermIntb); DECLARE("pfSlowNap", pfSlowNap); DECLARE("pfSlowNapb", pfSlowNapb); DECLARE("pfNoMuMMCK", pfNoMuMMCK); @@ -233,6 +229,8 @@ int main(int argc, char *argv[]) DECLARE("pfl3crOriginal", offsetof(struct per_proc_info *, pf.l3crOriginal)); DECLARE("pfBootConfig", offsetof(struct per_proc_info *, pf.pfBootConfig)); DECLARE("pfPowerModes", offsetof(struct per_proc_info *, pf.pfPowerModes)); + DECLARE("pfPowerTune0", offsetof(struct per_proc_info *, pf.pfPowerTune0)); + DECLARE("pfPowerTune1", offsetof(struct per_proc_info *, pf.pfPowerTune1)); DECLARE("pmDPLLVmin", pmDPLLVmin); DECLARE("pmDPLLVminb", pmDPLLVminb); DECLARE("pmPowerTune", pmPowerTune); @@ -246,13 +244,6 @@ int main(int argc, char *argv[]) DECLARE("pfMaxPAddr", offsetof(struct per_proc_info *, pf.pfMaxPAddr)); DECLARE("pfSize", sizeof(procFeatures)); - DECLARE("thrmmaxTemp", offsetof(struct per_proc_info *, thrm.maxTemp)); - DECLARE("thrmthrottleTemp", offsetof(struct per_proc_info *, thrm.throttleTemp)); - DECLARE("thrmlowTemp", offsetof(struct per_proc_info *, thrm.lowTemp)); - DECLARE("thrmhighTemp", offsetof(struct per_proc_info *, thrm.highTemp)); - DECLARE("thrm3val", offsetof(struct per_proc_info *, thrm.thrm3val)); - DECLARE("thrmSize", sizeof(thrmControl)); - DECLARE("validSegs", offsetof(struct per_proc_info *, validSegs)); DECLARE("ppUserPmapVirt", offsetof(struct per_proc_info *, ppUserPmapVirt)); DECLARE("ppUserPmap", offsetof(struct per_proc_info *, ppUserPmap)); diff --git a/osfmk/ppc/interrupt.c b/osfmk/ppc/interrupt.c index 498cad696..7924041cd 100644 --- a/osfmk/ppc/interrupt.c +++ b/osfmk/ppc/interrupt.c @@ -45,8 +45,7 @@ struct savearea * interrupt( unsigned int dsisr, unsigned int dar) { - int current_cpu, tmpr, targtemp; - unsigned int throttle; + int current_cpu; uint64_t now; thread_act_t act; @@ -68,48 +67,6 @@ struct savearea * interrupt( switch (type) { - case T_THERMAL: /* Fix the air conditioning, I'm dripping with sweat, or freezing, whatever... */ - -/* - * Note that this code is just a hackification until we have a real thermal plan. - */ - - tmpr = ml_read_temp(); /* Find out just how hot it is */ - targtemp = (dar >> 23) & 0x7F; /* Get the temprature we were looking for */ - if(dar & 4) { /* Did the temprature drop down? */ -#if 1 - kprintf("THERMAL below (cpu %d) target = %d; actual = %d; thrm = %08X\n", current_cpu, targtemp, tmpr, dar); -#endif -#if 0 - throttle = ml_throttle(0); /* Set throttle off */ -#if 1 - kprintf("THERMAL (cpu %d) throttle set off; last = %d\n", current_cpu, throttle); -#endif -#endif - ml_thrm_set(0, per_proc_info[current_cpu].thrm.throttleTemp); /* Set no low temp and max allowable as max */ - -#if 1 - kprintf("THERMAL (cpu %d) temp set to: off min, %d max\n", current_cpu, per_proc_info[current_cpu].thrm.throttleTemp); -#endif - } - else { -#if 1 - kprintf("THERMAL above (cpu %d) target = %d; actual = %d; thrm = %08X\n", current_cpu, targtemp, tmpr, dar); -#endif -#if 0 - throttle = ml_throttle(32); /* Set throttle on about 1/8th */ -#if 1 - kprintf("THERMAL (cpu %d) throttle set to 32; last = %d\n", current_cpu, throttle); -#endif -#endif - ml_thrm_set(per_proc_info[current_cpu].thrm.throttleTemp - 4, 0); /* Set low temp to max - 4 and max off */ -#if 1 - kprintf("THERMAL (cpu %d) temp set to: %d min, off max\n", current_cpu, per_proc_info[current_cpu].thrm.throttleTemp - 4); -#endif - - } - break; - case T_DECREMENTER: KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE, isync_mfdec(), (unsigned int)ssp->save_srr0, 0, 0, 0); diff --git a/osfmk/ppc/lowmem_vectors.s.orig b/osfmk/ppc/lowmem_vectors.s.orig new file mode 100644 index 000000000..6eb7f9f88 --- /dev/null +++ b/osfmk/ppc/lowmem_vectors.s.orig @@ -0,0 +1,3403 @@ +/* + * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * The contents of this file constitute Original Code as defined in and + * are subject to the Apple Public Source License Version 1.1 (the + * "License"). You may not use this file except in compliance with the + * License. Please obtain a copy of the License at + * http://www.apple.com/publicsource and read it before using this file. + * + * This Original Code and all software distributed under the License are + * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the + * License for the specific language governing rights and limitations + * under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#define ESPDEBUG 0 +#define INSTRUMENT 0 + +#define featAltivec 29 +#define wasNapping 30 + +#define VECTOR_SEGMENT .section __VECTORS, __interrupts + + VECTOR_SEGMENT + + + .globl EXT(ExceptionVectorsStart) + +EXT(ExceptionVectorsStart): /* Used if relocating the exception vectors */ +baseR: /* Used so we have more readable code */ + +; +; Handle system reset. +; We do not ever expect a hard reset so we do not actually check. +; When we come here, we check for a RESET_HANDLER_START (which means we are +; waking up from sleep), a RESET_HANDLER_BUPOR (which is using for bring up +; when starting directly from a POR), and RESET_HANDLER_IGNORE (which means +; ignore the interrupt). +; +; Some machines (so far, 32-bit guys) will always ignore a non-START interrupt. +; The ones who do take it, check if the interrupt is too be ignored. This is +; always the case until the previous reset is handled (i.e., we have exited +; from the debugger). +; + . = 0xf0 + .globl EXT(ResetHandler) +EXT(ResetHandler): + .long 0x0 + .long 0x0 + .long 0x0 + + . = 0x100 +.L_handler100: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + lwz r13,lo16(EXT(ResetHandler)-EXT(ExceptionVectorsStart)+RESETHANDLER_TYPE)(br0) ; Get reset type + mfcr r11 + cmpi cr0,r13,RESET_HANDLER_START + bne resetexc + + li r11,RESET_HANDLER_NULL + stw r11,lo16(EXT(ResetHandler)-EXT(ExceptionVectorsStart)+RESETHANDLER_TYPE)(br0) ; Clear reset type + + lwz r4,lo16(EXT(ResetHandler)-EXT(ExceptionVectorsStart)+RESETHANDLER_CALL)(br0) + lwz r3,lo16(EXT(ResetHandler)-EXT(ExceptionVectorsStart)+RESETHANDLER_ARG)(br0) + mtlr r4 + blr + +resetexc: cmplwi r13,RESET_HANDLER_BUPOR ; Special bring up POR sequence? + bne resetexc2 ; No... + lis r4,hi16(EXT(resetPOR)) ; Get POR code + ori r4,r4,lo16(EXT(resetPOR)) ; The rest + mtlr r4 ; Set it + blr ; Jump to it.... + +resetexc2: cmplwi cr1,r13,RESET_HANDLER_IGNORE ; Are we ignoring these? (Software debounce) + + mfsprg r13,0 ; Get per_proc + lwz r13,pfAvailable(r13) ; Get the features + rlwinm. r13,r13,0,pf64Bitb,pf64Bitb ; Is this a 64-bit machine? + cror cr1_eq,cr0_eq,cr1_eq ; See if we want to take this + bne-- cr1,rxCont ; Yes, continue... + bne-- rxIg64 ; 64-bit path... + + mtcr r11 ; Restore the CR + mfsprg r13,2 ; Restore R13 + mfsprg r11,0 ; Get per_proc + lwz r11,pfAvailable(r11) ; Get the features + mtsprg 2,r11 ; Restore sprg2 + mfsprg r11,3 ; Restore R11 + rfi ; Return and ignore the reset + +rxIg64: mtcr r11 ; Restore the CR + mfsprg r11,0 ; Get per_proc + mtspr hsprg0,r14 ; Save a register + lwz r14,UAW(r11) ; Get the User Assist Word + mfsprg r13,2 ; Restore R13 + lwz r11,pfAvailable(r11) ; Get the features + mtsprg 2,r11 ; Restore sprg2 + mfsprg r11,3 ; Restore R11 + mtsprg 3,r14 ; Set the UAW in sprg3 + mfspr r14,hsprg0 ; Restore R14 + rfid ; Return and ignore the reset + +rxCont: mtcr r11 + li r11,RESET_HANDLER_IGNORE ; Get set to ignore + stw r11,lo16(EXT(ResetHandler)-EXT(ExceptionVectorsStart)+RESETHANDLER_TYPE)(br0) ; Start ignoring these + mfsprg r13,1 /* Get the exception save area */ + li r11,T_RESET /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + +/* + * Machine check + */ + + . = 0x200 +.L_handler200: + mtsprg 2,r13 ; Save R13 + mtsprg 3,r11 ; Save R11 + + .globl EXT(extPatchMCK) +LEXT(extPatchMCK) ; This is patched to a nop for 64-bit + b h200aaa ; Skip 64-bit code... + +; +; Fall through here for 970 MCKs. +; + + li r11,1 ; ? + sldi r11,r11,32+3 ; ? + mfspr r13,hid4 ; ? + or r11,r11,r13 ; ? + sync + mtspr hid4,r11 ; ? + isync + li r11,1 ; ? + sldi r11,r11,32+8 ; ? + andc r13,r13,r11 ; ? + lis r11,0xE000 ; Get the unlikeliest ESID possible + sync + mtspr hid4,r13 ; ? + isync ; ? + + srdi r11,r11,1 ; ? + slbie r11 ; ? + sync + isync + + li r11,T_MACHINE_CHECK ; Set rupt code + b .L_exception_entry ; Join common... + +; +; Preliminary checking of other MCKs +; + +h200aaa: mfsrr1 r11 ; Get the SRR1 + mfcr r13 ; Save the CR + + rlwinm. r11,r11,0,dcmck,dcmck ; ? + beq+ notDCache ; ? + + sync + mfspr r11,msscr0 ; ? + dssall ; ? + sync + isync + + oris r11,r11,hi16(dl1hwfm) ; ? + mtspr msscr0,r11 ; ? + +rstbsy: mfspr r11,msscr0 ; ? + + rlwinm. r11,r11,0,dl1hwf,dl1hwf ; ? + bne rstbsy ; ? + + sync ; ? + + mfsprg r11,0 ; Get the per_proc + mtcrf 255,r13 ; Restore CRs + lwz r13,hwMachineChecks(r11) ; Get old count + addi r13,r13,1 ; Count this one + stw r13,hwMachineChecks(r11) ; Set new count + lwz r11,pfAvailable(r11) ; Get the feature flags + mfsprg r13,2 ; Restore R13 + mtsprg 2,r11 ; Set the feature flags + mfsprg r11,3 ; Restore R11 + rfi ; Return + +notDCache: mtcrf 255,r13 ; Restore CRs + li r11,T_MACHINE_CHECK ; Set rupt code + b .L_exception_entry ; Join common... + + +/* + * Data access - page fault, invalid memory rights for operation + */ + + . = 0x300 +.L_handler300: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_DATA_ACCESS /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + + +/* + * Data segment + */ + + . = 0x380 +.L_handler380: + mtsprg 2,r13 ; Save R13 + mtsprg 3,r11 ; Save R11 + li r11,T_DATA_SEGMENT ; Set rupt code + b .L_exception_entry ; Join common... + +/* + * Instruction access - as for data access + */ + + . = 0x400 +.L_handler400: + mtsprg 2,r13 ; Save R13 + mtsprg 3,r11 ; Save R11 + li r11,T_INSTRUCTION_ACCESS ; Set rupt code + b .L_exception_entry ; Join common... + +/* + * Instruction segment + */ + + . = 0x480 +.L_handler480: + mtsprg 2,r13 ; Save R13 + mtsprg 3,r11 ; Save R11 + li r11,T_INSTRUCTION_SEGMENT ; Set rupt code + b .L_exception_entry ; Join common... + +/* + * External interrupt + */ + + . = 0x500 +.L_handler500: + mtsprg 2,r13 ; Save R13 + mtsprg 3,r11 ; Save R11 + li r11,T_INTERRUPT ; Set rupt code + b .L_exception_entry ; Join common... + +/* + * Alignment - many reasons + */ + + . = 0x600 +.L_handler600: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_ALIGNMENT|T_FAM /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + +/* + * Program - floating point exception, illegal inst, priv inst, user trap + */ + + . = 0x700 +.L_handler700: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + +#if 0 + mfsrr1 r13 ; (BRINGUP) + mfcr r11 ; (BRINGUP) + rlwinm. r13,r13,0,12,12 ; (BRINGUP) + crmove cr1_eq,cr0_eq ; (BRINGUP) + mfsrr1 r13 ; (BRINGUP) + rlwinm. r13,r13,0,MSR_PR_BIT,MSR_PR_BIT ; (BRINGUP) + crorc cr0_eq,cr1_eq,cr0_eq ; (BRINGUP) + bf-- cr0_eq,. ; (BRINGUP) + mtcrf 255,r11 ; (BRINGUP) +#endif + + li r11,T_PROGRAM|T_FAM /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + +/* + * Floating point disabled + */ + + . = 0x800 +.L_handler800: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_FP_UNAVAILABLE /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + + +/* + * Decrementer - DEC register has passed zero. + */ + + . = 0x900 +.L_handler900: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_DECREMENTER /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + +/* + * I/O controller interface error - MACH does not use this + */ + + . = 0xA00 +.L_handlerA00: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_IO_ERROR /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + +/* + * Reserved + */ + + . = 0xB00 +.L_handlerB00: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_RESERVED /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + +; +; System call - generated by the sc instruction +; +; We handle the ultra-fast traps right here. They are: +; +; 0xFFFFFFFF - BlueBox only - MKIsPreemptiveTask +; 0xFFFFFFFE - BlueBox only - kcNKIsPreemptiveTaskEnv +; 0x00007FF2 - User state only - thread info +; 0x00007FF3 - User state only - floating point / vector facility status +; 0x00007FF4 - Kernel only - loadMSR - not used on 64-bit machines +; +; Note: none handled if virtual machine is running +; Also, it we treat SCs as kernel SCs if the RI bit is set +; + + . = 0xC00 +.L_handlerC00: + mtsprg 3,r11 ; Save R11 + mfsprg r11,2 ; Get the feature flags + + mtsprg 2,r13 ; Save R13 + rlwinm r11,r11,pf64Bitb-4,4,4 ; Get the 64-bit flag + mfsrr1 r13 ; Get SRR1 for loadMSR + rlwimi r11,r13,MSR_PR_BIT-5,5,5 ; Move the PR bit to bit 1 + mfcr r13 ; Save the CR + + mtcrf 0x40,r11 ; Get the top 3 CR bits to 64-bit, PR, sign + + cmpwi r0,lo16(-3) ; Eliminate all negatives but -1 and -2 + mfsprg r11,0 ; Get the per_proc + bf-- 5,uftInKern ; We came from the kernel... + ble-- notufp ; This is a mach call + + lwz r11,spcFlags(r11) ; Pick up the special flags + + cmpwi cr7,r0,lo16(-1) ; Is this a BlueBox call? + cmplwi cr2,r0,0x7FF2 ; Ultra fast path cthread info call? + cmplwi cr3,r0,0x7FF3 ; Ultra fast path facility status? + cror cr4_eq,cr2_eq,cr3_eq ; Is this one of the two ufts we handle here? + + ble-- cr7,uftBBCall ; We think this is blue box call... + + rlwinm r11,r11,16,16,31 ; Extract spcFlags upper bits + andi. r11,r11,hi16(runningVM|FamVMena|FamVMmode) + cmpwi cr0,r11,hi16(runningVM|FamVMena|FamVMmode) ; Test in VM FAM + beq-- cr0,ufpVM ; fast paths running VM ... + + bne-- cr4_eq,notufp ; Bail ifthis is not a uft... + +; +; Handle normal user ultra-fast trap +; + + li r3,spcFlags ; Assume facility status - 0x7FF3 + + beq-- cr3,uftFacStat ; This is a facilities status call... + + li r3,UAW ; This is really a thread info call - 0x7FF2 + +uftFacStat: mfsprg r11,0 ; Get the per_proc + lwzx r3,r11,r3 ; Get the UAW or spcFlags field + +uftExit: bt++ 4,uftX64 ; Go do the 64-bit exit... + + lwz r11,pfAvailable(r11) ; Get the feature flags + mtcrf 255,r13 ; Restore the CRs + mfsprg r13,2 ; Restore R13 + mtsprg 2,r11 ; Set the feature flags + mfsprg r11,3 ; Restore R11 + + rfi ; Back to our guy... + +uftX64: mtspr hsprg0,r14 ; Save a register + + lwz r14,UAW(r11) ; Get the User Assist Word + lwz r11,pfAvailable(r11) ; Get the feature flags + + mtcrf 255,r13 ; Restore the CRs + + mfsprg r13,2 ; Restore R13 + mtsprg 2,r11 ; Set the feature flags + mfsprg r11,3 ; Restore R11 + mtsprg 3,r14 ; Set the UAW in sprg3 + mfspr r14,hsprg0 ; Restore R14 + + rfid ; Back to our guy... + +; +; Handle BlueBox ultra-fast trap +; + +uftBBCall: andi. r11,r11,bbNoMachSC|bbPreemptive ; Clear what we do not need + cmplwi r11,bbNoMachSC ; See if we are trapping syscalls + blt-- notufp ; No... + + rlwimi r13,r11,bbPreemptivebit-cr0_eq,cr0_eq,cr0_eq ; Copy preemptive task flag into user cr0_eq + + mfsprg r11,0 ; Get the per proc + + beq++ cr7,uftExit ; For MKIsPreemptiveTask we are done... + + lwz r0,ppbbTaskEnv(r11) ; Get the shadowed taskEnv from per_proc_area + b uftExit ; We are really all done now... + +; Kernel ultra-fast trap + +uftInKern: cmplwi r0,0x7FF4 ; Ultra fast path loadMSR? + bne- notufp ; Someone is trying to cheat... + + mtsrr1 r3 ; Set new MSR + + b uftExit ; Go load the new MSR... + +notufp: mtcrf 0xFF,r13 ; Restore the used CRs + li r11,T_SYSTEM_CALL|T_FAM ; Set interrupt code + b .L_exception_entry ; Join common... + + + + + +/* + * Trace - generated by single stepping + * performance monitor BE branch enable tracing/logging + * is also done here now. while this is permanently in the + * system the impact is completely unnoticable as this code is + * only executed when (a) a single step or branch exception is + * hit, (b) in the single step debugger case there is so much + * overhead already the few extra instructions for testing for BE + * are not even noticable, (c) the BE logging code is *only* run + * when it is enabled by the tool which will not happen during + * normal system usage + * + * Note that this trace is available only to user state so we do not + * need to set sprg2 before returning. + */ + + . = 0xD00 +.L_handlerD00: + mtsprg 3,r11 ; Save R11 + mfsprg r11,2 ; Get the feature flags + mtsprg 2,r13 ; Save R13 + rlwinm r11,r11,pf64Bitb-4,4,4 ; Get the 64-bit flag + mfcr r13 ; Get the CR + mtcrf 0x40,r11 ; Set the CR + mfsrr1 r11 ; Get the old MSR + rlwinm. r11,r11,0,MSR_PR_BIT,MSR_PR_BIT ; Are we in supervisor state? + + mfsprg r11,0 ; Get the per_proc + lhz r11,PP_CPU_FLAGS(r11) ; Get the flags + crmove cr1_eq,cr0_eq ; Remember if we are in supervisor state + rlwinm. r11,r11,0,traceBEb+16,traceBEb+16 ; Special trace enabled? + cror cr0_eq,cr0_eq,cr1_eq ; Is trace off or supervisor state? + bf-- cr0_eq,specbrtr ; No, we need to trace... + +notspectr: mtcr r13 ; Restore CR + li r11,T_TRACE|T_FAM ; Set interrupt code + b .L_exception_entry ; Join common... + + .align 5 + +; +; We are doing the special branch trace +; + +specbrtr: mfsprg r11,0 ; Get the per_proc area + bt++ 4,sbxx64a ; Jump if 64-bit... + + stw r1,tempr0+4(r11) ; Save in a scratch area + stw r2,tempr1+4(r11) ; Save in a scratch area + stw r3,tempr2+4(r11) ; Save in a scratch area + b sbxx64b ; Skip... + +sbxx64a: std r1,tempr0(r11) ; Save in a scratch area + std r2,tempr1(r11) ; Save in a scratch area + std r3,tempr2(r11) ; Save in a scratch area + +sbxx64b: lis r2,hi16(EXT(pc_trace_buf)) ; Get the top of the buffer + lwz r3,spcTRp(r11) ; Pick up buffer position + ori r2,r2,lo16(EXT(pc_trace_buf)) ; Get the bottom of the buffer + cmplwi cr2,r3,4092 ; Set cr1_eq if we should take exception + mfsrr0 r1 ; Get the pc + stwx r1,r2,r3 ; Save it in the buffer + addi r3,r3,4 ; Point to the next slot + rlwinm r3,r3,0,20,31 ; Wrap the slot at one page + stw r3,spcTRp(r11) ; Save the new slot + + bt++ 4,sbxx64c ; Jump if 64-bit... + + lwz r1,tempr0+4(r11) ; Restore work register + lwz r2,tempr1+4(r11) ; Restore work register + lwz r3,tempr2+4(r11) ; Restore work register + beq cr2,notspectr ; Buffer filled, make a rupt... + b uftExit ; Go restore and leave... + +sbxx64c: ld r1,tempr0(r11) ; Restore work register + ld r2,tempr1(r11) ; Restore work register + ld r3,tempr2(r11) ; Restore work register + beq cr2,notspectr ; Buffer filled, make a rupt... + b uftExit ; Go restore and leave... + +/* + * Floating point assist + */ + + . = 0xE00 +.L_handlerE00: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_FP_ASSIST /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + + +/* + * Performance monitor interruption + */ + + . = 0xF00 +PMIhandler: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_PERF_MON /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + + +/* + * VMX exception + */ + + . = 0xF20 +VMXhandler: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_VMX /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + + + +; +; Instruction translation miss exception - not supported +; + + . = 0x1000 +.L_handler1000: + mtsprg 2,r13 ; Save R13 + mtsprg 3,r11 ; Save R11 + li r11,T_INVALID_EXCP0 ; Set rupt code + b .L_exception_entry ; Join common... + + + +; +; Data load translation miss exception - not supported +; + + . = 0x1100 +.L_handler1100: + mtsprg 2,r13 ; Save R13 + mtsprg 3,r11 ; Save R11 + li r11,T_INVALID_EXCP1 ; Set rupt code + b .L_exception_entry ; Join common... + + + +; +; Data store translation miss exception - not supported +; + + . = 0x1200 +.L_handler1200: + mtsprg 2,r13 ; Save R13 + mtsprg 3,r11 ; Save R11 + li r11,T_INVALID_EXCP2 ; Set rupt code + b .L_exception_entry ; Join common... + + +/* + * Instruction address breakpoint + */ + + . = 0x1300 +.L_handler1300: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_INSTRUCTION_BKPT /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + +/* + * System management interrupt + */ + + . = 0x1400 +.L_handler1400: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_SYSTEM_MANAGEMENT /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + + +/* + * Soft Patch + */ + + . = 0x1500 +.L_handler1500: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_SOFT_PATCH /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + +; +; Altivec Java Mode Assist interrupt or Maintenace interrupt +; + + . = 0x1600 +.L_handler1600: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_ALTIVEC_ASSIST /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + +; +; Altivec Java Mode Assist interrupt or Thermal interruption +; + + . = 0x1700 +.L_handler1700: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_THERMAL /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + +; +; Thermal interruption - 64-bit +; + + . = 0x1800 +.L_handler1800: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_ARCHDEP0 /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + +/* + * There is now a large gap of reserved traps + */ + +/* + * Instrumentation interruption + */ + + . = 0x2000 +.L_handler2000: + mtsprg 2,r13 /* Save R13 */ + mtsprg 3,r11 /* Save R11 */ + li r11,T_INSTRUMENTATION /* Set 'rupt code */ + b .L_exception_entry /* Join common... */ + + . = 0x2100 + +/* + * Filter Ultra Fast Path syscalls for VMM + */ +ufpVM: + cmpwi cr2,r0,0x6004 ; Is it vmm_dispatch + bne cr2,notufp ; Exit If not + cmpwi cr5,r3,kvmmResumeGuest ; Compare r3 with kvmmResumeGuest + cmpwi cr2,r3,kvmmSetGuestRegister ; Compare r3 with kvmmSetGuestRegister + cror cr1_eq,cr5_lt,cr2_gt ; Set true if out of VMM Fast syscall range + bt- cr1_eq,notufp ; Exit if out of range + b EXT(vmm_ufp) ; Ultra Fast Path syscall + +/* + * .L_exception_entry(type) + * + * This is the common exception handling routine called by any + * type of system exception. + * + * ENTRY: via a system exception handler, thus interrupts off, VM off. + * r3 has been saved in sprg3 and now contains a number + * representing the exception's origins + * + */ + + .data + .align ALIGN + .globl EXT(exception_entry) +EXT(exception_entry): + .long .L_exception_entry-EXT(ExceptionVectorsStart) /* phys addr of fn */ + + VECTOR_SEGMENT + .align 5 + +.L_exception_entry: + +/* + * + * Here we will save off a mess of registers, the special ones and R0-R12. We use the DCBZ + * instruction to clear and allcoate a line in the cache. This way we won't take any cache + * misses, so these stores won't take all that long. Except the first line that is because + * we can't do a DCBZ if the L1 D-cache is off. The rest we will skip if they are + * off also. + * + * Note that if we are attempting to sleep (as opposed to nap or doze) all interruptions + * are ignored. + */ + + + .globl EXT(extPatch32) + + +LEXT(extPatch32) + b extEntry64 ; Go do 64-bit (patched out for 32-bit) + mfsprg r13,0 ; Load per_proc + lwz r13,next_savearea+4(r13) ; Get the exception save area + stw r0,saver0+4(r13) ; Save register 0 + stw r1,saver1+4(r13) ; Save register 1 + + mfspr r1,hid0 ; Get HID0 + mfcr r0 ; Save the whole CR + + mtcrf 0x20,r1 ; Get set to test for sleep + cror doze,doze,nap ; Remember if we are napping + bf sleep,notsleep ; Skip if we are not trying to sleep + + mtcrf 0x20,r0 ; Restore the CR + lwz r0,saver0+4(r13) ; Restore R0 + lwz r1,saver1+4(r13) ; Restore R1 + mfsprg r13,0 ; Get the per_proc + lwz r11,pfAvailable(r13) ; Get back the feature flags + mfsprg r13,2 ; Restore R13 + mtsprg 2,r11 ; Set sprg2 to the features + mfsprg r11,3 ; Restore R11 + rfi ; Jump back into sleep code... + .long 0 ; Leave these here please... + .long 0 + .long 0 + .long 0 + .long 0 + .long 0 + .long 0 + .long 0 + + +; +; This is the 32-bit context saving stuff +; + + .align 5 + +notsleep: stw r2,saver2+4(r13) ; Save this one + bf doze,notspdo ; Skip the next if we are not napping/dozing... + rlwinm r2,r1,0,nap+1,doze-1 ; Clear any possible nap and doze bits + mtspr hid0,r2 ; Clear the nap/doze bits +notspdo: + +#if INSTRUMENT + mfspr r2,pmc1 ; INSTRUMENT - saveinstr[0] - Take earliest possible stamp + stw r2,0x6100+(0x00*16)+0x0(0) ; INSTRUMENT - Save it + mfspr r2,pmc2 ; INSTRUMENT - Get stamp + stw r2,0x6100+(0x00*16)+0x4(0) ; INSTRUMENT - Save it + mfspr r2,pmc3 ; INSTRUMENT - Get stamp + stw r2,0x6100+(0x00*16)+0x8(0) ; INSTRUMENT - Save it + mfspr r2,pmc4 ; INSTRUMENT - Get stamp + stw r2,0x6100+(0x00*16)+0xC(0) ; INSTRUMENT - Save it +#endif + + la r1,saver4(r13) ; Point to the next line in case we need it + crmove wasNapping,doze ; Remember if we were napping + mfsprg r2,0 ; Get the per_proc area + dcbz 0,r1 ; allocate r4-r7 32-byte line in cache + +; +; Remember, we are setting up CR6 with feature flags +; + andi. r1,r11,T_FAM ; Check FAM bit + + stw r3,saver3+4(r13) ; Save this one + stw r4,saver4+4(r13) ; Save this one + andc r11,r11,r1 ; Clear FAM bit + beq+ noFAM ; Is it FAM intercept + mfsrr1 r3 ; Load srr1 + rlwinm. r3,r3,0,MSR_PR_BIT,MSR_PR_BIT ; Are we trapping from supervisor state? + beq+ noFAM ; From supervisor state + lwz r1,spcFlags(r2) ; Load spcFlags + rlwinm r1,r1,1+FamVMmodebit,30,31 ; Extract FamVMenabit and FamVMmodebit + cmpwi cr0,r1,2 ; Check FamVMena set without FamVMmode + bne+ noFAM ; Can this context be FAM intercept + lwz r4,FAMintercept(r2) ; Load exceptions mask to intercept + srwi r1,r11,2 ; divide r11 by 4 + lis r3,0x8000 ; Set r3 to 0x80000000 + srw r1,r3,r1 ; Set bit for current exception + and. r1,r1,r4 ; And current exception with the intercept mask + beq+ noFAM ; Is it FAM intercept + b EXT(vmm_fam_exc) +noFAM: + lwz r1,pfAvailable(r2) ; Get the CPU features flags + la r3,saver8(r13) ; Point to line with r8-r11 + mtcrf 0xE2,r1 ; Put the features flags (that we care about) in the CR + dcbz 0,r3 ; allocate r8-r11 32-byte line in cache + la r3,saver12(r13) ; point to r12-r15 line + lis r4,hi16(MASK(MSR_VEC)|MASK(MSR_FP)|MASK(MSR_ME)) ; Set up the MSR we will use throughout. Note that ME come on here if MCK + stw r6,saver6+4(r13) ; Save this one + ori r4,r4,lo16(MASK(MSR_VEC)|MASK(MSR_FP)|MASK(MSR_ME)) ; Rest of MSR + stw r8,saver8+4(r13) ; Save this one + crmove featAltivec,pfAltivecb ; Set the Altivec flag + mtmsr r4 ; Set MSR + isync + mfsrr0 r6 ; Get the interruption SRR0 + la r8,savesrr0(r13) ; point to line with SRR0, SRR1, CR, XER, and LR + dcbz 0,r3 ; allocate r12-r15 32-byte line in cache + la r3,saver16(r13) ; point to next line + dcbz 0,r8 ; allocate 32-byte line with SRR0, SRR1, CR, XER, and LR + stw r7,saver7+4(r13) ; Save this one + lhz r8,PP_CPU_FLAGS(r2) ; Get the flags + mfsrr1 r7 ; Get the interrupt SRR1 + rlwinm r8,r8,(((31-MSR_BE_BIT)+(traceBEb+16+1))&31),MSR_BE_BIT,MSR_BE_BIT ; Set BE bit if special trace is on + stw r6,savesrr0+4(r13) ; Save the SRR0 + rlwinm r6,r7,(((31-MSR_BE_BIT)+(MSR_PR_BIT+1))&31),MSR_BE_BIT,MSR_BE_BIT ; Move PR bit to BE bit + stw r5,saver5+4(r13) ; Save this one + and r8,r6,r8 ; Remove BE bit only if problem state and special tracing on + mfsprg r6,2 ; Get interrupt time R13 + mtsprg 2,r1 ; Set the feature flags + andc r7,r7,r8 ; Clear BE bit if special trace is on and PR is set + mfsprg r8,3 ; Get rupt time R11 + stw r7,savesrr1+4(r13) ; Save SRR1 + stw r8,saver11+4(r13) ; Save rupt time R11 + stw r6,saver13+4(r13) ; Save rupt R13 + dcbz 0,r3 ; allocate 32-byte line with r16-r19 + la r3,saver20(r13) ; point to next line + +getTB: mftbu r6 ; Get the upper timebase + mftb r7 ; Get the lower timebase + mftbu r8 ; Get the upper one again + cmplw r6,r8 ; Did the top tick? + bne- getTB ; Yeah, need to get it again... + +#if INSTRUMENT + mfspr r6,pmc1 ; INSTRUMENT - saveinstr[1] - Save halfway context save stamp + stw r6,0x6100+(0x01*16)+0x0(0) ; INSTRUMENT - Save it + mfspr r6,pmc2 ; INSTRUMENT - Get stamp + stw r6,0x6100+(0x01*16)+0x4(0) ; INSTRUMENT - Save it + mfspr r6,pmc3 ; INSTRUMENT - Get stamp + stw r6,0x6100+(0x01*16)+0x8(0) ; INSTRUMENT - Save it + mfspr r6,pmc4 ; INSTRUMENT - Get stamp + stw r6,0x6100+(0x01*16)+0xC(0) ; INSTRUMENT - Save it +#endif + + stw r8,ruptStamp(r2) ; Save the top of time stamp + stw r8,SAVtime(r13) ; Save the top of time stamp + stw r7,ruptStamp+4(r2) ; Save the bottom of time stamp + stw r7,SAVtime+4(r13) ; Save the bottom of time stamp + + dcbz 0,r3 ; allocate 32-byte line with r20-r23 + stw r9,saver9+4(r13) ; Save this one + + stw r10,saver10+4(r13) ; Save this one + mflr r4 ; Get the LR + mfxer r10 ; Get the XER + + bf+ wasNapping,notNapping ; Skip if not waking up from nap... + + lwz r6,napStamp+4(r2) ; Pick up low order nap stamp + lis r3,hi16(EXT(machine_idle_ret)) ; Get high part of nap/doze return + lwz r5,napStamp(r2) ; and high order + subfc r7,r6,r7 ; Subtract low stamp from now + lwz r6,napTotal+4(r2) ; Pick up low total + subfe r5,r5,r8 ; Subtract high stamp and borrow from now + lwz r8,napTotal(r2) ; Pick up the high total + addc r6,r6,r7 ; Add low to total + ori r3,r3,lo16(EXT(machine_idle_ret)) ; Get low part of nap/doze return + adde r8,r8,r5 ; Add high and carry to total + stw r6,napTotal+4(r2) ; Save the low total + stw r8,napTotal(r2) ; Save the high total + stw r3,savesrr0+4(r13) ; Modify to return to nap/doze exit + + rlwinm. r3,r1,0,pfSlowNapb,pfSlowNapb ; Should HID1 be restored? + beq notInSlowNap + + lwz r3,pfHID1(r2) ; Get saved HID1 value + mtspr hid1,r3 ; Restore HID1 + +notInSlowNap: + rlwinm. r3,r1,0,pfNoL2PFNapb,pfNoL2PFNapb ; Should MSSCR0 be restored? + beq notNapping + + lwz r3,pfMSSCR0(r2) ; Get saved MSSCR0 value + mtspr msscr0,r3 ; Restore MSSCR0 + sync + isync + +notNapping: stw r12,saver12+4(r13) ; Save this one + + stw r14,saver14+4(r13) ; Save this one + stw r15,saver15+4(r13) ; Save this one + la r14,saver24(r13) ; Point to the next block to save into + mfctr r6 ; Get the CTR + stw r16,saver16+4(r13) ; Save this one + la r15,savectr(r13) ; point to line with CTR, DAR, DSISR, Exception code, and VRSAVE + stw r4,savelr+4(r13) ; Save rupt LR + + dcbz 0,r14 ; allocate 32-byte line with r24-r27 + la r16,saver28(r13) ; point to line with r28-r31 + dcbz 0,r15 ; allocate line with CTR, DAR, DSISR, Exception code, and VRSAVE + stw r17,saver17+4(r13) ; Save this one + stw r18,saver18+4(r13) ; Save this one + stw r6,savectr+4(r13) ; Save rupt CTR + stw r0,savecr(r13) ; Save rupt CR + stw r19,saver19+4(r13) ; Save this one + mfdar r6 ; Get the rupt DAR + stw r20,saver20+4(r13) ; Save this one + dcbz 0,r16 ; allocate 32-byte line with r28-r31 + + stw r21,saver21+4(r13) ; Save this one + lwz r21,spcFlags(r2) ; Get the special flags from per_proc + stw r10,savexer+4(r13) ; Save the rupt XER + stw r30,saver30+4(r13) ; Save this one + lhz r30,pfrptdProc(r2) ; Get the reported processor type + stw r31,saver31+4(r13) ; Save this one + stw r22,saver22+4(r13) ; Save this one + stw r23,saver23+4(r13) ; Save this one + stw r24,saver24+4(r13) ; Save this one + stw r25,saver25+4(r13) ; Save this one + mfdsisr r7 ; Get the rupt DSISR + stw r26,saver26+4(r13) ; Save this one + stw r27,saver27+4(r13) ; Save this one + andis. r21,r21,hi16(perfMonitor) ; Is the performance monitor enabled? + stw r28,saver28+4(r13) ; Save this one + cmpwi cr1, r30,CPU_SUBTYPE_POWERPC_750 ; G3? + la r27,savevscr(r13) ; point to 32-byte line with VSCR and FPSCR + cmpwi cr2,r30,CPU_SUBTYPE_POWERPC_7400 ; This guy? + stw r29,saver29+4(r13) ; Save R29 + stw r6,savedar+4(r13) ; Save the rupt DAR + li r10,savepmc ; Point to pmc savearea + + beq+ noPerfMonSave32 ; No perfmon on here... + + dcbz r10,r13 ; Clear first part of pmc area + li r10,savepmc+0x20 ; Point to pmc savearea second part + li r22,0 ; r22: zero + dcbz r10,r13 ; Clear second part of pmc area + + beq cr1,perfMonSave32_750 ; This is a G3... + + beq cr2,perfMonSave32_7400 ; Regular olde G4... + + mfspr r24,pmc5 ; Here for a 7450 + mfspr r25,pmc6 + stw r24,savepmc+16(r13) ; Save PMC5 + stw r25,savepmc+20(r13) ; Save PMC6 + mtspr pmc5,r22 ; Leave PMC5 clear + mtspr pmc6,r22 ; Leave PMC6 clear + +perfMonSave32_7400: + mfspr r25,mmcr2 + stw r25,savemmcr2+4(r13) ; Save MMCR2 + mtspr mmcr2,r22 ; Leave MMCR2 clear + +perfMonSave32_750: + mfspr r23,mmcr0 + mfspr r24,mmcr1 + stw r23,savemmcr0+4(r13) ; Save MMCR0 + stw r24,savemmcr1+4(r13) ; Save MMCR1 + mtspr mmcr0,r22 ; Leave MMCR0 clear + mtspr mmcr1,r22 ; Leave MMCR1 clear + mfspr r23,pmc1 + mfspr r24,pmc2 + mfspr r25,pmc3 + mfspr r26,pmc4 + stw r23,savepmc+0(r13) ; Save PMC1 + stw r24,savepmc+4(r13) ; Save PMC2 + stw r25,savepmc+8(r13) ; Save PMC3 + stw r26,savepmc+12(r13) ; Save PMC4 + mtspr pmc1,r22 ; Leave PMC1 clear + mtspr pmc2,r22 ; Leave PMC2 clear + mtspr pmc3,r22 ; Leave PMC3 clear + mtspr pmc4,r22 ; Leave PMC4 clear + +noPerfMonSave32: + dcbz 0,r27 ; allocate line with VSCR and FPSCR + + stw r7,savedsisr(r13) ; Save the rupt code DSISR + stw r11,saveexception(r13) ; Save the exception code + + +; +; Everything is saved at this point, except for FPRs, and VMX registers. +; Time for us to get a new savearea and then trace interrupt if it is enabled. +; + + lwz r25,traceMask(0) ; Get the trace mask + li r0,SAVgeneral ; Get the savearea type value + lhz r19,PP_CPU_NUMBER(r2) ; Get the logical processor number + rlwinm r22,r11,30,0,31 ; Divide interrupt code by 2 + stb r0,SAVflags+2(r13) ; Mark valid context + addi r22,r22,10 ; Adjust code so we shift into CR5 + li r23,trcWork ; Get the trace work area address + rlwnm r7,r25,r22,22,22 ; Set CR5_EQ bit position to 0 if tracing allowed + li r26,0x8 ; Get start of cpu mask + srw r26,r26,r19 ; Get bit position of cpu number + mtcrf 0x04,r7 ; Set CR5 to show trace or not + and. r26,r26,r25 ; See if we trace this cpu + crandc cr5_eq,cr5_eq,cr0_eq ; Turn off tracing if cpu is disabled +; +; At this point, we can take another exception and lose nothing. +; + +#if INSTRUMENT + mfspr r26,pmc1 ; INSTRUMENT - saveinstr[2] - Take stamp after save is done + stw r26,0x6100+(0x02*16)+0x0(0) ; INSTRUMENT - Save it + mfspr r26,pmc2 ; INSTRUMENT - Get stamp + stw r26,0x6100+(0x02*16)+0x4(0) ; INSTRUMENT - Save it + mfspr r26,pmc3 ; INSTRUMENT - Get stamp + stw r26,0x6100+(0x02*16)+0x8(0) ; INSTRUMENT - Save it + mfspr r26,pmc4 ; INSTRUMENT - Get stamp + stw r26,0x6100+(0x02*16)+0xC(0) ; INSTRUMENT - Save it +#endif + + bne+ cr5,xcp32xit ; Skip all of this if no tracing here... + +; +; We select a trace entry using a compare and swap on the next entry field. +; Since we do not lock the actual trace buffer, there is a potential that +; another processor could wrap an trash our entry. Who cares? +; + + lwz r25,traceStart(0) ; Get the start of trace table + lwz r26,traceEnd(0) ; Get end of trace table + +trcsel: lwarx r20,0,r23 ; Get and reserve the next slot to allocate + + addi r22,r20,LTR_size ; Point to the next trace entry + cmplw r22,r26 ; Do we need to wrap the trace table? + bne+ gotTrcEnt ; No wrap, we got us a trace entry... + + mr r22,r25 ; Wrap back to start + +gotTrcEnt: stwcx. r22,0,r23 ; Try to update the current pointer + bne- trcsel ; Collision, try again... + +#if ESPDEBUG + dcbf 0,r23 ; Force to memory + sync +#endif + + dcbz 0,r20 ; Clear and allocate first trace line + +; +; Let us cut that trace entry now. +; + + lwz r16,ruptStamp(r2) ; Get top of time base + lwz r17,ruptStamp+4(r2) ; Get the bottom of time stamp + + li r14,32 ; Offset to second line + + lwz r0,saver0+4(r13) ; Get back interrupt time R0 + lwz r1,saver1+4(r13) ; Get back interrupt time R1 + lwz r8,savecr(r13) ; Get the CR value + + dcbz r14,r20 ; Zap the second line + + sth r19,LTR_cpu(r20) ; Stash the cpu number + li r14,64 ; Offset to third line + sth r11,LTR_excpt(r20) ; Save the exception type + lwz r7,saver2+4(r13) ; Get back interrupt time R2 + lwz r3,saver3+4(r13) ; Restore this one + + dcbz r14,r20 ; Zap the third half + + mfdsisr r9 ; Get the DSISR + li r14,96 ; Offset to forth line + stw r16,LTR_timeHi(r20) ; Set the upper part of TB + stw r17,LTR_timeLo(r20) ; Set the lower part of TB + lwz r10,savelr+4(r13) ; Get the LR + mfsrr0 r17 ; Get SRR0 back, it is still good + + dcbz r14,r20 ; Zap the forth half + lwz r4,saver4+4(r13) ; Restore this one + lwz r5,saver5+4(r13) ; Restore this one + mfsrr1 r18 ; SRR1 is still good in here + + stw r8,LTR_cr(r20) ; Save the CR + lwz r6,saver6+4(r13) ; Get R6 + mfdar r16 ; Get this back + stw r9,LTR_dsisr(r20) ; Save the DSISR + stw r17,LTR_srr0+4(r20) ; Save the SSR0 + + stw r18,LTR_srr1+4(r20) ; Save the SRR1 + stw r16,LTR_dar+4(r20) ; Save the DAR + mfctr r17 ; Get the CTR (still good in register) + stw r13,LTR_save+4(r20) ; Save the savearea + stw r10,LTR_lr+4(r20) ; Save the LR + + stw r17,LTR_ctr+4(r20) ; Save off the CTR + stw r0,LTR_r0+4(r20) ; Save off register 0 + stw r1,LTR_r1+4(r20) ; Save off register 1 + stw r7,LTR_r2+4(r20) ; Save off register 2 + + + stw r3,LTR_r3+4(r20) ; Save off register 3 + stw r4,LTR_r4+4(r20) ; Save off register 4 + stw r5,LTR_r5+4(r20) ; Save off register 5 + stw r6,LTR_r6+4(r20) ; Save off register 6 + +#if ESPDEBUG + addi r17,r20,32 ; Second line + addi r16,r20,64 ; Third line + dcbst br0,r20 ; Force to memory + dcbst br0,r17 ; Force to memory + addi r17,r17,32 ; Fourth line + dcbst br0,r16 ; Force to memory + dcbst br0,r17 ; Force to memory + + sync ; Make sure it all goes +#endif +xcp32xit: mr r14,r11 ; Save the interrupt code across the call + bl EXT(save_get_phys_32) ; Grab a savearea + mfsprg r2,0 ; Get the per_proc info + li r10,emfp0 ; Point to floating point save + mr r11,r14 ; Get the exception code back + dcbz r10,r2 ; Clear for speed + stw r3,next_savearea+4(r2) ; Store the savearea for the next rupt + +#if INSTRUMENT + mfspr r4,pmc1 ; INSTRUMENT - saveinstr[3] - Take stamp after next savearea + stw r4,0x6100+(0x03*16)+0x0(0) ; INSTRUMENT - Save it + mfspr r4,pmc2 ; INSTRUMENT - Get stamp + stw r4,0x6100+(0x03*16)+0x4(0) ; INSTRUMENT - Save it + mfspr r4,pmc3 ; INSTRUMENT - Get stamp + stw r4,0x6100+(0x03*16)+0x8(0) ; INSTRUMENT - Save it + mfspr r4,pmc4 ; INSTRUMENT - Get stamp + stw r4,0x6100+(0x03*16)+0xC(0) ; INSTRUMENT - Save it +#endif + b xcpCommon ; Go join the common interrupt processing... + +; +; +; This is the 64-bit context saving stuff +; + + .align 5 + +extEntry64: mfsprg r13,0 ; Load per_proc + ld r13,next_savearea(r13) ; Get the exception save area + std r0,saver0(r13) ; Save register 0 + lis r0,hi16(MASK(MSR_VEC)|MASK(MSR_FP)|MASK(MSR_ME)) ; Set up the MSR we will use throughout. Note that ME come on here if MCK + std r1,saver1(r13) ; Save register 1 + ori r1,r0,lo16(MASK(MSR_VEC)|MASK(MSR_FP)|MASK(MSR_ME)) ; Rest of MSR + lis r0,0x0010 ; Get rupt code transform validity mask + mtmsr r1 ; Set MSR + isync + + ori r0,r0,0x0200 ; Get rupt code transform validity mask + std r2,saver2(r13) ; Save this one + lis r1,0x00F0 ; Top half of xform XOR + rlwinm r2,r11,29,27,31 ; Get high 5 bits of rupt code + std r3,saver3(r13) ; Save this one + slw r0,r0,r2 ; Move transform validity bit to bit 0 + std r4,saver4(r13) ; Save this one + std r5,saver5(r13) ; Save this one + ori r1,r1,0x04EC ; Bottom half of xform XOR + mfxer r5 ; Save the XER because we are about to muck with it + rlwinm r4,r11,1,27,28 ; Get bottom of interrupt code * 8 + lis r3,hi16(dozem|napm) ; Get the nap and doze bits + srawi r0,r0,31 ; Get 0xFFFFFFFF of xform valid, 0 otherwise + rlwnm r4,r1,r4,24,31 ; Extract the xform XOR + li r1,saver16 ; Point to the next line + and r4,r4,r0 ; Only keep transform if we are to use it + li r2,lgKillResv ; Point to the killing field + mfcr r0 ; Save the CR + stwcx. r2,0,r2 ; Kill any pending reservation + dcbz128 r1,r13 ; Blow away the line + sldi r3,r3,32 ; Position it + mfspr r1,hid0 ; Get HID0 + andc r3,r1,r3 ; Clear nap and doze + xor r11,r11,r4 ; Transform 970 rupt code to standard keeping FAM bit + cmpld r3,r1 ; See if nap and/or doze was on + std r6,saver6(r13) ; Save this one + mfsprg r2,0 ; Get the per_proc area + la r6,savesrr0(r13) ; point to line with SRR0, SRR1, CR, XER, and LR + beq++ eE64NoNap ; No nap here, skip all this... + + sync ; Make sure we are clean + mtspr hid0,r3 ; Set the updated hid0 + mfspr r1,hid0 ; Yes, this is silly, keep it here + mfspr r1,hid0 ; Yes, this is a duplicate, keep it here + mfspr r1,hid0 ; Yes, this is a duplicate, keep it here + mfspr r1,hid0 ; Yes, this is a duplicate, keep it here + mfspr r1,hid0 ; Yes, this is a duplicate, keep it here + mfspr r1,hid0 ; Yes, this is a duplicate, keep it here + +eE64NoNap: crnot wasNapping,cr0_eq ; Remember if we were napping + andi. r1,r11,T_FAM ; Check FAM bit + beq++ eEnoFAM ; Is it FAM intercept + mfsrr1 r3 ; Load srr1 + andc r11,r11,r1 ; Clear FAM bit + rlwinm. r3,r3,0,MSR_PR_BIT,MSR_PR_BIT ; Are we trapping from supervisor state? + beq+ eEnoFAM ; From supervisor state + lwz r1,spcFlags(r2) ; Load spcFlags + rlwinm r1,r1,1+FamVMmodebit,30,31 ; Extract FamVMenabit and FamVMmodebit + cmpwi cr0,r1,2 ; Check FamVMena set without FamVMmode + bne++ eEnoFAM ; Can this context be FAM intercept + lwz r4,FAMintercept(r2) ; Load exceptions mask to intercept + li r3,0 ; Clear + srwi r1,r11,2 ; divide r11 by 4 + oris r3,r3,0x8000 ; Set r3 to 0x80000000 + srw r1,r3,r1 ; Set bit for current exception + and. r1,r1,r4 ; And current exception with the intercept mask + beq++ eEnoFAM ; Is it FAM intercept + b EXT(vmm_fam_exc) + + .align 5 + +eEnoFAM: lwz r1,pfAvailable(r2) ; Get the CPU features flags + dcbz128 0,r6 ; allocate 128-byte line with SRR0, SRR1, CR, XER, and LR + +; +; Remember, we are setting up CR6 with feature flags +; + std r7,saver7(r13) ; Save this one + mtcrf 0x80,r1 ; Put the features flags (that we care about) in the CR + std r8,saver8(r13) ; Save this one + mtcrf 0x40,r1 ; Put the features flags (that we care about) in the CR + mfsrr0 r6 ; Get the interruption SRR0 + lhz r8,PP_CPU_FLAGS(r2) ; Get the flags + mtcrf 0x20,r1 ; Put the features flags (that we care about) in the CR + mfsrr1 r7 ; Get the interrupt SRR1 + rlwinm r8,r8,(((31-MSR_BE_BIT)+(traceBEb+16+1))&31),MSR_BE_BIT,MSR_BE_BIT ; Set BE bit if special trace is on + std r6,savesrr0(r13) ; Save the SRR0 + mtcrf 0x02,r1 ; Put the features flags (that we care about) in the CR + rlwinm r6,r7,(((31-MSR_BE_BIT)+(MSR_PR_BIT+1))&31),MSR_BE_BIT,MSR_BE_BIT ; Move PR bit to BE bit + and r8,r6,r8 ; Remove BE bit only if problem state and special tracing on + std r9,saver9(r13) ; Save this one + andc r7,r7,r8 ; Clear BE bit if special trace is on and PR is set + crmove featAltivec,pfAltivecb ; Set the Altivec flag + std r7,savesrr1(r13) ; Save SRR1 + mfsprg r9,3 ; Get rupt time R11 + std r10,saver10(r13) ; Save this one + mfsprg r6,2 ; Get interrupt time R13 + std r9,saver11(r13) ; Save rupt time R11 + mtsprg 2,r1 ; Set the feature flags + std r12,saver12(r13) ; Save this one + mflr r4 ; Get the LR + mftb r7 ; Get the timebase + std r6,saver13(r13) ; Save rupt R13 + std r7,ruptStamp(r2) ; Save the time stamp + std r7,SAVtime(r13) ; Save the time stamp + + bf++ wasNapping,notNappingSF ; Skip if not waking up from nap... + + ld r6,napStamp(r2) ; Pick up nap stamp + lis r3,hi16(EXT(machine_idle_ret)) ; Get high part of nap/doze return + sub r7,r7,r6 ; Subtract stamp from now + ld r6,napTotal(r2) ; Pick up total + add r6,r6,r7 ; Add low to total + ori r3,r3,lo16(EXT(machine_idle_ret)) ; Get low part of nap/doze return + std r6,napTotal(r2) ; Save the high total + std r3,savesrr0(r13) ; Modify to return to nap/doze exit + +notNappingSF: + std r14,saver14(r13) ; Save this one + std r15,saver15(r13) ; Save this one + stw r0,savecr(r13) ; Save rupt CR + mfctr r6 ; Get the CTR + std r16,saver16(r13) ; Save this one + std r4,savelr(r13) ; Save rupt LR + + std r17,saver17(r13) ; Save this one + li r7,savepmc ; Point to pmc area + std r18,saver18(r13) ; Save this one + lwz r17,spcFlags(r2) ; Get the special flags from per_proc + std r6,savectr(r13) ; Save rupt CTR + std r19,saver19(r13) ; Save this one + mfdar r6 ; Get the rupt DAR + std r20,saver20(r13) ; Save this one + + dcbz128 r7,r13 ; Clear out the pmc spot + + std r21,saver21(r13) ; Save this one + std r5,savexer(r13) ; Save the rupt XER + std r22,saver22(r13) ; Save this one + std r23,saver23(r13) ; Save this one + std r24,saver24(r13) ; Save this one + std r25,saver25(r13) ; Save this one + mfdsisr r7 ; Get the rupt DSISR + std r26,saver26(r13) ; Save this one + andis. r17,r17,hi16(perfMonitor) ; Is the performance monitor enabled? + std r27,saver27(r13) ; Save this one + li r10,emfp0 ; Point to floating point save + std r28,saver28(r13) ; Save this one + la r27,savevscr(r13) ; point to 32-byte line with VSCR and FPSCR + std r29,saver29(r13) ; Save R29 + std r30,saver30(r13) ; Save this one + std r31,saver31(r13) ; Save this one + std r6,savedar(r13) ; Save the rupt DAR + stw r7,savedsisr(r13) ; Save the rupt code DSISR + stw r11,saveexception(r13) ; Save the exception code + + beq++ noPerfMonSave64 ; Performance monitor not on... + + li r22,0 ; r22: zero + + mfspr r23,mmcr0_gp + mfspr r24,mmcr1_gp + mfspr r25,mmcra_gp + std r23,savemmcr0(r13) ; Save MMCR0 + std r24,savemmcr1(r13) ; Save MMCR1 + std r25,savemmcr2(r13) ; Save MMCRA + mtspr mmcr0_gp,r22 ; Leave MMCR0 clear + mtspr mmcr1_gp,r22 ; Leave MMCR1 clear + mtspr mmcra_gp,r22 ; Leave MMCRA clear + mfspr r23,pmc1_gp + mfspr r24,pmc2_gp + mfspr r25,pmc3_gp + mfspr r26,pmc4_gp + stw r23,savepmc+0(r13) ; Save PMC1 + stw r24,savepmc+4(r13) ; Save PMC2 + stw r25,savepmc+8(r13) ; Save PMC3 + stw r26,savepmc+12(r13) ; Save PMC4 + mfspr r23,pmc5_gp + mfspr r24,pmc6_gp + mfspr r25,pmc7_gp + mfspr r26,pmc8_gp + stw r23,savepmc+16(r13) ; Save PMC5 + stw r24,savepmc+20(r13) ; Save PMC6 + stw r25,savepmc+24(r13) ; Save PMC7 + stw r26,savepmc+28(r13) ; Save PMC8 + mtspr pmc1_gp,r22 ; Leave PMC1 clear + mtspr pmc2_gp,r22 ; Leave PMC2 clear + mtspr pmc3_gp,r22 ; Leave PMC3 clear + mtspr pmc4_gp,r22 ; Leave PMC4 clear + mtspr pmc5_gp,r22 ; Leave PMC5 clear + mtspr pmc6_gp,r22 ; Leave PMC6 clear + mtspr pmc7_gp,r22 ; Leave PMC7 clear + mtspr pmc8_gp,r22 ; Leave PMC8 clear + +noPerfMonSave64: + +; +; Everything is saved at this point, except for FPRs, and VMX registers. +; Time for us to get a new savearea and then trace interrupt if it is enabled. +; + + lwz r25,traceMask(0) ; Get the trace mask + li r0,SAVgeneral ; Get the savearea type value + lhz r19,PP_CPU_NUMBER(r2) ; Get the logical processor number + stb r0,SAVflags+2(r13) ; Mark valid context + ori r23,r23,lo16(EXT(trcWork)) ; Get the rest + rlwinm r22,r11,30,0,31 ; Divide interrupt code by 2 + li r23,trcWork ; Get the trace work area address + addi r22,r22,10 ; Adjust code so we shift into CR5 + li r26,0x8 ; Get start of cpu mask + rlwnm r7,r25,r22,22,22 ; Set CR5_EQ bit position to 0 if tracing allowed + srw r26,r26,r19 ; Get bit position of cpu number + mtcrf 0x04,r7 ; Set CR5 to show trace or not + and. r26,r26,r25 ; See if we trace this cpu + crandc cr5_eq,cr5_eq,cr0_eq ; Turn off tracing if cpu is disabled + + bne++ cr5,xcp64xit ; Skip all of this if no tracing here... + +; +; We select a trace entry using a compare and swap on the next entry field. +; Since we do not lock the actual trace buffer, there is a potential that +; another processor could wrap an trash our entry. Who cares? +; + + lwz r25,traceStart(0) ; Get the start of trace table + lwz r26,traceEnd(0) ; Get end of trace table + +trcselSF: lwarx r20,0,r23 ; Get and reserve the next slot to allocate + + addi r22,r20,LTR_size ; Point to the next trace entry + cmplw r22,r26 ; Do we need to wrap the trace table? + bne+ gotTrcEntSF ; No wrap, we got us a trace entry... + + mr r22,r25 ; Wrap back to start + +gotTrcEntSF: + stwcx. r22,0,r23 ; Try to update the current pointer + bne- trcselSF ; Collision, try again... + +#if ESPDEBUG + dcbf 0,r23 ; Force to memory + sync +#endif + +; +; Let us cut that trace entry now. +; + + dcbz128 0,r20 ; Zap the trace entry + + ld r16,ruptStamp(r2) ; Get top of time base + ld r0,saver0(r13) ; Get back interrupt time R0 (we need this whether we trace or not) + std r16,LTR_timeHi(r20) ; Set the upper part of TB + ld r1,saver1(r13) ; Get back interrupt time R1 + ld r18,saver2(r13) ; Get back interrupt time R2 + std r0,LTR_r0(r20) ; Save off register 0 + ld r3,saver3(r13) ; Restore this one + sth r19,LTR_cpu(r20) ; Stash the cpu number + std r1,LTR_r1(r20) ; Save off register 1 + ld r4,saver4(r13) ; Restore this one + std r18,LTR_r2(r20) ; Save off register 2 + ld r5,saver5(r13) ; Restore this one + ld r6,saver6(r13) ; Get R6 + std r3,LTR_r3(r20) ; Save off register 3 + lwz r16,savecr(r13) ; Get the CR value + std r4,LTR_r4(r20) ; Save off register 4 + mfsrr0 r17 ; Get SRR0 back, it is still good + std r5,LTR_r5(r20) ; Save off register 5 + std r6,LTR_r6(r20) ; Save off register 6 + mfsrr1 r18 ; SRR1 is still good in here + stw r16,LTR_cr(r20) ; Save the CR + std r17,LTR_srr0(r20) ; Save the SSR0 + std r18,LTR_srr1(r20) ; Save the SRR1 + + mfdar r17 ; Get this back + ld r16,savelr(r13) ; Get the LR + std r17,LTR_dar(r20) ; Save the DAR + mfctr r17 ; Get the CTR (still good in register) + std r16,LTR_lr(r20) ; Save the LR + std r17,LTR_ctr(r20) ; Save off the CTR + mfdsisr r17 ; Get the DSISR + std r13,LTR_save(r20) ; Save the savearea + stw r17,LTR_dsisr(r20) ; Save the DSISR + sth r11,LTR_excpt(r20) ; Save the exception type + +#if ESPDEBUG + dcbf 0,r20 ; Force to memory + sync ; Make sure it all goes +#endif +xcp64xit: mr r14,r11 ; Save the interrupt code across the call + bl EXT(save_get_phys_64) ; Grab a savearea + mfsprg r2,0 ; Get the per_proc info + li r10,emfp0 ; Point to floating point save + mr r11,r14 ; Get the exception code back + dcbz128 r10,r2 ; Clear for speed + std r3,next_savearea(r2) ; Store the savearea for the next rupt + b xcpCommon ; Go join the common interrupt processing... + +; +; All of the context is saved. Now we will get a +; fresh savearea. After this we can take an interrupt. +; + + .align 5 + +xcpCommon: + +; +; Here we will save some floating point and vector status +; and we also set a clean default status for a new interrupt level. +; Note that we assume that emfp0 is on an altivec boundary +; and that R10 points to it (as a displacemnt from R2). +; +; We need to save the FPSCR as if it is normal context. +; This is because pending exceptions will cause an exception even if +; FP is disabled. We need to clear the FPSCR when we first start running in the +; kernel. +; + + stfd f0,emfp0(r2) ; Save FPR0 + stfd f1,emfp1(r2) ; Save FPR1 + li r19,0 ; Assume no Altivec + mffs f0 ; Get the FPSCR + lfd f1,Zero(0) ; Make a 0 + stfd f0,savefpscrpad(r13) ; Save the FPSCR + li r9,0 ; Get set to clear VRSAVE + mtfsf 0xFF,f1 ; Clear it + addi r14,r10,16 ; Displacement to second vector register + lfd f0,emfp0(r2) ; Restore FPR0 + la r28,savevscr(r13) ; Point to the status area + lfd f1,emfp1(r2) ; Restore FPR1 + + bf featAltivec,noavec ; No Altivec on this CPU... + + stvxl v0,r10,r2 ; Save a register + stvxl v1,r14,r2 ; Save a second register + mfspr r19,vrsave ; Get the VRSAVE register + mfvscr v0 ; Get the vector status register + vspltish v1,1 ; Turn on the non-Java bit and saturate + stvxl v0,0,r28 ; Save the vector status + vspltisw v0,1 ; Turn on the saturate bit + vxor v1,v1,v0 ; Turn off saturate + mtvscr v1 ; Set the non-java, no saturate status for new level + mtspr vrsave,r9 ; Clear VRSAVE for each interrupt level + + lvxl v0,r10,r2 ; Restore first work register + lvxl v1,r14,r2 ; Restore second work register + +noavec: stw r19,savevrsave(r13) ; Save the vector register usage flags + +; +; We are now done saving all of the context. Start filtering the interrupts. +; Note that a Redrive will count as an actual interrupt. +; Note also that we take a lot of system calls so we will start decode here. +; + +Redrive: + + +#if INSTRUMENT + mfspr r20,pmc1 ; INSTRUMENT - saveinstr[4] - Take stamp before exception filter + stw r20,0x6100+(0x04*16)+0x0(0) ; INSTRUMENT - Save it + mfspr r20,pmc2 ; INSTRUMENT - Get stamp + stw r20,0x6100+(0x04*16)+0x4(0) ; INSTRUMENT - Save it + mfspr r20,pmc3 ; INSTRUMENT - Get stamp + stw r20,0x6100+(0x04*16)+0x8(0) ; INSTRUMENT - Save it + mfspr r20,pmc4 ; INSTRUMENT - Get stamp + stw r20,0x6100+(0x04*16)+0xC(0) ; INSTRUMENT - Save it +#endif + lwz r22,SAVflags(r13) ; Pick up the flags + lwz r0,saver0+4(r13) ; Get back interrupt time syscall number + mfsprg r2,0 ; Restore per_proc + + li r20,lo16(xcpTable) ; Point to the vector table (note: this must be in 1st 64k of physical memory) + la r12,hwCounts(r2) ; Point to the exception count area + rlwinm r22,r22,SAVredriveb+1,31,31 ; Get a 1 if we are redriving + add r12,r12,r11 ; Point to the count + lwzx r20,r20,r11 ; Get the interrupt handler + lwz r25,0(r12) ; Get the old value + lwz r23,hwRedrives(r2) ; Get the redrive count + xori r24,r22,1 ; Get the NOT of the redrive + mtctr r20 ; Point to the interrupt handler + mtcrf 0x80,r0 ; Set our CR0 to the high nybble of possible syscall code + add r25,r25,r24 ; Count this one if not a redrive + add r23,r23,r24 ; Count this one if if is a redrive + crandc cr0_lt,cr0_lt,cr0_gt ; See if we have R0 equal to 0b10xx...x + stw r25,0(r12) ; Store it back + stw r23,hwRedrives(r2) ; Save the redrive count + bctr ; Go process the exception... + + +; +; Exception vector filter table +; + + .align 7 + +xcpTable: + .long EatRupt ; T_IN_VAIN + .long PassUpTrap ; T_RESET + .long MachineCheck ; T_MACHINE_CHECK + .long EXT(handlePF) ; T_DATA_ACCESS + .long EXT(handlePF) ; T_INSTRUCTION_ACCESS + .long PassUpRupt ; T_INTERRUPT + .long EXT(AlignAssist) ; T_ALIGNMENT + .long EXT(Emulate) ; T_PROGRAM + .long PassUpFPU ; T_FP_UNAVAILABLE + .long PassUpRupt ; T_DECREMENTER + .long PassUpTrap ; T_IO_ERROR + .long PassUpTrap ; T_RESERVED + .long xcpSyscall ; T_SYSTEM_CALL + .long PassUpTrap ; T_TRACE + .long PassUpTrap ; T_FP_ASSIST + .long PassUpTrap ; T_PERF_MON + .long PassUpVMX ; T_VMX + .long PassUpTrap ; T_INVALID_EXCP0 + .long PassUpTrap ; T_INVALID_EXCP1 + .long PassUpTrap ; T_INVALID_EXCP2 + .long PassUpTrap ; T_INSTRUCTION_BKPT + .long PassUpRupt ; T_SYSTEM_MANAGEMENT + .long EXT(AltivecAssist) ; T_ALTIVEC_ASSIST + .long PassUpRupt ; T_THERMAL + .long PassUpTrap ; T_INVALID_EXCP5 + .long PassUpTrap ; T_INVALID_EXCP6 + .long PassUpTrap ; T_INVALID_EXCP7 + .long PassUpTrap ; T_INVALID_EXCP8 + .long PassUpTrap ; T_INVALID_EXCP9 + .long PassUpTrap ; T_INVALID_EXCP10 + .long PassUpTrap ; T_INVALID_EXCP11 + .long PassUpTrap ; T_INVALID_EXCP12 + .long PassUpTrap ; T_INVALID_EXCP13 + + .long PassUpTrap ; T_RUNMODE_TRACE + + .long PassUpRupt ; T_SIGP + .long PassUpTrap ; T_PREEMPT + .long conswtch ; T_CSWITCH + .long PassUpRupt ; T_SHUTDOWN + .long PassUpAbend ; T_CHOKE + + .long EXT(handleDSeg) ; T_DATA_SEGMENT + .long EXT(handleISeg) ; T_INSTRUCTION_SEGMENT + + .long WhoaBaby ; T_SOFT_PATCH + .long WhoaBaby ; T_MAINTENANCE + .long WhoaBaby ; T_INSTRUMENTATION + +; +; Just what the heck happened here???? +; + + .align 5 + +WhoaBaby: b . ; Open the hood and wait for help + + +; +; System call +; + + .align 5 + +xcpSyscall: lis r20,hi16(EXT(shandler)) ; Assume this is a normal one, get handler address + rlwinm r6,r0,1,0,31 ; Move sign bit to the end + ori r20,r20,lo16(EXT(shandler)) ; Assume this is a normal one, get handler address + bnl++ cr0,PassUp ; R0 not 0b10xxx...x, can not be any kind of magical system call, just pass it up... + lwz r7,savesrr1+4(r13) ; Get the entering MSR (low half) + lwz r1,dgFlags(0) ; Get the flags + cmplwi cr2,r6,1 ; See if original R0 had the CutTrace request code in it + + rlwinm. r7,r7,0,MSR_PR_BIT,MSR_PR_BIT ; Did we come from user state? + beq++ FCisok ; From supervisor state... + + rlwinm. r1,r1,0,enaUsrFCallb,enaUsrFCallb ; Are they valid? + beq++ PassUp ; No, treat as a normal one... + +FCisok: beq++ cr2,EatRupt ; This is a CutTrace system call, we are done with it... + +; +; Here is where we call the firmware. If it returns T_IN_VAIN, that means +; that it has handled the interruption. Remember: thou shalt not trash R13 +; while you are away. Anything else is ok. +; + + lwz r3,saver3+4(r13) ; Restore the first parameter + b EXT(FirmwareCall) ; Go handle the firmware call.... + +; +; Here is where we return from the firmware call +; + + .align 5 + .globl EXT(FCReturn) + +LEXT(FCReturn) + cmplwi r3,T_IN_VAIN ; Was it handled? + beq+ EatRupt ; Interrupt was handled... + mr r11,r3 ; Put the rupt code into the right register + b Redrive ; Go through the filter again... + + +; +; Here is where we return from the PTE miss and segment exception handler +; + + .align 5 + .globl EXT(PFSExit) + +LEXT(PFSExit) + +#if 0 + mfsprg r2,0 ; (BRINGUP) + lwz r0,savedsisr(r13) ; (BRINGUP) + andis. r0,r0,hi16(dsiAC) ; (BRINGUP) + beq++ didnthit ; (BRINGUP) + lwz r0,20(0) ; (BRINGUP) + mr. r0,r0 ; (BRINGUP) + bne-- didnthit ; (BRINGUP) +#if 0 + li r0,1 ; (BRINGUP) + stw r0,20(0) ; (BRINGUP) + lis r0,hi16(Choke) ; (BRINGUP) + ori r0,r0,lo16(Choke) ; (BRINGUP) + sc ; (BRINGUP) +#endif + + lwz r4,savesrr0+4(r13) ; (BRINGUP) + lwz r8,savesrr1+4(r13) ; (BRINGUP) + lwz r6,savedar+4(r13) ; (BRINGUP) + rlwinm. r0,r8,0,MSR_IR_BIT,MSR_IR_BIT ; (BRINGUP) + mfmsr r9 ; (BRINGUP) + ori r0,r9,lo16(MASK(MSR_DR)) ; (BRINGUP) + beq-- hghg ; (BRINGUP) + mtmsr r0 ; (BRINGUP) + isync ; (BRINGUP) + +hghg: lwz r5,0(r4) ; (BRINGUP) + beq-- hghg1 ; (BRINGUP) + mtmsr r9 ; (BRINGUP) + isync ; (BRINGUP) + +hghg1: rlwinm r7,r5,6,26,31 ; (BRINGUP) + rlwinm r27,r5,14,24,28 ; (BRINGUP) + addi r3,r13,saver0+4 ; (BRINGUP) + lwzx r3,r3,r27 ; (BRINGUP) + +#if 0 + lwz r27,patcharea+4(r2) ; (BRINGUP) + mr. r3,r3 ; (BRINGUP) + bne++ nbnbnb ; (BRINGUP) + addi r27,r27,1 ; (BRINGUP) + stw r27,patcharea+4(r2) ; (BRINGUP) +nbnbnb: +#endif + + rlwinm. r28,r8,0,MSR_DR_BIT,MSR_DR_BIT ; (BRINGUP) + rlwinm r27,r6,0,0,29 ; (BRINGUP) + ori r28,r9,lo16(MASK(MSR_DR)) ; (BRINGUP) + mfspr r10,dabr ; (BRINGUP) + li r0,0 ; (BRINGUP) + mtspr dabr,r0 ; (BRINGUP) + cmplwi cr1,r7,31 ; (BRINGUP) + beq-- qqq0 ; (BRINGUP) + mtmsr r28 ; (BRINGUP) +qqq0: + isync ; (BRINGUP) + + lwz r27,0(r27) ; (BRINGUP) - Get original value + + bne cr1,qqq1 ; (BRINGUP) + + rlwinm r5,r5,31,22,31 ; (BRINGUP) + cmplwi cr1,r5,151 ; (BRINGUP) + beq cr1,qqq3 ; (BRINGUP) + cmplwi cr1,r5,407 ; (BRINGUP) + beq cr1,qqq2 ; (BRINGUP) + cmplwi cr1,r5,215 ; (BRINGUP) + beq cr1,qqq0q ; (BRINGUP) + cmplwi cr1,r5,1014 ; (BRINGUP) + beq cr1,qqqm1 ; (BRINGUP) + + lis r0,hi16(Choke) ; (BRINGUP) + ori r0,r0,lo16(Choke) ; (BRINGUP) + sc ; (BRINGUP) + +qqqm1: rlwinm r7,r6,0,0,26 ; (BRINGUP) + stw r0,0(r7) ; (BRINGUP) + stw r0,4(r7) ; (BRINGUP) + stw r0,8(r7) ; (BRINGUP) + stw r0,12(r7) ; (BRINGUP) + stw r0,16(r7) ; (BRINGUP) + stw r0,20(r7) ; (BRINGUP) + stw r0,24(r7) ; (BRINGUP) + stw r0,28(r7) ; (BRINGUP) + b qqq9 + +qqq1: cmplwi r7,38 ; (BRINGUP) + bgt qqq2 ; (BRINGUP) + blt qqq3 ; (BRINGUP) + +qqq0q: stb r3,0(r6) ; (BRINGUP) + b qqq9 ; (BRINGUP) + +qqq2: sth r3,0(r6) ; (BRINGUP) + b qqq9 ; (BRINGUP) + +qqq3: stw r3,0(r6) ; (BRINGUP) + +qqq9: +#if 0 + rlwinm r7,r6,0,0,29 ; (BRINGUP) + lwz r0,0(r7) ; (BRINGUP) - Get newest value +#else + lis r7,hi16(0x000792B8) ; (BRINGUP) + ori r7,r7,lo16(0x000792B8) ; (BRINGUP) + lwz r0,0(r7) ; (BRINGUP) - Get newest value +#endif + mtmsr r9 ; (BRINGUP) + mtspr dabr,r10 ; (BRINGUP) + isync ; (BRINGUP) + +#if 0 + lwz r28,patcharea+12(r2) ; (BRINGUP) + mr. r28,r28 ; (BRINGUP) + bne++ qqq12 ; (BRINGUP) + lis r28,0x4000 ; (BRINGUP) + +qqq12: stw r27,0(r28) ; (BRINGUP) + lwz r6,savedar+4(r13) ; (BRINGUP) + stw r0,4(r28) ; (BRINGUP) + stw r4,8(r28) ; (BRINGUP) + stw r6,12(r28) ; (BRINGUP) + addi r28,r28,16 ; (BRINGUP) + mr. r3,r3 ; (BRINGUP) + stw r28,patcharea+12(r2) ; (BRINGUP) + lwz r10,patcharea+8(r2) ; (BRINGUP) + lwz r0,patcharea+4(r2) ; (BRINGUP) +#endif + +#if 1 + stw r0,patcharea(r2) ; (BRINGUP) +#endif + +#if 0 + xor r28,r0,r27 ; (BRINGUP) - See how much it changed + rlwinm r28,r28,24,24,31 ; (BRINGUP) + cmplwi r28,1 ; (BRINGUP) + + ble++ qqq10 ; (BRINGUP) + + mr r7,r0 ; (BRINGUP) + li r0,1 ; (BRINGUP) + stw r0,20(0) ; (BRINGUP) + lis r0,hi16(Choke) ; (BRINGUP) + ori r0,r0,lo16(Choke) ; (BRINGUP) + sc ; (BRINGUP) +#endif + + +qqq10: addi r4,r4,4 ; (BRINGUP) + stw r4,savesrr0+4(r13) ; (BRINGUP) + + li r11,T_IN_VAIN ; (BRINGUP) + b EatRupt ; (BRINGUP) + +didnthit: ; (BRINGUP) +#endif +#if 0 + lwz r0,20(0) ; (BRINGUP) + mr. r0,r0 ; (BRINGUP) + beq++ opopop ; (BRINGUP) + li r0,0 ; (BRINGUP) + stw r0,20(0) ; (BRINGUP) + lis r0,hi16(Choke) ; (BRINGUP) + ori r0,r0,lo16(Choke) ; (BRINGUP) + sc ; (BRINGUP) +opopop: +#endif + lwz r0,savesrr1+4(r13) ; Get the MSR in use at exception time + cmplwi cr1,r11,T_IN_VAIN ; Was it handled? + rlwinm. r4,r0,0,MSR_PR_BIT,MSR_PR_BIT ; Are we trapping from supervisor state? + beq++ cr1,EatRupt ; Yeah, just blast back to the user... + beq-- NoFamPf + mfsprg r2,0 ; Get back per_proc + lwz r1,spcFlags(r2) ; Load spcFlags + rlwinm r1,r1,1+FamVMmodebit,30,31 ; Extract FamVMenabit and FamVMmodebit + cmpi cr0,r1,2 ; Check FamVMena set without FamVMmode + bne-- cr0,NoFamPf + lwz r6,FAMintercept(r2) ; Load exceptions mask to intercept + li r5,0 ; Clear + srwi r1,r11,2 ; divide r11 by 4 + oris r5,r5,0x8000 ; Set r5 to 0x80000000 + srw r1,r5,r1 ; Set bit for current exception + and. r1,r1,r6 ; And current exception with the intercept mask + beq++ NoFamPf ; Is it FAM intercept + bl EXT(vmm_fam_pf) + b EatRupt + +NoFamPf: andi. r4,r0,lo16(MASK(MSR_RI)) ; See if the recover bit is on + lis r0,0x8000 ; Get 0xFFFFFFFF80000000 + add r0,r0,r0 ; Get 0xFFFFFFFF00000000 + beq++ PassUpTrap ; Not on, normal case... +; +; Here is where we handle the "recovery mode" stuff. +; This is set by an emulation routine to trap any faults when it is fetching data or +; instructions. +; +; If we get a fault, we turn off RI, set CR0_EQ to false, bump the PC, and set R0 +; and R1 to the DAR and DSISR, respectively. +; + lwz r3,savesrr0(r13) ; Get the failing instruction address + lwz r4,savesrr0+4(r13) ; Get the failing instruction address + lwz r5,savecr(r13) ; Get the condition register + or r4,r4,r0 ; Fill the high part with foxes + lwz r0,savedar(r13) ; Get the DAR + addic r4,r4,4 ; Skip failing instruction + lwz r6,savedar+4(r13) ; Get the DAR + addze r3,r3 ; Propagate carry + rlwinm r5,r5,0,3,1 ; Clear CR0_EQ to let emulation code know we failed + lwz r7,savedsisr(r13) ; Grab the DSISR + stw r3,savesrr0(r13) ; Save resume address + stw r4,savesrr0+4(r13) ; Save resume address + stw r5,savecr(r13) ; And the resume CR + stw r0,saver0(r13) ; Pass back the DAR + stw r6,saver0+4(r13) ; Pass back the DAR + stw r7,saver1+4(r13) ; Pass back the DSISR + b EatRupt ; Resume emulated code + +; +; Here is where we handle the context switch firmware call. The old +; context has been saved. The new savearea is in kind of hokey, the high order +; half is stored in saver7 and the low half is in saver3. We will just +; muck around with the savearea pointers, and then join the exit routine +; + + .align 5 + +conswtch: + li r0,0xFFF ; Get page boundary + mr r29,r13 ; Save the save + andc r30,r13,r0 ; Round down to page boundary (64-bit safe) + lwz r5,saver3+4(r13) ; Switch to the new savearea + bf-- pf64Bitb,xcswNo64 ; Not 64-bit... + lwz r6,saver7+4(r13) ; Get the high order half + sldi r6,r6,32 ; Position high half + or r5,r5,r6 ; Merge them + +xcswNo64: lwz r30,SACvrswap+4(r30) ; get real to virtual translation + mr r13,r5 ; Switch saveareas + li r0,0 ; Clear this + xor r27,r29,r30 ; Flip to virtual + stw r0,saver3(r5) ; Push the new virtual savearea to the switch to routine + stw r27,saver3+4(r5) ; Push the new virtual savearea to the switch to routine + b EatRupt ; Start it up... + +; +; Handle machine check here. +; +; ? +; + + .align 5 + +MachineCheck: + + bt++ pf64Bitb,mck64 ; ? + + lwz r27,savesrr1+4(r13) ; Pick up srr1 + +; +; Check if the failure was in +; ml_probe_read. If so, this is expected, so modify the PC to +; ml_proble_read_mck and then eat the exception. +; + lwz r30,savesrr0+4(r13) ; Get the failing PC + lis r28,hi16(EXT(ml_probe_read_mck)) ; High order part + lis r27,hi16(EXT(ml_probe_read)) ; High order part + ori r28,r28,lo16(EXT(ml_probe_read_mck)) ; Get the low part + ori r27,r27,lo16(EXT(ml_probe_read)) ; Get the low part + cmplw r30,r28 ; Check highest possible + cmplw cr1,r30,r27 ; Check lowest + bge- PassUpTrap ; Outside of range + blt- cr1,PassUpTrap ; Outside of range +; +; We need to fix up the BATs here because the probe +; routine messed them all up... As long as we are at it, +; fix up to return directly to caller of probe. +; + + lis r11,hi16(EXT(shadow_BAT)+shdDBAT) ; Get shadow address + ori r11,r11,lo16(EXT(shadow_BAT)+shdDBAT) ; Get shadow address + + lwz r30,0(r11) ; Pick up DBAT 0 high + lwz r28,4(r11) ; Pick up DBAT 0 low + lwz r27,8(r11) ; Pick up DBAT 1 high + lwz r18,16(r11) ; Pick up DBAT 2 high + lwz r11,24(r11) ; Pick up DBAT 3 high + + sync + mtdbatu 0,r30 ; Restore DBAT 0 high + mtdbatl 0,r28 ; Restore DBAT 0 low + mtdbatu 1,r27 ; Restore DBAT 1 high + mtdbatu 2,r18 ; Restore DBAT 2 high + mtdbatu 3,r11 ; Restore DBAT 3 high + sync + + lwz r28,savelr+4(r13) ; Get return point + lwz r27,saver0+4(r13) ; Get the saved MSR + li r30,0 ; Get a failure RC + stw r28,savesrr0+4(r13) ; Set the return point + stw r27,savesrr1+4(r13) ; Set the continued MSR + stw r30,saver3+4(r13) ; Set return code + b EatRupt ; Yum, yum, eat it all up... + +; +; 64-bit machine checks +; + +mck64: + +; +; NOTE: WE NEED TO RETHINK RECOVERABILITY A BIT - radar 3167190 +; + + ld r23,savesrr0(r13) ; Grab the SRR0 in case we need bad instruction + ld r20,savesrr1(r13) ; Grab the SRR1 so we can decode the thing + lwz r21,savedsisr(r13) ; We might need this in a bit + ld r22,savedar(r13) ; We might need this in a bit + + lis r8,AsyMCKSrc ; Get the Async MCK Source register address + mfsprg r19,2 ; Get the feature flags + ori r8,r8,0x8000 ; Set to read data + rlwinm. r0,r19,0,pfSCOMFixUpb,pfSCOMFixUpb ; Do we need to fix the SCOM data? + + sync + + mtspr scomc,r8 ; Request the MCK source + mfspr r24,scomd ; Get the source + mfspr r8,scomc ; Get back the status (we just ignore it) + sync + isync + + lis r8,AsyMCKRSrc ; Get the Async MCK Source AND mask address + li r9,0 ; Get and AND mask of 0 + + sync + + mtspr scomd,r9 ; Set the AND mask to 0 + mtspr scomc,r8 ; Write the AND mask and clear conditions + mfspr r8,scomc ; Get back the status (we just ignore it) + sync + isync + + lis r8,cFIR ; Get the Core FIR register address + ori r8,r8,0x8000 ; Set to read data + + sync + + mtspr scomc,r8 ; Request the Core FIR + mfspr r25,scomd ; Get the source + mfspr r8,scomc ; Get back the status (we just ignore it) + sync + isync + + lis r8,cFIRrst ; Get the Core FIR AND mask address + + sync + + mtspr scomd,r9 ; Set the AND mask to 0 + mtspr scomc,r8 ; Write the AND mask and clear conditions + mfspr r8,scomc ; Get back the status (we just ignore it) + sync + isync + + lis r8,l2FIR ; Get the L2 FIR register address + ori r8,r8,0x8000 ; Set to read data + + sync + + mtspr scomc,r8 ; Request the L2 FIR + mfspr r26,scomd ; Get the source + mfspr r8,scomc ; Get back the status (we just ignore it) + sync + isync + + lis r8,l2FIRrst ; Get the L2 FIR AND mask address + + sync + + mtspr scomd,r9 ; Set the AND mask to 0 + mtspr scomc,r8 ; Write the AND mask and clear conditions + mfspr r8,scomc ; Get back the status (we just ignore it) + sync + isync + + lis r8,busFIR ; Get the Bus FIR register address + ori r8,r8,0x8000 ; Set to read data + + sync + + mtspr scomc,r8 ; Request the Bus FIR + mfspr r27,scomd ; Get the source + mfspr r8,scomc ; Get back the status (we just ignore it) + sync + isync + + lis r8,busFIRrst ; Get the Bus FIR AND mask address + + sync + + mtspr scomd,r9 ; Set the AND mask to 0 + mtspr scomc,r8 ; Write the AND mask and clear conditions + mfspr r8,scomc ; Get back the status (we just ignore it) + sync + isync + +; Note: bug in early chips where scom reads are shifted right by 1. We fix that here. +; Also note that we will lose bit 63 + + beq++ mckNoFix ; No fix up is needed + sldi r24,r24,1 ; Shift left 1 + sldi r25,r25,1 ; Shift left 1 + sldi r26,r26,1 ; Shift left 1 + sldi r27,r27,1 ; Shift left 1 + +mckNoFix: std r24,savexdat0(r13) ; Save the MCK source in case we pass the error + std r25,savexdat1(r13) ; Save the Core FIR in case we pass the error + std r26,savexdat2(r13) ; Save the L2 FIR in case we pass the error + std r27,savexdat3(r13) ; Save the BUS FIR in case we pass the error + + rlwinm. r0,r20,0,mckIFUE-32,mckIFUE-32 ; Is this some kind of uncorrectable? + bne mckUE ; Yeah... + + rlwinm. r0,r20,0,mckLDST-32,mckLDST-32 ; Some kind of load/store error? + bne mckHandleLDST ; Yes... + + rldicl. r0,r20,46,62 ; Get the error cause code + beq mckNotSure ; We need some more checks for this one... + + cmplwi r0,2 ; Check for TLB parity error + blt mckSLBparity ; This is an SLB parity error... + bgt mckhIFUE ; This is an IFetch tablewalk reload UE... + +; IFetch TLB parity error + + isync + tlbiel r23 ; Locally invalidate TLB entry for iaddr + sync ; Wait for it + b ceMck ; All recovered... + +; SLB parity error. This could be software caused. We get one if there is +; more than 1 valid SLBE with a matching ESID. That one we do not want to +; try to recover from. Search for it and if we get it, panic. + +mckSLBparity: + crclr cr0_eq ; Make sure we are not equal so we take correct exit + + la r3,emvr0(r2) ; Use this to keep track of valid ESIDs we find + li r5,0 ; Start with index 0 + +mckSLBck: la r4,emvr0(r2) ; Use this to keep track of valid ESIDs we find + slbmfee r6,r5 ; Get the next SLBE + andis. r0,r6,0x0800 ; See if valid bit is on + beq mckSLBnx ; Skip invalid and go to next + +mckSLBck2: cmpld r4,r3 ; Have we reached the end of the table? + beq mckSLBne ; Yes, go enter this one... + ld r7,0(r4) ; Pick up the saved ESID + cmpld r6,r7 ; Is this a match? + beq mckSLBrec ; Whoops, I did bad, recover and pass up... + addi r4,r4,8 ; Next table entry + b mckSLBck2 ; Check the next... + +mckSLBnx: addi r5,r5,1 ; Point to next SLBE + cmplwi r5,64 ; Have we checked all of them? + bne++ mckSLBck ; Not yet, check again... + b mckSLBrec ; We looked at them all, go recover... + +mckSLBne: std r6,0(r3) ; Save this ESID + addi r3,r3,8 ; Point to the new slot + b mckSLBnx ; Go do the next SLBE... + +; Recover an SLB error + +mckSLBrec: li r0,0 ; Set an SLB slot index of 0 + slbia ; Trash all SLB entries (except for entry 0 that is) + slbmfee r7,r0 ; Get the entry that is in SLB index 0 + rldicr r7,r7,0,35 ; Clear the valid bit and the rest + slbie r7 ; Invalidate it + + li r3,0 ; Set the first SLBE + +mckSLBclr: slbmte r0,r3 ; Clear the whole entry to 0s + addi r3,r3,1 ; Bump index + cmplwi cr1,r3,64 ; Have we done them all? + bne++ cr1,mckSLBclr ; Yup.... + + sth r3,ppInvSeg(r2) ; Store non-zero to trigger SLB reload + bne++ ceMck ; This was not a programming error, all recovered... + b ueMck ; Pass the software error up... + +; +; Handle a load/store unit error. We need to decode the DSISR +; + +mckHandleLDST: + rlwinm. r0,r21,0,mckL1DCPE,mckL1DCPE ; An L1 data cache parity error? + bne++ mckL1D ; Yeah, we dealt with this back in the vector... + + rlwinm. r0,r21,0,mckL1DTPE,mckL1DTPE ; An L1 tag error? + bne++ mckL1T ; Yeah, we dealt with this back in the vector... + + rlwinm. r0,r21,0,mckUEdfr,mckUEdfr ; Is the a "deferred" UE? + bne mckDUE ; Yeah, go see if expected... + + rlwinm. r0,r21,0,mckUETwDfr,mckUETwDfr ; Is the a "deferred" tablewalk UE? + bne mckDTW ; Yeah, no recovery... + + rlwinm. r0,r21,0,mckSLBPE,mckSLBPE ; SLB parity error? + bne mckSLBparity ; Yeah, go attempt recovery.... + +; This is a recoverable D-ERAT or TLB error + + la r9,hwMckERCPE(r2) ; Get DERAT parity error count + +mckInvDAR: isync + tlbiel r22 ; Locally invalidate the TLB entry + sync + + lwz r21,0(r9) ; Get count + addi r21,r21,1 ; Count this one + stw r21,0(r9) ; Stick it back + + b ceMck ; All recovered... + +; +; When we come here, we are not quite sure what the error is. We need to +; dig a bit further. +; +; R24 is interrupt source +; R25 is Core FIR +; +; Note that both have been cleared already. +; + +mckNotSure: + rldicl. r0,r24,AsyMCKfir+1,63 ; Something in the FIR? + bne-- mckFIR ; Yup, go check some more... + + rldicl. r0,r24,AsyMCKhri+1,63 ; Hang recovery? + bne-- mckHangRcvr ; Yup... + + rldicl. r0,r24,AsyMCKext+1,63 ; External signal? + bne-- mckExtMck ; Yup... + +; +; We really do not know what this one is or what to do with it... +; + +mckUnk: lwz r21,hwMckUnk(r2) ; Get unknown error count + addi r21,r21,1 ; Count it + stw r21,hwMckUnk(r2) ; Stuff it + b ueMck ; Go south, young man... + +; +; Hang recovery. This is just a notification so we only count. +; + +mckHangRcrvr: + lwz r21,hwMckHang(r2) ; Get hang recovery count + addi r21,r21,1 ; Count this one + stw r21,hwMckHang(r2) ; Stick it back + b ceMck ; All recovered... + +; +; Externally signaled MCK. No recovery for the moment, but we this may be +; where we handle ml_probe_read problems eventually. +; +mckExtMck: + lwz r21,hwMckHang(r2) ; Get hang recovery count + addi r21,r21,1 ; Count this one + stw r21,hwMckHang(r2) ; Stick it back + b ceMck ; All recovered... + +; +; Machine check cause is in a FIR. Suss it out here. +; Core FIR is in R25 and has been cleared in HW. +; + +mckFIR: rldicl. r0,r25,cFIRICachePE+1,63 ; I-Cache parity error? + la r19,hwMckICachePE(r2) ; Point to counter + bne mckInvICache ; Go invalidate I-Cache... + + rldicl. r0,r25,cFIRITagPE0+1,63 ; I-Cache tag parity error? + la r19,hwMckITagPE(r2) ; Point to counter + bne mckInvICache ; Go invalidate I-Cache... + + rldicl. r0,r25,cFIRITagPE1+1,63 ; I-Cache tag parity error? + la r19,hwMckITagPE(r2) ; Point to counter + bne mckInvICache ; Go invalidate I-Cache... + + rldicl. r0,r25,cFIRIEratPE+1,63 ; IERAT parity error? + la r19,hwMckIEratPE(r2) ; Point to counter + bne mckInvERAT ; Go invalidate ERATs... + + rldicl. r0,r25,cFIRIFUL2UE+1,63 ; IFetch got L2 UE? + bne mckhIFUE ; Go count and pass up... + + rldicl. r0,r25,cFIRDCachePE+1,63 ; D-Cache PE? + bne mckL1D ; Handled, just go count... + + rldicl. r0,r25,cFIRDTagPE+1,63 ; D-Cache tag PE? + bne mckL1T ; Handled, just go count... + + rldicl. r0,r25,cFIRDEratPE+1,63 ; DERAT PE? + la r19,hwMckDEratPE(r2) ; Point to counter + bne mckInvERAT ; Go invalidate ERATs... + + rldicl. r0,r25,cFIRTLBPE+1,63 ; TLB PE? + la r9,hwMckTLBPE(r2) ; Get TLB parity error count + bne mckInvDAR ; Go recover... + + rldicl. r0,r25,cFIRSLBPE+1,63 ; SLB PE? + bne mckSLBparity ; Cope with it... + + b mckUnk ; Have not a clue... + +; +; General recovery for I-Cache errors. Just flush it completely. +; + + .align 7 ; Force into cache line + +mckInvICache: + lis r0,0x0080 ; Get a 0x0080 (bit 9 >> 32) + mfspr r21,hid1 ; Get the current HID1 + sldi r0,r0,32 ; Get the "forced ICBI match" bit + or r0,r0,r21 ; Set forced match + + isync + mtspr hid1,r0 ; Stick it + mtspr hid1,r0 ; Stick it again + isync + + li r6,0 ; Start at 0 + +mckIcbi: icbi 0,r6 ; Kill I$ + addi r6,r6,128 ; Next line + andis. r5,r6,1 ; Have we done them all? + beq++ mckIcbi ; Not yet... + + isync + mtspr hid1,r21 ; Restore original HID1 + mtspr hid1,r21 ; Stick it again + isync + + lwz r5,0(r19) ; Get the counter + addi r5,r5,1 ; Count it + stw r5,0(r19) ; Stuff it back + b ceMck ; All recovered... + + +; General recovery for ERAT problems - handled in exception vector already + +mckInvERAT: lwz r21,0(r19) ; Get the exception count spot + addi r21,r21,1 ; Count this one + stw r21,0(r19) ; Save count + b ceMck ; All recovered... + +; General hang recovery - this is a notification only, just count. + +mckHangRcvr: + lwz r21,hwMckHang(r2) ; Get hang recovery count + addi r21,r21,1 ; Count this one + stw r21,hwMckHang(r2) ; Stick it back + b ceMck ; All recovered... + + +; +; These are the uncorrectable errors, just count them then pass it along. +; + +mckUE: lwz r21,hwMckUE(r2) ; Get general uncorrectable error count + addi r21,r21,1 ; Count it + stw r21,hwMckUE(r2) ; Stuff it + b ueMck ; Go south, young man... + +mckhIFUE: lwz r21,hwMckIUEr(r2) ; Get I-Fetch TLB reload uncorrectable error count + addi r21,r21,1 ; Count it + stw r21,hwMckIUEr(r2) ; Stuff it + b ueMck ; Go south, young man... + +mckDUE: lwz r21,hwMckDUE(r2) ; Get deferred uncorrectable error count + addi r21,r21,1 ; Count it + stw r21,hwMckDUE(r2) ; Stuff it + +; +; Right here is where we end up after a failure on a ml_probe_read_64. +; We will check if that is the case, and if so, fix everything up and +; return from it. + + lis r8,hi16(EXT(ml_probe_read_64)) ; High of start + lis r9,hi16(EXT(ml_probe_read_mck_64)) ; High of end + ori r8,r8,lo16(EXT(ml_probe_read_64)) ; Low of start + ori r9,r9,lo16(EXT(ml_probe_read_mck_64)) ; Low of end + cmpld r23,r8 ; Too soon? + cmpld cr1,r23,r9 ; Too late? + + cror cr0_lt,cr0_lt,cr1_gt ; Too soon or too late? + ld r3,saver12(r13) ; Get the original MSR + ld r5,savelr(r13) ; Get the return address + li r4,0 ; Get fail code + blt-- ueMck ; This is a normal machine check, just pass up... + std r5,savesrr0(r13) ; Set the return MSR + + std r3,savesrr1(r13) ; Set the return address + std r4,saver3(r13) ; Set failure return code + b ceMck ; All recovered... + +mckDTW: lwz r21,hwMckDTW(r2) ; Get deferred tablewalk uncorrectable error count + addi r21,r21,1 ; Count it + stw r21,hwMckDTW(r2) ; Stuff it + b ueMck ; Go south, young man... + +mckL1D: lwz r21,hwMckL1DPE(r2) ; Get data cache parity error count + addi r21,r21,1 ; Count it + stw r21,hwMckL1DPE(r2) ; Stuff it + b ceMck ; All recovered... + +mckL1T: lwz r21,hwMckL1TPE(r2) ; Get TLB parity error count + addi r21,r21,1 ; Count it + stw r21,hwMckL1TPE(r2) ; Stuff it + +ceMck: li r0,1 ; Set the recovered flag before passing up + stw r0,savemisc3(r13) ; Set it + b PassUpTrap ; Go up and log error... + +ueMck: li r0,0 ; Set the unrecovered flag before passing up + stw r0,savemisc3(r13) ; Set it + b PassUpTrap ; Go up and log error and probably panic + + +/* + * Here's where we come back from some instruction emulator. If we come back with + * T_IN_VAIN, the emulation is done and we should just reload state and directly + * go back to the interrupted code. Otherwise, we'll check to see if + * we need to redrive with a different interrupt, i.e., DSI. + * Note that this we are actually not redriving the rupt, rather changing it + * into a different one. Thus we clear the redrive bit. + */ + + .align 5 + .globl EXT(EmulExit) + +LEXT(EmulExit) + + cmplwi cr1,r11,T_IN_VAIN ; Was it emulated? + lis r1,hi16(SAVredrive) ; Get redrive request + beq++ cr1,EatRupt ; Yeah, just blast back to the user... + lwz r4,SAVflags(r13) ; Pick up the flags + + and. r0,r4,r1 ; Check if redrive requested + + beq++ PassUpTrap ; No redrive, just keep on going... + + b Redrive ; Redrive the exception... + +; +; Jump into main handler code switching on VM at the same time. +; +; We assume kernel data is mapped contiguously in physical +; memory, otherwise we would need to switch on (at least) virtual data. +; SRs are already set up. +; + + .align 5 + +PassUpTrap: lis r20,hi16(EXT(thandler)) ; Get thandler address + ori r20,r20,lo16(EXT(thandler)) ; Get thandler address + b PassUp ; Go pass it up... + +PassUpRupt: lis r20,hi16(EXT(ihandler)) ; Get ihandler address + ori r20,r20,lo16(EXT(ihandler)) ; Get ihandler address + b PassUp ; Go pass it up... + + .align 5 + +PassUpFPU: lis r20,hi16(EXT(fpu_switch)) ; Get FPU switcher address + ori r20,r20,lo16(EXT(fpu_switch)) ; Get FPU switcher address + b PassUp ; Go pass it up... + +PassUpVMX: lis r20,hi16(EXT(vec_switch)) ; Get VMX switcher address + ori r20,r20,lo16(EXT(vec_switch)) ; Get VMX switcher address + bt++ featAltivec,PassUp ; We have VMX on this CPU... + li r11,T_PROGRAM ; Say that it is a program exception + li r20,8 ; Set invalid instruction + stw r11,saveexception(r13) ; Set the new the exception code + sth r20,savesrr1+4(r13) ; Set the invalid instruction SRR code + + b PassUpTrap ; Go pass it up... + + .align 5 + +PassUpAbend: + lis r20,hi16(EXT(chandler)) ; Get choke handler address + ori r20,r20,lo16(EXT(chandler)) ; Get choke handler address + b PassUp ; Go pass it up... + + .align 5 + +PassUp: +#if INSTRUMENT + mfspr r29,pmc1 ; INSTRUMENT - saveinstr[11] - Take stamp at passup or eatrupt + stw r29,0x6100+(11*16)+0x0(0) ; INSTRUMENT - Save it + mfspr r29,pmc2 ; INSTRUMENT - Get stamp + stw r29,0x6100+(11*16)+0x4(0) ; INSTRUMENT - Save it + mfspr r29,pmc3 ; INSTRUMENT - Get stamp + stw r29,0x6100+(11*16)+0x8(0) ; INSTRUMENT - Save it + mfspr r29,pmc4 ; INSTRUMENT - Get stamp + stw r29,0x6100+(11*16)+0xC(0) ; INSTRUMENT - Save it +#endif + + lwz r10,SAVflags(r13) ; Pick up the flags + + li r0,0xFFF ; Get a page mask + li r2,MASK(MSR_BE)|MASK(MSR_SE) ; Get the mask to save trace bits + andc r5,r13,r0 ; Back off to the start of savearea block + mfmsr r3 ; Get our MSR + rlwinm r10,r10,0,SAVredriveb+1,SAVredriveb-1 ; Clear the redrive before we pass it up + li r21,MSR_SUPERVISOR_INT_OFF ; Get our normal MSR value + and r3,r3,r2 ; Clear all but trace + lwz r5,SACvrswap+4(r5) ; Get real to virtual conversion + or r21,r21,r3 ; Keep the trace bits if they are on + stw r10,SAVflags(r13) ; Set the flags with the cleared redrive flag + mr r3,r11 ; Pass the exception code in the paramter reg + xor r4,r13,r5 ; Pass up the virtual address of context savearea + mfsprg r29,0 ; Get the per_proc block back + rlwinm r4,r4,0,0,31 ; Clean top half of virtual savearea if 64-bit + + mr r3,r21 ; Pass in the MSR we will go to + bl EXT(switchSegs) ; Go handle the segment registers/STB + +#if INSTRUMENT + mfspr r30,pmc1 ; INSTRUMENT - saveinstr[7] - Take stamp afer switchsegs + stw r30,0x6100+(7*16)+0x0(0) ; INSTRUMENT - Save it + mfspr r30,pmc2 ; INSTRUMENT - Get stamp + stw r30,0x6100+(7*16)+0x4(0) ; INSTRUMENT - Save it + mfspr r30,pmc3 ; INSTRUMENT - Get stamp + stw r30,0x6100+(7*16)+0x8(0) ; INSTRUMENT - Save it + mfspr r30,pmc4 ; INSTRUMENT - Get stamp + stw r30,0x6100+(7*16)+0xC(0) ; INSTRUMENT - Save it +#endif + lwz r3,saveexception(r13) ; Recall the exception code + + mtsrr0 r20 ; Set up the handler address + mtsrr1 r21 ; Set up our normal MSR value + + bt++ pf64Bitb,puLaunch ; Handle 64-bit machine... + + rfi ; Launch the exception handler + +puLaunch: rfid ; Launch the exception handler + +/* + * This routine is the main place where we return from an interruption. + * + * This is also where we release the quickfret list. These are saveareas + * that were released as part of the exception exit path in hw_exceptions. + * In order to save an atomic operation (which actually will not work + * properly on a 64-bit machine) we use holdQFret to indicate that the list + * is in flux and should not be looked at here. This comes into play only + * when we take a PTE miss when we are queuing a savearea onto qfret. + * Quite rare but could happen. If the flag is set, this code does not + * release the list and waits until next time. + * + * All we need to remember here is that R13 must point to the savearea + * that has the context we need to load up. Translation and interruptions + * must be disabled. + * + * This code always loads the context in the savearea pointed to + * by R13. In the process, it throws away the savearea. If there + * is any tomfoolery with savearea stacks, it must be taken care of + * before we get here. + * + */ + + .align 5 + +EatRupt: mfsprg r29,0 ; Get the per_proc block back + mr r31,r13 ; Move the savearea pointer to the far end of the register set + mfsprg r27,2 ; Get the processor features + + lwz r3,holdQFret(r29) ; Get the release hold off flag + + bt++ pf64Bitb,eat64a ; Skip down to the 64-bit version of this + +; +; This starts the 32-bit version +; + + mr. r3,r3 ; Should we hold off the quick release? + lwz r30,quickfret+4(r29) ; Pick up the quick fret list, if any + la r21,saver0(r31) ; Point to the first thing we restore + bne- ernoqfret ; Hold off set, do not release just now... + +erchkfret: mr. r3,r30 ; Any savearea to quickly release? + beq+ ernoqfret ; No quickfrets... + lwz r30,SAVprev+4(r30) ; Chain back now + + bl EXT(save_ret_phys) ; Put it on the free list + stw r30,quickfret+4(r29) ; Dequeue previous guy (really, it is ok to wait until after the release) + b erchkfret ; Try the next one... + + .align 5 + +ernoqfret: +#if INSTRUMENT + mfspr r30,pmc1 ; INSTRUMENT - saveinstr[5] - Take stamp at saveareas released + stw r30,0x6100+(5*16)+0x0(0) ; INSTRUMENT - Save it + mfspr r30,pmc2 ; INSTRUMENT - Get stamp + stw r30,0x6100+(5*16)+0x4(0) ; INSTRUMENT - Save it + mfspr r30,pmc3 ; INSTRUMENT - Get stamp + stw r30,0x6100+(5*16)+0x8(0) ; INSTRUMENT - Save it + mfspr r30,pmc4 ; INSTRUMENT - Get stamp + stw r30,0x6100+(5*16)+0xC(0) ; INSTRUMENT - Save it +#endif + + dcbt 0,r21 ; Touch in the first thing we need + +; +; Here we release the savearea. +; +; Important!!!! The savearea is released before we are done with it. When the +; local free savearea list (anchored at lclfree) gets too long, save_ret_phys +; will trim the list, making the extra saveareas allocatable by another processor +; The code in there must ALWAYS leave our savearea on the local list, otherwise +; we could be very, very unhappy. The code there always queues the "just released" +; savearea to the head of the local list. Then, if it needs to trim, it will +; start with the SECOND savearea, leaving ours intact. +; +; + + mr r3,r31 ; Get the exiting savearea in parm register + bl EXT(save_ret_phys) ; Put it on the free list +#if INSTRUMENT + mfspr r3,pmc1 ; INSTRUMENT - saveinstr[6] - Take stamp afer savearea released + stw r3,0x6100+(6*16)+0x0(0) ; INSTRUMENT - Save it + mfspr r3,pmc2 ; INSTRUMENT - Get stamp + stw r3,0x6100+(6*16)+0x4(0) ; INSTRUMENT - Save it + mfspr r3,pmc3 ; INSTRUMENT - Get stamp + stw r3,0x6100+(6*16)+0x8(0) ; INSTRUMENT - Save it + mfspr r3,pmc4 ; INSTRUMENT - Get stamp + stw r3,0x6100+(6*16)+0xC(0) ; INSTRUMENT - Save it +#endif + + lwz r3,savesrr1+4(r31) ; Pass in the MSR we are going to + bl EXT(switchSegs) ; Go handle the segment registers/STB +#if INSTRUMENT + mfspr r30,pmc1 ; INSTRUMENT - saveinstr[10] - Take stamp afer switchsegs + stw r30,0x6100+(10*16)+0x0(0) ; INSTRUMENT - Save it + mfspr r30,pmc2 ; INSTRUMENT - Get stamp + stw r30,0x6100+(10*16)+0x4(0) ; INSTRUMENT - Save it + mfspr r30,pmc3 ; INSTRUMENT - Get stamp + stw r30,0x6100+(10*16)+0x8(0) ; INSTRUMENT - Save it + mfspr r30,pmc4 ; INSTRUMENT - Get stamp + stw r30,0x6100+(10*16)+0xC(0) ; INSTRUMENT - Save it +#endif + li r3,savesrr1+4 ; Get offset to the srr1 value + + lhz r9,PP_CPU_FLAGS(r29) ; Get the processor flags + lwarx r26,r3,r31 ; Get destination MSR and take reservation along the way (just so we can blow it away) + + rlwinm r25,r26,27,22,22 ; Move PR bit to BE + + cmplw cr3,r14,r14 ; Set that we do not need to stop streams + + rlwinm r9,r9,(((31-MSR_BE_BIT)+(traceBEb+16+1))&31),MSR_BE_BIT,MSR_BE_BIT ; Set BE bit if special trace is on + li r21,emfp0 ; Point to the fp savearea + and r9,r9,r25 ; Clear BE if supervisor state + or r26,r26,r9 ; Flip on the BE bit for special trace if needed + stwcx. r26,r3,r31 ; Blow away any reservations we hold (and set BE) + + lwz r25,savesrr0+4(r31) ; Get the SRR0 to use + + la r28,saver4(r31) ; Point to the 32-byte line with r4-r7 + dcbz r21,r29 ; Clear a work area + lwz r0,saver0+4(r31) ; Restore R0 + dcbt 0,r28 ; Touch in r4-r7 + lwz r1,saver1+4(r31) ; Restore R1 + lwz r2,saver2+4(r31) ; Restore R2 + la r28,saver8(r31) ; Point to the 32-byte line with r8-r11 + lwz r3,saver3+4(r31) ; Restore R3 + andis. r6,r27,hi16(pfAltivec) ; Do we have altivec on the machine? + dcbt 0,r28 ; touch in r8-r11 + lwz r4,saver4+4(r31) ; Restore R4 + la r28,saver12(r31) ; Point to the 32-byte line with r12-r15 + mtsrr0 r25 ; Restore the SRR0 now + lwz r5,saver5+4(r31) ; Restore R5 + mtsrr1 r26 ; Restore the SRR1 now + lwz r6,saver6+4(r31) ; Restore R6 + + dcbt 0,r28 ; touch in r12-r15 + la r28,saver16(r31) + + lwz r7,saver7+4(r31) ; Restore R7 + lwz r8,saver8+4(r31) ; Restore R8 + lwz r9,saver9+4(r31) ; Restore R9 + + dcbt 0,r28 ; touch in r16-r19 + la r28,saver20(r31) + + lwz r10,saver10+4(r31) ; Restore R10 + lwz r11,saver11+4(r31) ; Restore R11 + + dcbt 0,r28 ; touch in r20-r23 + la r28,savevscr(r31) ; Point to the status area + + lwz r12,saver12+4(r31) ; Restore R12 + lwz r13,saver13+4(r31) ; Restore R13 + + la r14,savectr+4(r31) + dcbt 0,r28 ; Touch in VSCR and FPSCR + dcbt 0,r14 ; touch in CTR, DAR, DSISR, VRSAVE, and Exception code + + lwz r26,next_savearea+4(r29) ; Get the exception save area + la r28,saver24(r31) + + lwz r14,saver14+4(r31) ; Restore R14 + lwz r15,saver15+4(r31) ; Restore R15 + + + stfd f0,emfp0(r29) ; Save FP0 + lwz r27,savevrsave(r31) ; Get the vrsave + dcbt 0,r28 ; touch in r24-r27 + la r28,savevscr(r31) ; Point to the status area + lfd f0,savefpscrpad(r31) ; Get the fpscr + la r22,saver28(r31) + mtfsf 0xFF,f0 ; Restore fpscr + lfd f0,emfp0(r29) ; Restore the used register + + beq noavec3 ; No Altivec on this CPU... + + stvxl v0,r21,r29 ; Save a vector register + lvxl v0,0,r28 ; Get the vector status + mtspr vrsave,r27 ; Set the vrsave + mtvscr v0 ; Set the vector status + lvxl v0,r21,r29 ; Restore work vector register + +noavec3: dcbt 0,r22 ; touch in r28-r31 + + lwz r23,spcFlags(r29) ; Get the special flags from per_proc + la r17,savesrr0(r31) + la r26,saver0(r26) ; Point to the first part of the next savearea + dcbt 0,r17 ; touch in SRR0, SRR1, CR, XER, LR + lhz r28,pfrptdProc(r29) ; Get the reported processor type + + lwz r16,saver16+4(r31) ; Restore R16 + lwz r17,saver17+4(r31) ; Restore R17 + lwz r18,saver18+4(r31) ; Restore R18 + lwz r19,saver19+4(r31) ; Restore R19 + lwz r20,saver20+4(r31) ; Restore R20 + lwz r21,saver21+4(r31) ; Restore R21 + lwz r22,saver22+4(r31) ; Restore R22 + + cmpwi cr1,r28,CPU_SUBTYPE_POWERPC_750 ; G3? + + dcbz 0,r26 ; Clear and allocate next savearea we use in the off chance it is still in when we next interrupt + + andis. r23,r23,hi16(perfMonitor) ; Is the performance monitor enabled? + lwz r23,saver23+4(r31) ; Restore R23 + cmpwi cr2,r28,CPU_SUBTYPE_POWERPC_7400 ; Yer standard G4? + lwz r24,saver24+4(r31) ; Restore R24 + lwz r25,saver25+4(r31) ; Restore R25 + lwz r26,saver26+4(r31) ; Restore R26 + lwz r27,saver27+4(r31) ; Restore R27 + + beq+ noPerfMonRestore32 ; No perf monitor... + + beq- cr1,perfMonRestore32_750 ; This is a G3... + beq- cr2,perfMonRestore32_7400 ; Standard G4... + + lwz r28,savepmc+16(r31) + lwz r29,savepmc+20(r31) + mtspr pmc5,r28 ; Restore PMC5 + mtspr pmc6,r29 ; Restore PMC6 + +perfMonRestore32_7400: + lwz r28,savemmcr2+4(r31) + mtspr mmcr2,r28 ; Restore MMCR2 + +perfMonRestore32_750: + lwz r28,savepmc+0(r31) + lwz r29,savepmc+4(r31) + mtspr pmc1,r28 ; Restore PMC1 + mtspr pmc2,r29 ; Restore PMC2 + lwz r28,savepmc+8(r31) + lwz r29,savepmc+12(r31) + mtspr pmc3,r28 ; Restore PMC3 + mtspr pmc4,r29 ; Restore PMC4 + lwz r28,savemmcr1+4(r31) + lwz r29,savemmcr0+4(r31) + mtspr mmcr1,r28 ; Restore MMCR1 + mtspr mmcr0,r29 ; Restore MMCR0 + +noPerfMonRestore32: + lwz r28,savecr(r31) ; Get CR to restore + lwz r29,savexer+4(r31) ; Get XER to restore + mtcr r28 ; Restore the CR + lwz r28,savelr+4(r31) ; Get LR to restore + mtxer r29 ; Restore the XER + lwz r29,savectr+4(r31) ; Get the CTR to restore + mtlr r28 ; Restore the LR + lwz r28,saver30+4(r31) ; Get R30 + mtctr r29 ; Restore the CTR + lwz r29,saver31+4(r31) ; Get R31 + mtsprg 2,r28 ; Save R30 for later + lwz r28,saver28+4(r31) ; Restore R28 + mtsprg 3,r29 ; Save R31 for later + lwz r29,saver29+4(r31) ; Restore R29 + + mfsprg r31,0 ; Get per_proc + mfsprg r30,2 ; Restore R30 + lwz r31,pfAvailable(r31) ; Get the feature flags + mtsprg 2,r31 ; Set the feature flags + mfsprg r31,3 ; Restore R31 + + rfi ; Click heels three times and think very hard that there is no place like home... + + .long 0 ; Leave this here + .long 0 + .long 0 + .long 0 + .long 0 + .long 0 + .long 0 + .long 0 + + +; +; This starts the 64-bit version +; + + .align 7 + +eat64a: ld r30,quickfret(r29) ; Pick up the quick fret list, if any + + mr. r3,r3 ; Should we hold off the quick release? + la r21,saver0(r31) ; Point to the first thing we restore + bne-- ernoqfre64 ; Hold off set, do not release just now... + +erchkfre64: mr. r3,r30 ; Any savearea to quickly release? + beq+ ernoqfre64 ; No quickfrets... + ld r30,SAVprev(r30) ; Chain back now + + bl EXT(save_ret_phys) ; Put it on the free list + + std r30,quickfret(r29) ; Dequeue previous guy (really, it is ok to wait until after the release) + b erchkfre64 ; Try the next one... + + .align 7 + +ernoqfre64: dcbt 0,r21 ; Touch in the first thing we need + +; +; Here we release the savearea. +; +; Important!!!! The savearea is released before we are done with it. When the +; local free savearea list (anchored at lclfree) gets too long, save_ret_phys +; will trim the list, making the extra saveareas allocatable by another processor +; The code in there must ALWAYS leave our savearea on the local list, otherwise +; we could be very, very unhappy. The code there always queues the "just released" +; savearea to the head of the local list. Then, if it needs to trim, it will +; start with the SECOND savearea, leaving ours intact. +; +; + + li r3,lgKillResv ; Get spot to kill reservation + stdcx. r3,0,r3 ; Blow away any reservations we hold + + mr r3,r31 ; Get the exiting savearea in parm register + bl EXT(save_ret_phys) ; Put it on the free list + + lwz r3,savesrr1+4(r31) ; Pass in the MSR we will be going to + bl EXT(switchSegs) ; Go handle the segment registers/STB + + lhz r9,PP_CPU_FLAGS(r29) ; Get the processor flags + ld r26,savesrr1(r31) ; Get destination MSR + cmplw cr3,r14,r14 ; Set that we do not need to stop streams + rlwinm r25,r26,27,22,22 ; Move PR bit to BE + + rlwinm r9,r9,(((31-MSR_BE_BIT)+(traceBEb+16+1))&31),MSR_BE_BIT,MSR_BE_BIT ; Set BE bit if special trace is on + li r21,emfp0 ; Point to a workarea + and r9,r9,r25 ; Clear BE if supervisor state + or r26,r26,r9 ; Flip on the BE bit for special trace if needed + + ld r25,savesrr0(r31) ; Get the SRR0 to use + la r28,saver16(r31) ; Point to the 128-byte line with r16-r31 + dcbz128 r21,r29 ; Clear a work area + ld r0,saver0(r31) ; Restore R0 + dcbt 0,r28 ; Touch in r16-r31 + ld r1,saver1(r31) ; Restore R1 + ld r2,saver2(r31) ; Restore R2 + ld r3,saver3(r31) ; Restore R3 + mtcrf 0x80,r27 ; Get facility availability flags (do not touch CR1-7) + ld r4,saver4(r31) ; Restore R4 + mtsrr0 r25 ; Restore the SRR0 now + ld r5,saver5(r31) ; Restore R5 + mtsrr1 r26 ; Restore the SRR1 now + ld r6,saver6(r31) ; Restore R6 + + ld r7,saver7(r31) ; Restore R7 + ld r8,saver8(r31) ; Restore R8 + ld r9,saver9(r31) ; Restore R9 + + la r28,savevscr(r31) ; Point to the status area + + ld r10,saver10(r31) ; Restore R10 + ld r11,saver11(r31) ; Restore R11 + ld r12,saver12(r31) ; Restore R12 + ld r13,saver13(r31) ; Restore R13 + + ld r26,next_savearea(r29) ; Get the exception save area + + ld r14,saver14(r31) ; Restore R14 + ld r15,saver15(r31) ; Restore R15 + lwz r27,savevrsave(r31) ; Get the vrsave + + bf-- pfAltivecb,noavec2s ; Skip if no VMX... + + stvxl v0,r21,r29 ; Save a vector register + lvxl v0,0,r28 ; Get the vector status + mtvscr v0 ; Set the vector status + + lvxl v0,r21,r29 ; Restore work vector register + +noavec2s: mtspr vrsave,r27 ; Set the vrsave + + lwz r28,saveexception(r31) ; Get exception type + stfd f0,emfp0(r29) ; Save FP0 + lfd f0,savefpscrpad(r31) ; Get the fpscr + mtfsf 0xFF,f0 ; Restore fpscr + lfd f0,emfp0(r29) ; Restore the used register + ld r16,saver16(r31) ; Restore R16 + lwz r30,spcFlags(r29) ; Get the special flags from per_proc + ld r17,saver17(r31) ; Restore R17 + ld r18,saver18(r31) ; Restore R18 + cmplwi cr1,r28,T_RESET ; Are we returning from a reset? + ld r19,saver19(r31) ; Restore R19 + ld r20,saver20(r31) ; Restore R20 + li r27,0 ; Get a zero + ld r21,saver21(r31) ; Restore R21 + la r26,saver0(r26) ; Point to the first part of the next savearea + andis. r30,r30,hi16(perfMonitor) ; Is the performance monitor enabled? + ld r22,saver22(r31) ; Restore R22 + ld r23,saver23(r31) ; Restore R23 + bne++ cr1,er64rrst ; We are not returning from a reset... + stw r27,lo16(EXT(ResetHandler)-EXT(ExceptionVectorsStart)+RESETHANDLER_TYPE)(br0) ; Allow resets again + +er64rrst: ld r24,saver24(r31) ; Restore R24 + + dcbz128 0,r26 ; Clear and allocate next savearea we use in the off chance it is still in when we next interrupt + + ld r25,saver25(r31) ; Restore R25 + ld r26,saver26(r31) ; Restore R26 + ld r27,saver27(r31) ; Restore R27 + + beq++ noPerfMonRestore64 ; Nope... + + lwz r28,savepmc+0(r31) + lwz r29,savepmc+4(r31) + mtspr pmc1_gp,r28 ; Restore PMC1 + mtspr pmc2_gp,r29 ; Restore PMC2 + lwz r28,savepmc+8(r31) + lwz r29,savepmc+12(r31) + mtspr pmc3_gp,r28 ; Restore PMC3 + mtspr pmc4_gp,r29 ; Restore PMC4 + lwz r28,savepmc+16(r31) + lwz r29,savepmc+20(r31) + mtspr pmc5_gp,r28 ; Restore PMC5 + mtspr pmc6_gp,r29 ; Restore PMC6 + lwz r28,savepmc+24(r31) + lwz r29,savepmc+28(r31) + mtspr pmc7_gp,r28 ; Restore PMC7 + mtspr pmc8_gp,r29 ; Restore PMC8 + ld r28,savemmcr1(r31) + ld r29,savemmcr2(r31) + mtspr mmcr1_gp,r28 ; Restore MMCR1 + mtspr mmcra_gp,r29 ; Restore MMCRA + ld r28,savemmcr0(r31) + + mtspr mmcr0_gp,r28 ; Restore MMCR0 + +noPerfMonRestore64: + mfsprg r30,0 ; Get per_proc + lwz r28,savecr(r31) ; Get CR to restore + ld r29,savexer(r31) ; Get XER to restore + mtcr r28 ; Restore the CR + ld r28,savelr(r31) ; Get LR to restore + mtxer r29 ; Restore the XER + ld r29,savectr(r31) ; Get the CTR to restore + mtlr r28 ; Restore the LR + ld r28,saver30(r31) ; Get R30 + mtctr r29 ; Restore the CTR + ld r29,saver31(r31) ; Get R31 + mtspr hsprg0,r28 ; Save R30 for later + ld r28,saver28(r31) ; Restore R28 + mtsprg 3,r29 ; Save R31 for later + ld r29,saver29(r31) ; Restore R29 + + lwz r31,pfAvailable(r30) ; Get the feature flags + lwz r30,UAW(r30) ; Get the User Assist Word + mtsprg 2,r31 ; Set the feature flags + mfsprg r31,3 ; Restore R31 + mtsprg 3,r30 ; Set the UAW + mfspr r30,hsprg0 ; Restore R30 + + rfid ; Click heels three times and think very hard that there is no place like home... + + + +/* + * exception_exit(savearea *) + * + * + * ENTRY : IR and/or DR and/or interruptions can be on + * R3 points to the virtual address of a savearea + */ + + .align 5 + .globl EXT(exception_exit) + +LEXT(exception_exit) + + mfsprg r29,2 ; Get feature flags + mr r31,r3 ; Get the savearea in the right register + mtcrf 0x04,r29 ; Set the features + li r0,1 ; Get this just in case + mtcrf 0x02,r29 ; Set the features + lis r30,hi16(MASK(MSR_VEC)|MASK(MSR_FP)|MASK(MSR_ME)) ; Set up the MSR we will use throughout. Note that ME come on here if MCK + rlwinm r4,r3,0,0,19 ; Round down to savearea block base + lis r1,hi16(SAVredrive) ; Get redrive request + mfsprg r2,0 ; Get the per_proc block + ori r30,r30,lo16(MASK(MSR_VEC)|MASK(MSR_FP)|MASK(MSR_ME)) ; Rest of MSR + bt++ pf64Bitb,eeSixtyFour ; We are 64-bit... + + lwz r4,SACvrswap+4(r4) ; Get the virtual to real translation + + bt pfNoMSRirb,eeNoMSR ; No MSR... + + mtmsr r30 ; Translation and all off + isync ; Toss prefetch + b eeNoMSRx + + .align 5 + +eeSixtyFour: + ld r4,SACvrswap(r4) ; Get the virtual to real translation + rldimi r30,r0,63,MSR_SF_BIT ; Set SF bit (bit 0) + mtmsrd r30 ; Set 64-bit mode, turn off EE, DR, and IR + isync ; Toss prefetch + b eeNoMSRx + + .align 5 + +eeNoMSR: li r0,loadMSR ; Get the MSR setter SC + mr r3,r30 ; Get new MSR + sc ; Set it + +eeNoMSRx: xor r31,r31,r4 ; Convert the savearea to physical addressing + lwz r4,SAVflags(r31) ; Pick up the flags + mr r13,r31 ; Put savearea here also + +#if INSTRUMENT + mfspr r5,pmc1 ; INSTRUMENT - saveinstr[8] - stamp exception exit + stw r5,0x6100+(8*16)+0x0(0) ; INSTRUMENT - Save it + mfspr r5,pmc2 ; INSTRUMENT - Get stamp + stw r5,0x6100+(8*16)+0x4(0) ; INSTRUMENT - Save it + mfspr r5,pmc3 ; INSTRUMENT - Get stamp + stw r5,0x6100+(8*16)+0x8(0) ; INSTRUMENT - Save it + mfspr r5,pmc4 ; INSTRUMENT - Get stamp + stw r5,0x6100+(8*16)+0xC(0) ; INSTRUMENT - Save it +#endif + + + and. r0,r4,r1 ; Check if redrive requested + + dcbt br0,r2 ; We will need this in just a sec + + beq+ EatRupt ; No redrive, just exit... + + lwz r11,saveexception(r13) ; Restore exception code + b Redrive ; Redrive the exception... + + + + .align 12 ; Force page alignment + + .globl EXT(ExceptionVectorsEnd) +EXT(ExceptionVectorsEnd): /* Used if relocating the exception vectors */ + + + + +; +; Here is where we keep the low memory globals +; + + . = 0x5000 + .globl EXT(lowGlo) + +EXT(lowGlo): + + .ascii "Hagfish " ; 5000 Unique eyecatcher + .long 0 ; 5008 Zero + .long 0 ; 500C Zero cont... + .long EXT(per_proc_info) ; 5010 pointer to per_procs + .long EXT(chudxnu_fn_table) ; 5014 pointer to chudxnu function glue table + .long 0 ; 5018 reserved + .long 0 ; 501C reserved + .long 0 ; 5020 reserved + .long 0 ; 5024 reserved + .long 0 ; 5028 reserved + .long 0 ; 502C reserved + .long 0 ; 5030 reserved + .long 0 ; 5034 reserved + .long 0 ; 5038 reserved + .long 0 ; 503C reserved + .long 0 ; 5040 reserved + .long 0 ; 5044 reserved + .long 0 ; 5048 reserved + .long 0 ; 504C reserved + .long 0 ; 5050 reserved + .long 0 ; 5054 reserved + .long 0 ; 5058 reserved + .long 0 ; 505C reserved + .long 0 ; 5060 reserved + .long 0 ; 5064 reserved + .long 0 ; 5068 reserved + .long 0 ; 506C reserved + .long 0 ; 5070 reserved + .long 0 ; 5074 reserved + .long 0 ; 5078 reserved + .long 0 ; 507C reserved + + .globl EXT(trcWork) +EXT(trcWork): + .long 0 ; 5080 The next trace entry to use +#if DEBUG + .long 0xFFFFFFFF ; 5084 All enabled +#else + .long 0x00000000 ; 5084 All disabled on non-debug systems +#endif + .long 0 ; 5088 Start of the trace table + .long 0 ; 508C End (wrap point) of the trace + .long 0 ; 5090 Saved mask while in debugger + .long 0 ; 5094 Size of trace table (1 - 256 pages) + .long 0 ; 5098 traceGas[0] + .long 0 ; 509C traceGas[1] + + .long 0 ; 50A0 reserved + .long 0 ; 50A4 reserved + .long 0 ; 50A8 reserved + .long 0 ; 50AC reserved + .long 0 ; 50B0 reserved + .long 0 ; 50B4 reserved + .long 0 ; 50B8 reserved + .long 0 ; 50BC reserved + .long 0 ; 50C0 reserved + .long 0 ; 50C4 reserved + .long 0 ; 50C8 reserved + .long 0 ; 50CC reserved + .long 0 ; 50D0 reserved + .long 0 ; 50D4 reserved + .long 0 ; 50D8 reserved + .long 0 ; 50DC reserved + .long 0 ; 50E0 reserved + .long 0 ; 50E4 reserved + .long 0 ; 50E8 reserved + .long 0 ; 50EC reserved + .long 0 ; 50F0 reserved + .long 0 ; 50F4 reserved + .long 0 ; 50F8 reserved + .long 0 ; 50FC reserved + + .globl EXT(saveanchor) + +EXT(saveanchor): ; 5100 saveanchor + .set .,.+SVsize + + .long 0 ; 5140 reserved + .long 0 ; 5144 reserved + .long 0 ; 5148 reserved + .long 0 ; 514C reserved + .long 0 ; 5150 reserved + .long 0 ; 5154 reserved + .long 0 ; 5158 reserved + .long 0 ; 515C reserved + .long 0 ; 5160 reserved + .long 0 ; 5164 reserved + .long 0 ; 5168 reserved + .long 0 ; 516C reserved + .long 0 ; 5170 reserved + .long 0 ; 5174 reserved + .long 0 ; 5178 reserved + .long 0 ; 517C reserved + + .long 0 ; 5180 tlbieLock + + .long 0 ; 5184 reserved + .long 0 ; 5188 reserved + .long 0 ; 518C reserved + .long 0 ; 5190 reserved + .long 0 ; 5194 reserved + .long 0 ; 5198 reserved + .long 0 ; 519C reserved + .long 0 ; 51A0 reserved + .long 0 ; 51A4 reserved + .long 0 ; 51A8 reserved + .long 0 ; 51AC reserved + .long 0 ; 51B0 reserved + .long 0 ; 51B4 reserved + .long 0 ; 51B8 reserved + .long 0 ; 51BC reserved + .long 0 ; 51C0 reserved + .long 0 ; 51C4 reserved + .long 0 ; 51C8 reserved + .long 0 ; 51CC reserved + .long 0 ; 51D0 reserved + .long 0 ; 51D4 reserved + .long 0 ; 51D8 reserved + .long 0 ; 51DC reserved + .long 0 ; 51E0 reserved + .long 0 ; 51E4 reserved + .long 0 ; 51E8 reserved + .long 0 ; 51EC reserved + .long 0 ; 51F0 reserved + .long 0 ; 51F4 reserved + .long 0 ; 51F8 reserved + .long 0 ; 51FC reserved + + .globl EXT(dgWork) + +EXT(dgWork): + + .long 0 ; 5200 dgLock + .long 0 ; 5204 dgFlags + .long 0 ; 5208 dgMisc0 + .long 0 ; 520C dgMisc1 + .long 0 ; 5210 dgMisc2 + .long 0 ; 5214 dgMisc3 + .long 0 ; 5218 dgMisc4 + .long 0 ; 521C dgMisc5 + + .long 0 ; 5220 reserved + .long 0 ; 5224 reserved + .long 0 ; 5228 reserved + .long 0 ; 522C reserved + .long 0 ; 5230 reserved + .long 0 ; 5234 reserved + .long 0 ; 5238 reserved + .long 0 ; 523C reserved + .long 0 ; 5240 reserved + .long 0 ; 5244 reserved + .long 0 ; 5248 reserved + .long 0 ; 524C reserved + .long 0 ; 5250 reserved + .long 0 ; 5254 reserved + .long 0 ; 5258 reserved + .long 0 ; 525C reserved + .long 0 ; 5260 reserved + .long 0 ; 5264 reserved + .long 0 ; 5268 reserved + .long 0 ; 526C reserved + .long 0 ; 5270 reserved + .long 0 ; 5274 reserved + .long 0 ; 5278 reserved + .long 0 ; 527C reserved + + .long 0 ; 5280 reserved + .long 0 ; 5284 reserved + .long 0 ; 5288 reserved + .long 0 ; 528C reserved + .long 0 ; 5290 reserved + .long 0 ; 5294 reserved + .long 0 ; 5298 reserved + .long 0 ; 529C reserved + .long 0 ; 52A0 reserved + .long 0 ; 52A4 reserved + .long 0 ; 52A8 reserved + .long 0 ; 52AC reserved + .long 0 ; 52B0 reserved + .long 0 ; 52B4 reserved + .long 0 ; 52B8 reserved + .long 0 ; 52BC reserved + .long 0 ; 52C0 reserved + .long 0 ; 52C4 reserved + .long 0 ; 52C8 reserved + .long 0 ; 52CC reserved + .long 0 ; 52D0 reserved + .long 0 ; 52D4 reserved + .long 0 ; 52D8 reserved + .long 0 ; 52DC reserved + .long 0 ; 52E0 reserved + .long 0 ; 52E4 reserved + .long 0 ; 52E8 reserved + .long 0 ; 52EC reserved + .long 0 ; 52F0 reserved + .long 0 ; 52F4 reserved + .long 0 ; 52F8 reserved + .long 0 ; 52FC reserved + + .globl EXT(killresv) +EXT(killresv): + + .long 0 ; 5300 Used to kill reservations + .long 0 ; 5304 Used to kill reservations + .long 0 ; 5308 Used to kill reservations + .long 0 ; 530C Used to kill reservations + .long 0 ; 5310 Used to kill reservations + .long 0 ; 5314 Used to kill reservations + .long 0 ; 5318 Used to kill reservations + .long 0 ; 531C Used to kill reservations + .long 0 ; 5320 Used to kill reservations + .long 0 ; 5324 Used to kill reservations + .long 0 ; 5328 Used to kill reservations + .long 0 ; 532C Used to kill reservations + .long 0 ; 5330 Used to kill reservations + .long 0 ; 5334 Used to kill reservations + .long 0 ; 5338 Used to kill reservations + .long 0 ; 533C Used to kill reservations + .long 0 ; 5340 Used to kill reservations + .long 0 ; 5344 Used to kill reservations + .long 0 ; 5348 Used to kill reservations + .long 0 ; 534C Used to kill reservations + .long 0 ; 5350 Used to kill reservations + .long 0 ; 5354 Used to kill reservations + .long 0 ; 5358 Used to kill reservations + .long 0 ; 535C Used to kill reservations + .long 0 ; 5360 Used to kill reservations + .long 0 ; 5364 Used to kill reservations + .long 0 ; 5368 Used to kill reservations + .long 0 ; 536C Used to kill reservations + .long 0 ; 5370 Used to kill reservations + .long 0 ; 5374 Used to kill reservations + .long 0 ; 5378 Used to kill reservations + .long 0 ; 537C Used to kill reservations + + .long 0 ; 5380 reserved + .long 0 ; 5384 reserved + .long 0 ; 5388 reserved + .long 0 ; 538C reserved + .long 0 ; 5390 reserved + .long 0 ; 5394 reserved + .long 0 ; 5398 reserved + .long 0 ; 539C reserved + .long 0 ; 53A0 reserved + .long 0 ; 53A4 reserved + .long 0 ; 53A8 reserved + .long 0 ; 53AC reserved + .long 0 ; 53B0 reserved + .long 0 ; 53B4 reserved + .long 0 ; 53B8 reserved + .long 0 ; 53BC reserved + .long 0 ; 53C0 reserved + .long 0 ; 53C4 reserved + .long 0 ; 53C8 reserved + .long 0 ; 53CC reserved + .long 0 ; 53D0 reserved + .long 0 ; 53D4 reserved + .long 0 ; 53D8 reserved + .long 0 ; 53DC reserved + .long 0 ; 53E0 reserved + .long 0 ; 53E4 reserved + .long 0 ; 53E8 reserved + .long 0 ; 53EC reserved + .long 0 ; 53F0 reserved + .long 0 ; 53F4 reserved + .long 0 ; 53F8 reserved + .long 0 ; 53FC reserved + + +; +; The "shared page" is used for low-level debugging +; + + . = 0x6000 + .globl EXT(sharedPage) + +EXT(sharedPage): ; Per processor data area + .long 0xC24BC195 ; Comm Area validity value + .long 0x87859393 ; Comm Area validity value + .long 0xE681A2C8 ; Comm Area validity value + .long 0x8599855A ; Comm Area validity value + .long 0xD74BD296 ; Comm Area validity value + .long 0x8388E681 ; Comm Area validity value + .long 0xA2C88599 ; Comm Area validity value + .short 0x855A ; Comm Area validity value + .short 1 ; Comm Area version number + .fill 1016*4,1,0 ; (filled with 0s) + + .data + .align ALIGN + .globl EXT(exception_end) +EXT(exception_end): + .long EXT(ExceptionVectorsEnd) -EXT(ExceptionVectorsStart) /* phys fn */ + + + diff --git a/osfmk/ppc/machine_routines.c b/osfmk/ppc/machine_routines.c index b4fa62fd5..00fbfc6af 100644 --- a/osfmk/ppc/machine_routines.c +++ b/osfmk/ppc/machine_routines.c @@ -247,6 +247,11 @@ ml_processor_register( per_proc_info[target_cpu].cpu_id = processor_info->cpu_id; per_proc_info[target_cpu].start_paddr = processor_info->start_paddr; + if (per_proc_info[target_cpu].pf.pfPowerModes & pmPowerTune) { + per_proc_info[target_cpu].pf.pfPowerTune0 = processor_info->power_mode_0; + per_proc_info[target_cpu].pf.pfPowerTune1 = processor_info->power_mode_1; + } + donap = processor_info->supports_nap; /* Assume we use requested nap */ if(forcenap) donap = forcenap - 1; /* If there was an override, use that */ diff --git a/osfmk/ppc/machine_routines.h b/osfmk/ppc/machine_routines.h index 1ad4f014b..5ce506b19 100644 --- a/osfmk/ppc/machine_routines.h +++ b/osfmk/ppc/machine_routines.h @@ -158,6 +158,8 @@ struct ml_processor_info { boolean_t supports_nap; unsigned long l2cr_value; time_base_enable_t time_base_enable; + uint32_t power_mode_0; + uint32_t power_mode_1; }; typedef struct ml_processor_info ml_processor_info_t; @@ -262,6 +264,8 @@ void ml_cpu_get_info(ml_cpu_info_t *cpu_info); void ml_set_processor_speed(unsigned long speed); void ml_set_processor_voltage(unsigned long voltage); +unsigned int ml_scom_write(uint32_t reg, uint64_t data); +unsigned int ml_scom_read(uint32_t reg, uint64_t *data); #endif /* __APPLE_API_PRIVATE */ diff --git a/osfmk/ppc/machine_routines_asm.s b/osfmk/ppc/machine_routines_asm.s index d68bc83b1..67b1da4f0 100644 --- a/osfmk/ppc/machine_routines_asm.s +++ b/osfmk/ppc/machine_routines_asm.s @@ -1725,10 +1725,7 @@ cdNoL3: /* Initialize processor thermal monitoring * void ml_thrm_init(void) * - * Build initial TAU registers and start them all going. - * We ca not do this at initial start up because we need to have the processor frequency first. - * And just why is this in assembler when it does not have to be?? Cause I am just too - * lazy to open up a "C" file, thats why. + * Obsolete, deprecated and will be removed. */ ; Force a line boundry here @@ -1736,53 +1733,12 @@ cdNoL3: .globl EXT(ml_thrm_init) LEXT(ml_thrm_init) - - mfsprg r12,0 ; Get the per_proc blok - lis r11,hi16(EXT(gPEClockFrequencyInfo)) ; Get top of processor information - mfsprg r10,2 ; Get CPU specific features - ori r11,r11,lo16(EXT(gPEClockFrequencyInfo)) ; Get bottom of processor information - mtcrf 0x40,r10 ; Get the installed features - - li r3,lo16(thrmtidm|thrmvm) ; Set for lower-than thermal event at 0 degrees - bflr pfThermalb ; No thermal monitoring on this cpu - mtspr thrm1,r3 ; Do it - - lwz r3,thrmthrottleTemp(r12) ; Get our throttle temprature - rlwinm r3,r3,31-thrmthre,thrmthrs,thrmthre ; Position it - ori r3,r3,lo16(thrmvm) ; Set for higher-than event - mtspr thrm2,r3 ; Set it - - lis r4,hi16(1000000) ; Top of million -; -; Note: some CPU manuals say this is processor clocks, some say bus rate. The latter -; makes more sense because otherwise we can not get over about 400MHz. -#if 0 - lwz r3,PECFIcpurate(r11) ; Get the processor speed -#else - lwz r3,PECFIbusrate(r11) ; Get the bus speed -#endif - ori r4,r4,lo16(1000000) ; Bottom of million - lis r7,hi16(thrmsitvm>>1) ; Get top of highest possible value - divwu r3,r3,r4 ; Get number of cycles per microseconds - ori r7,r7,lo16(thrmsitvm>>1) ; Get the bottom of the highest possible value - addi r3,r3,1 ; Insure we have enough - mulli r3,r3,20 ; Get 20 microseconds worth of cycles - cmplw r3,r7 ; Check against max - ble+ smallenuf ; It is ok... - mr r3,r7 ; Saturate - -smallenuf: rlwinm r3,r3,31-thrmsitve,thrmsitvs,thrmsitve ; Position - ori r3,r3,lo16(thrmem) ; Enable with at least 20micro sec sample - stw r3,thrm3val(r12) ; Save this in case we need it later - mtspr thrm3,r3 ; Do it blr - /* Set thermal monitor bounds * void ml_thrm_set(unsigned int low, unsigned int high) * - * Set TAU to interrupt below low and above high. A value of - * zero disables interruptions in that direction. + * Obsolete, deprecated and will be removed. */ ; Force a line boundry here @@ -1790,42 +1746,12 @@ smallenuf: rlwinm r3,r3,31-thrmsitve,thrmsitvs,thrmsitve ; Position .globl EXT(ml_thrm_set) LEXT(ml_thrm_set) - - mfmsr r0 ; Get the MSR - rlwinm r0,r0,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off - rlwinm r0,r0,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off - rlwinm r6,r0,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Clear EE bit - mtmsr r6 - isync - - mfsprg r12,0 ; Get the per_proc blok - - rlwinm. r6,r3,31-thrmthre,thrmthrs,thrmthre ; Position it and see if enabled - mfsprg r9,2 ; Get CPU specific features - stw r3,thrmlowTemp(r12) ; Set the low temprature - mtcrf 0x40,r9 ; See if we can thermal this machine - rlwinm r9,r9,(((31-thrmtie)+(pfThermIntb+1))&31),thrmtie,thrmtie ; Set interrupt enable if this machine can handle it - bf pfThermalb,tsetcant ; No can do... - beq tsetlowo ; We are setting the low off... - ori r6,r6,lo16(thrmtidm|thrmvm) ; Set the lower-than and valid bit - or r6,r6,r9 ; Set interruption request if supported - -tsetlowo: mtspr thrm1,r6 ; Cram the register - - rlwinm. r6,r4,31-thrmthre,thrmthrs,thrmthre ; Position it and see if enabled - stw r4,thrmhighTemp(r12) ; Set the high temprature - beq tsethigho ; We are setting the high off... - ori r6,r6,lo16(thrmvm) ; Set valid bit - or r6,r6,r9 ; Set interruption request if supported - -tsethigho: mtspr thrm2,r6 ; Cram the register - -tsetcant: mtmsr r0 ; Reenable interruptions - blr ; Leave... + blr /* Read processor temprature * unsigned int ml_read_temp(void) * + * Obsolete, deprecated and will be removed. */ ; Force a line boundry here @@ -1833,57 +1759,8 @@ tsetcant: mtmsr r0 ; Reenable interruptions .globl EXT(ml_read_temp) LEXT(ml_read_temp) - - mfmsr r9 ; Save the MSR - li r5,15 ; Starting point for ranging (start at 15 so we do not overflow) - rlwinm r9,r9,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off - rlwinm r9,r9,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off - rlwinm r8,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Turn off interruptions - mfsprg r7,2 ; Get CPU specific features - mtmsr r8 ; Do not allow interruptions - mtcrf 0x40,r7 ; See if we can thermal this machine - bf pfThermalb,thrmcant ; No can do... - - mfspr r11,thrm1 ; Save thrm1 - -thrmrange: rlwinm r4,r5,31-thrmthre,thrmthrs,thrmthre ; Position it - ori r4,r4,lo16(thrmtidm|thrmvm) ; Flip on the valid bit and make comparision for less than - - mtspr thrm1,r4 ; Set the test value - -thrmreada: mfspr r3,thrm1 ; Get the thermal register back - rlwinm. r0,r3,0,thrmtiv,thrmtiv ; Has it settled yet? - beq+ thrmreada ; Nope... - - rlwinm. r0,r3,0,thrmtin,thrmtin ; Are we still under the threshold? - bne thrmsearch ; No, we went over... - - addi r5,r5,16 ; Start by trying every 16 degrees - cmplwi r5,127 ; Have we hit the max? - blt- thrmrange ; Got some more to do... - -thrmsearch: rlwinm r4,r5,31-thrmthre,thrmthrs,thrmthre ; Position it - ori r4,r4,lo16(thrmtidm|thrmvm) ; Flip on the valid bit and make comparision for less than - - mtspr thrm1,r4 ; Set the test value - -thrmread: mfspr r3,thrm1 ; Get the thermal register back - rlwinm. r0,r3,0,thrmtiv,thrmtiv ; Has it settled yet? - beq+ thrmread ; Nope... - - rlwinm. r0,r3,0,thrmtin,thrmtin ; Are we still under the threshold? - beq thrmdone ; No, we hit it... - addic. r5,r5,-1 ; Go down a degree - bge+ thrmsearch ; Try again (until we are below freezing)... - -thrmdone: addi r3,r5,1 ; Return the temprature (bump it up to make it correct) - mtspr thrm1,r11 ; Restore the thermal register - mtmsr r9 ; Re-enable interruptions - blr ; Leave... - -thrmcant: eqv r3,r3,r3 ; Return bogus temprature because we can not read it - mtmsr r9 ; Re-enable interruptions - blr ; Leave... + li r3,-1 + blr /* Throttle processor speed up or down * unsigned int ml_throttle(unsigned int step) @@ -1891,6 +1768,7 @@ thrmcant: eqv r3,r3,r3 ; Return bogus temprature because we can not read i * Returns old speed and sets new. Both step and return are values from 0 to * 255 that define number of throttle steps, 0 being off and "ictcfim" is max * 2. * + * Obsolete, deprecated and will be removed. */ ; Force a line boundry here @@ -1898,26 +1776,8 @@ thrmcant: eqv r3,r3,r3 ; Return bogus temprature because we can not read i .globl EXT(ml_throttle) LEXT(ml_throttle) - - mfmsr r9 ; Save the MSR - rlwinm r9,r9,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off - rlwinm r9,r9,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off - rlwinm r8,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Turn off interruptions - cmplwi r3,lo16(ictcfim>>1) ; See if we are going too far - mtmsr r8 ; Do not allow interruptions - isync - ble+ throtok ; Throttle value is ok... - li r3,lo16(ictcfim>>1) ; Set max - -throtok: rlwinm. r4,r3,1,ictcfib,ictcfie ; Set the throttle - beq throtoff ; Skip if we are turning it off... - ori r4,r4,lo16(thrmvm) ; Turn on the valid bit - -throtoff: mfspr r3,ictc ; Get the old throttle - mtspr ictc,r4 ; Set the new - rlwinm r3,r3,31,1,31 ; Shift throttle value over - mtmsr r9 ; Restore interruptions - blr ; Return... + li r3,0 + blr /* ** ml_get_timebase() @@ -2034,17 +1894,25 @@ LEXT(ml_sense_nmi) .globl EXT(ml_set_processor_speed) LEXT(ml_set_processor_speed) - mfsprg r5, 0 ; Get the per_proc_info + mflr r0 ; Save the link register + stwu r1, -(FM_ALIGN(4*4)+FM_SIZE)(r1) ; Make some space on the stack + stw r28, FM_ARG0+0x00(r1) ; Save a register + stw r29, FM_ARG0+0x04(r1) ; Save a register + stw r30, FM_ARG0+0x08(r1) ; Save a register + stw r31, FM_ARG0+0x0C(r1) ; Save a register + stw r0, (FM_ALIGN(4*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return - lwz r6, pfPowerModes(r5) ; Get the supported power modes + mfsprg r31, 0 ; Get the per_proc_info + + lwz r30, pfPowerModes(r31) ; Get the supported power modes - rlwinm. r0, r6, 0, pmDualPLLb, pmDualPLLb ; Is DualPLL supported? + rlwinm. r0, r30, 0, pmDualPLLb, pmDualPLLb ; Is DualPLL supported? bne spsDPLL - rlwinm. r0, r6, 0, pmDFSb, pmDFSb ; Is DFS supported? + rlwinm. r0, r30, 0, pmDFSb, pmDFSb ; Is DFS supported? bne spsDFS - rlwinm. r0, r6, 0, pmPowerTuneb, pmPowerTuneb ; Is DualPLL supported? + rlwinm. r0, r30, 0, pmPowerTuneb, pmPowerTuneb ; Is PowerTune supported? bne spsPowerTune b spsDone ; No supported power modes @@ -2062,12 +1930,12 @@ spsDPLL: spsDPLL1: mfspr r4, hid1 ; Get the current PLL settings rlwimi r4, r3, 31-hid1ps, hid1ps, hid1ps ; Copy the PLL Select bit - stw r4, pfHID1(r5) ; Save the new hid1 value + stw r4, pfHID1(r31) ; Save the new hid1 value mtspr hid1, r4 ; Select desired PLL cmpli cr0, r3, 0 ; Restore BTIC after high speed bne spsDone - lwz r4, pfHID0(r5) ; Load the hid0 value + lwz r4, pfHID0(r31) ; Load the hid0 value sync mtspr hid0, r4 ; Set the hid0 value isync @@ -2080,8 +1948,9 @@ spsDFS: rlwinm r3, r3, 0, hid1dfs1+1, hid1dfs0-1 ; assume full speed, clear dfs bits beq spsDFS1 oris r3, r3, hi16(hid1dfs1m) ; slow, set half speed dfs1 bit + spsDFS1: - stw r3, pfHID1(r5) ; Save the new hid1 value + stw r3, pfHID1(r31) ; Save the new hid1 value sync mtspr hid1, r3 ; Set the new HID1 sync @@ -2089,9 +1958,76 @@ spsDFS1: b spsDone spsPowerTune: + rlwinm r28, r3, 31-dnap, dnap, dnap ; Shift the 1 bit to the dnap+32 bit + rlwinm r3, r3, 2, 29, 29 ; Shift the 1 to a 4 and mask + addi r3, r3, pfPowerTune0 ; Add in the pfPowerTune0 offset + lwzx r29, r31, r3 ; Load the PowerTune number 0 or 1 + + sldi r28, r28, 32 ; Shift to the top half + ld r3, pfHID0(r31) ; Load the saved hid0 value + and r28, r28, r3 ; Save the dnap bit + lis r4, hi16(dnapm) ; Make a mask for the dnap bit + sldi r4, r4, 32 ; Shift to the top half + andc r3, r3, r4 ; Clear the dnap bit + or r28, r28, r3 ; Insert the dnap bit as needed for later + + sync + mtspr hid0, r3 ; Turn off dnap in hid0 + mfspr r3, hid0 ; Yes, this is silly, keep it here + mfspr r3, hid0 ; Yes, this is a duplicate, keep it here + mfspr r3, hid0 ; Yes, this is a duplicate, keep it here + mfspr r3, hid0 ; Yes, this is a duplicate, keep it here + mfspr r3, hid0 ; Yes, this is a duplicate, keep it here + mfspr r3, hid0 ; Yes, this is a duplicate, keep it here + isync ; Make sure it is set + + lis r3, hi16(PowerTuneControlReg) ; Write zero to the PCR + ori r3, r3, lo16(PowerTuneControlReg) + li r4, 0 + li r5, 0 + bl _ml_scom_write + + lis r3, hi16(PowerTuneControlReg) ; Write the PowerTune value to the PCR + ori r3, r3, lo16(PowerTuneControlReg) + li r4, 0 + mr r5, r29 + bl _ml_scom_write + + rlwinm r29, r29, 13-6, 6, 7 ; Move to PSR speed location and isolate the requested speed +spsPowerTuneLoop: + lis r3, hi16(PowerTuneStatusReg) ; Read the status from the PSR + ori r3, r3, lo16(PowerTuneStatusReg) + li r4, 0 + bl _ml_scom_read + srdi r5, r5, 32 + rlwinm r0, r5, 0, 6, 7 ; Isolate the current speed + rlwimi r0, r5, 0, 2, 2 ; Copy in the change in progress bit + cmpw r0, r29 ; Compare the requested and current speeds + beq spsPowerTuneDone + rlwinm. r0, r5, 0, 3, 3 + beq spsPowerTuneLoop + +spsPowerTuneDone: + sync + mtspr hid0, r28 ; Turn on dnap in hid0 if needed + mfspr r28, hid0 ; Yes, this is silly, keep it here + mfspr r28, hid0 ; Yes, this is a duplicate, keep it here + mfspr r28, hid0 ; Yes, this is a duplicate, keep it here + mfspr r28, hid0 ; Yes, this is a duplicate, keep it here + mfspr r28, hid0 ; Yes, this is a duplicate, keep it here + mfspr r28, hid0 ; Yes, this is a duplicate, keep it here + isync ; Make sure it is set + b spsDone spsDone: + lwz r0, (FM_ALIGN(4*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Get the return + lwz r28, FM_ARG0+0x00(r1) ; Restore a register + lwz r29, FM_ARG0+0x04(r1) ; Restore a register + lwz r30, FM_ARG0+0x08(r1) ; Restore a register + lwz r31, FM_ARG0+0x0C(r1) ; Restore a register + lwz r1, FM_BACKPTR(r1) ; Pop the stack + mtlr r0 blr /* @@ -2117,3 +2053,59 @@ LEXT(ml_set_processor_voltage) spvDone: blr + + +; +; unsigned int ml_scom_write(unsigned int reg, unsigned long long data) +; 64-bit machines only +; returns status +; + + .align 5 + .globl EXT(ml_scom_write) + +LEXT(ml_scom_write) + + rldicr r3,r3,8,47 ; Align register it correctly + rldimi r5,r4,32,0 ; Merge the high part of data + sync ; Clean up everything + + mtspr scomd,r5 ; Stick in the data + mtspr scomc,r3 ; Set write to register + sync + isync + + mfspr r3,scomc ; Read back status + blr ; leave.... + +; +; unsigned int ml_read_scom(unsigned int reg, unsigned long long *data) +; 64-bit machines only +; returns status +; ASM Callers: data (r4) can be zero and the 64 bit data will be returned in r5 +; + + .align 5 + .globl EXT(ml_scom_read) + +LEXT(ml_scom_read) + + mfsprg r0,2 ; Get the feature flags + rldicr r3,r3,8,47 ; Align register it correctly + rlwinm r0,r0,pfSCOMFixUpb+1,31,31 ; Set shift if we need a fix me up + + ori r3,r3,0x8000 ; Set to read data + sync + + mtspr scomc,r3 ; Request the register + mfspr r5,scomd ; Get the register contents + mfspr r3,scomc ; Get back the status + sync + isync + + sld r5,r5,r0 ; Fix up if needed + + cmplwi r4, 0 ; If data pointer is null, just return + beqlr ; the received data in r5 + std r5,0(r4) ; Pass back the received data + blr ; Leave... diff --git a/osfmk/ppc/start.s b/osfmk/ppc/start.s index 8ad798baa..970a6b824 100644 --- a/osfmk/ppc/start.s +++ b/osfmk/ppc/start.s @@ -45,15 +45,13 @@ #define ptPatch 20 #define ptInitRout 24 #define ptRptdProc 28 -#define ptTempMax 32 -#define ptTempThr 36 -#define ptLineSize 40 -#define ptl1iSize 44 -#define ptl1dSize 48 -#define ptPTEG 52 -#define ptMaxVAddr 56 -#define ptMaxPAddr 60 -#define ptSize 64 +#define ptLineSize 32 +#define ptl1iSize 36 +#define ptl1dSize 40 +#define ptPTEG 44 +#define ptMaxVAddr 48 +#define ptMaxPAddr 52 +#define ptSize 56 #define bootCPU 10 #define firstInit 9 @@ -182,11 +180,6 @@ donePVR: lwz r20,ptInitRout(r26) ; Grab the special init routine lwz r13,ptPwrModes(r26) ; Get the supported power modes stw r13,pfPowerModes(r30) ; Set the supported power modes - lwz r13,ptTempMax(r26) ; Get maximum operating temperature - stw r13,thrmmaxTemp(r30) ; Set the maximum - lwz r13,ptTempThr(r26) ; Get temprature to throttle down when exceeded - stw r13,thrmthrottleTemp(r30) ; Set the temperature that we throttle - lwz r13,ptLineSize(r26) ; Get the cache line size sth r13,pflineSize(r30) ; Save it lwz r13,ptl1iSize(r26) ; Get icache size @@ -248,7 +241,7 @@ doOurInit: mr. r20,r20 ; See if initialization routine cpyFCpu: addic. r2,r2,-1 ; Count down la r8,pfAvailable(r23) ; Point to features of boot processor la r7,pfAvailable(r6) ; Point to features of our processor - li r9,(pfSize+thrmSize)/4 ; Get size of a features area + li r9,pfSize/4 ; Get size of a features area ble-- nofeatcpy ; Copied all we need cpyFeat: subi r9,r9,1 ; Count word @@ -435,18 +428,7 @@ noVector: rlwinm. r0,r17,0,pfSMPcapb,pfSMPcapb ; See if we can do SMP lhz r13,PP_CPU_NUMBER(r30) ; Get the CPU number mtspr pir,r13 ; Set the PIR -noSMP: rlwinm. r0,r17,0,pfThermalb,pfThermalb ; See if there is an TAU - beq- noThermometer ; Nope... - - li r13,0 ; Disable thermals for now - mtspr thrm3,r13 ; Do it - li r13,lo16(thrmtidm|thrmvm) ; Set for lower-than thermal event at 0 degrees - mtspr thrm1,r13 ; Do it - lis r13,hi16(thrmthrm) ; Set 127 degrees - ori r13,r13,lo16(thrmvm) ; Set for higher-than event - mtspr thrm2,r13 ; Set it - -noThermometer: +noSMP: bl EXT(cacheInit) ; Initializes all caches (including the TLB) @@ -783,7 +765,7 @@ init7450done: init970: li r20,0 ; Clear this - mtspr hior,r20 ; Make sure that 0 is interrupt prefix + mtspr hior,r20 ; Make sure that 0 is interrupt prefix bf firstBoot,init970nb ; No init for wakeup or second processor.... @@ -795,7 +777,21 @@ init970: std r11,pfHID4(r30) ; Save original mfspr r11,hid5 ; Get original hid5 std r11,pfHID5(r30) ; Save original - + + lis r0, hi16(dnapm) ; Create a mask for the dnap bit + sldi r0, r0, 32 ; Shift to the top half + ld r11,pfHID0(r30) ; Load the hid0 value + andc r11, r11, r0 ; Clear the dnap bit + isync + mtspr hid0,r11 ; Stuff it + mfspr r11,hid0 ; Get it + mfspr r11,hid0 ; Get it + mfspr r11,hid0 ; Get it + mfspr r11,hid0 ; Get it + mfspr r11,hid0 ; Get it + mfspr r11,hid0 ; Get it + isync + ; ; We can not query or change the L2 size. We will just ; phoney up a L2CR to make sysctl "happy" and set the @@ -814,7 +810,11 @@ init970: ; Start up code for second processor or wake up from sleep ; -init970nb: ld r11,pfHID0(r30) ; Get it +init970nb: + lis r0, hi16(dnapm) ; Create a mask for the dnap bit + sldi r0, r0, 32 ; Shift to the top half + ld r11,pfHID0(r30) ; Load the hid0 value + andc r11, r11, r0 ; Clear the dnap bit isync mtspr hid0,r11 ; Stuff it mfspr r11,hid0 ; Get it @@ -862,8 +862,6 @@ initUnsupported: ; .long ptPatch - Patch features ; .long ptInitRout - Initilization routine. Can modify any of the other attributes. ; .long ptRptdProc - Processor type reported -; .long ptTempMax - Maximum operating temprature -; .long ptTempThr - Temprature threshold. We throttle if above ; .long ptLineSize - Level 1 cache line size ; .long ptl1iSize - Level 1 instruction cache size ; .long ptl1dSize - Level 1 data cache size @@ -875,27 +873,6 @@ initUnsupported: .align 2 processor_types: -; 750 (ver 2.2) - - .align 2 - .long 0xFFFFFFFF ; Exact match - .short PROCESSOR_VERSION_750 - .short 0x4202 - .long pfFloat | pfCanSleep | pfCanNap | pfCanDoze | pf32Byte | pfL2 - .long kCache32 | kHasGraphicsOps | kHasStfiwx - .long 0 - .long PatchExt32 - .long init750 - .long CPU_SUBTYPE_POWERPC_750 - .long 105 - .long 90 - .long 32 - .long 32*1024 - .long 32*1024 - .long 64 - .long 52 - .long 32 - ; 750CX (ver 2.x) .align 2 @@ -908,8 +885,6 @@ processor_types: .long PatchExt32 .long init750CX .long CPU_SUBTYPE_POWERPC_750 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -923,14 +898,12 @@ processor_types: .long 0xFFFF0000 ; All revisions .short PROCESSOR_VERSION_750 .short 0 - .long pfFloat | pfCanSleep | pfCanNap | pfCanDoze | pfThermal | pf32Byte | pfL2 + .long pfFloat | pfCanSleep | pfCanNap | pfCanDoze | pf32Byte | pfL2 .long kCache32 | kHasGraphicsOps | kHasStfiwx .long 0 .long PatchExt32 .long init750 .long CPU_SUBTYPE_POWERPC_750 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -950,8 +923,6 @@ processor_types: .long PatchExt32 .long init750FX .long CPU_SUBTYPE_POWERPC_750 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -971,8 +942,6 @@ processor_types: .long PatchExt32 .long init750FXV2 .long CPU_SUBTYPE_POWERPC_750 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -986,14 +955,12 @@ processor_types: .long 0xFFFFFFF8 ; ver 2.0 - 2.7 .short PROCESSOR_VERSION_7400 .short 0x0200 - .long pfFloat | pfAltivec | pfSMPcap | pfCanSleep | pfCanNap | pfCanDoze | pfThermal | pf32Byte | pfL1fa | pfL2 | pfL2fa | pfHasDcba + .long pfFloat | pfAltivec | pfSMPcap | pfCanSleep | pfCanNap | pfCanDoze | pf32Byte | pfL1fa | pfL2 | pfL2fa | pfHasDcba .long kHasAltivec | kCache32 | kDcbaAvailable | kDataStreamsAvailable | kHasGraphicsOps | kHasStfiwx .long 0 .long PatchExt32 .long init7400v2_7 .long CPU_SUBTYPE_POWERPC_7400 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -1007,14 +974,12 @@ processor_types: .long 0xFFFF0000 ; All revisions .short PROCESSOR_VERSION_7400 .short 0 - .long pfFloat | pfAltivec | pfSMPcap | pfCanSleep | pfCanNap | pfCanDoze | pfThermal | pf32Byte | pfL1fa | pfL2 | pfL2fa | pfHasDcba + .long pfFloat | pfAltivec | pfSMPcap | pfCanSleep | pfCanNap | pfCanDoze | pf32Byte | pfL1fa | pfL2 | pfL2fa | pfHasDcba .long kHasAltivec | kCache32 | kDcbaAvailable | kDataStreamsRecommended | kDataStreamsAvailable | kHasGraphicsOps | kHasStfiwx .long 0 .long PatchExt32 .long init7400 .long CPU_SUBTYPE_POWERPC_7400 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -1034,8 +999,6 @@ processor_types: .long PatchExt32 .long init7410 .long CPU_SUBTYPE_POWERPC_7400 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -1055,8 +1018,6 @@ processor_types: .long PatchExt32 .long init7410 .long CPU_SUBTYPE_POWERPC_7400 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -1076,8 +1037,6 @@ processor_types: .long PatchExt32 .long init7450 .long CPU_SUBTYPE_POWERPC_7450 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -1097,8 +1056,6 @@ processor_types: .long PatchExt32 .long init7450 .long CPU_SUBTYPE_POWERPC_7450 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -1118,8 +1075,6 @@ processor_types: .long PatchExt32 .long init7450 .long CPU_SUBTYPE_POWERPC_7450 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -1139,8 +1094,6 @@ processor_types: .long PatchExt32 .long init745X .long CPU_SUBTYPE_POWERPC_7450 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -1160,8 +1113,6 @@ processor_types: .long PatchExt32 .long init745X .long CPU_SUBTYPE_POWERPC_7450 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -1181,8 +1132,6 @@ processor_types: .long PatchExt32 .long init745X .long CPU_SUBTYPE_POWERPC_7450 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -1202,8 +1151,6 @@ processor_types: .long PatchExt32 .long init745X .long CPU_SUBTYPE_POWERPC_7450 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -1223,8 +1170,6 @@ processor_types: .long PatchExt32 .long init745X .long CPU_SUBTYPE_POWERPC_7450 - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 @@ -1232,27 +1177,6 @@ processor_types: .long 52 .long 36 -; 970FX DD1.0 - - .align 2 - .long 0xFFFFFF00 ; All versions so far - .short PROCESSOR_VERSION_970 - .short 0x1100 - .long pfFloat | pfAltivec | pfSMPcap | pfCanSleep | pfCanNap | pf128Byte | pf64Bit | pfL2 - .long kHasAltivec | k64Bit | kCache128 | kDataStreamsAvailable | kDcbtStreamsRecommended | kDcbtStreamsAvailable | kHasGraphicsOps | kHasStfiwx | kHasFsqrt - .long pmPowerTune - .long PatchLwsync - .long init970 - .long CPU_SUBTYPE_POWERPC_970 - .long 105 - .long 90 - .long 128 - .long 64*1024 - .long 32*1024 - .long 128 - .long 65 - .long 42 - ; 970 .align 2 @@ -1265,8 +1189,6 @@ processor_types: .long PatchLwsync .long init970 .long CPU_SUBTYPE_POWERPC_970 - .long 105 - .long 90 .long 128 .long 64*1024 .long 32*1024 @@ -1286,8 +1208,6 @@ processor_types: .long PatchLwsync .long init970 .long CPU_SUBTYPE_POWERPC_970 - .long 105 - .long 90 .long 128 .long 64*1024 .long 32*1024 @@ -1307,8 +1227,6 @@ processor_types: .long PatchExt32 .long initUnsupported .long CPU_SUBTYPE_POWERPC_ALL - .long 105 - .long 90 .long 32 .long 32*1024 .long 32*1024 diff --git a/osfmk/vm/task_working_set.c b/osfmk/vm/task_working_set.c index 220c05606..51900109d 100644 --- a/osfmk/vm/task_working_set.c +++ b/osfmk/vm/task_working_set.c @@ -352,7 +352,7 @@ tws_internal_lookup( int age_of_cache; age_of_cache = ((sched_tick - tws->time_of_creation) >> SCHED_TICK_SHIFT); - if (age_of_cache > 35) { + if (age_of_cache > 45) { return KERN_OPERATION_TIMED_OUT; } } @@ -364,7 +364,7 @@ tws_internal_lookup( int age_of_cache; age_of_cache = ((sched_tick - tws->time_of_creation) >> SCHED_TICK_SHIFT); - if (age_of_cache > 60) { + if (age_of_cache > 45) { return KERN_OPERATION_TIMED_OUT; } } @@ -442,7 +442,7 @@ tws_expand_working_set( if(entry->object != 0) { paddr = 0; for(page_index = 1; page_index != 0; - page_index = page_index << 1); { + page_index = page_index << 1) { if (entry->page_cache & page_index) { tws_insert(new_tws, entry->offset+paddr, @@ -549,7 +549,7 @@ tws_insert( unsigned int startup_cache_line; vm_offset_t startup_page_addr; int cache_full = 0; - int ask_for_startup_cache_release = 0; + int age_of_cache = 0; if(!tws_lock_try(tws)) { @@ -559,20 +559,15 @@ tws_insert( current_line = 0xFFFFFFFF; startup_cache_line = 0; - startup_page_addr = - page_addr - (offset - (offset & TWS_HASH_OFF_MASK)); - if(tws->startup_cache) { - int age_of_cache; - age_of_cache = ((sched_tick - tws->time_of_creation) - >> SCHED_TICK_SHIFT); - startup_cache_line = tws_startup_list_lookup( - tws->startup_cache, startup_page_addr); -if(tws == test_tws) { -printf("cache_lookup, result = 0x%x, addr = 0x%x, object 0x%x, offset 0x%x%x\n", startup_cache_line, startup_page_addr, object, offset); -} - if(age_of_cache > 60) { - ask_for_startup_cache_release = 1; - } + + if (tws->startup_cache) { + vm_offset_t startup_page_addr; + + startup_page_addr = page_addr - (offset - (offset & TWS_HASH_OFF_MASK)); + + age_of_cache = ((sched_tick - tws->time_of_creation) >> SCHED_TICK_SHIFT); + + startup_cache_line = tws_startup_list_lookup(tws->startup_cache, startup_page_addr); } /* This next bit of code, the and alternate hash */ /* are all made necessary because of IPC COW */ @@ -610,7 +605,7 @@ printf("cache_lookup, result = 0x%x, addr = 0x%x, object 0x%x, offset 0x%x%x\n", (1<<(((vm_offset_t) (offset & TWS_INDEX_MASK))>>12)); tws_unlock(tws); - if(ask_for_startup_cache_release) + if (age_of_cache > 45) return KERN_OPERATION_TIMED_OUT; return KERN_SUCCESS; } @@ -838,29 +833,16 @@ printf("cache_lookup, result = 0x%x, addr = 0x%x, object 0x%x, offset 0x%x%x\n", } vm_object_lock(object); } else { - int age_of_cache; - age_of_cache = - ((sched_tick - - tws->time_of_creation) - >> SCHED_TICK_SHIFT); - - if((tws->startup_cache) && - (age_of_cache > 60)) { - ask_for_startup_cache_release = 1; - } - if((tws->startup_name != NULL) && - (age_of_cache > 15)) { - tws->current_line--; - tws_unlock(tws); - return KERN_OPERATION_TIMED_OUT; - } - if((tws->startup_name != NULL) && - (age_of_cache < 15)) { - /* If we are creating a */ - /* cache, don't lose the */ - /* early info */ + if (tws->startup_name != NULL) { tws->current_line--; + + age_of_cache = ((sched_tick - tws->time_of_creation) >> SCHED_TICK_SHIFT); + tws_unlock(tws); + + if (age_of_cache > 45) + return KERN_OPERATION_TIMED_OUT; + return KERN_FAILURE; } tws->lookup_count = 0; @@ -939,16 +921,15 @@ printf("cache_lookup, result = 0x%x, addr = 0x%x, object 0x%x, offset 0x%x%x\n", target_element->map = map; target_element->line = current_line + (set * tws->number_of_lines); - if(startup_cache_line) { - target_element->page_cache = startup_cache_line; - } - target_element->page_cache |= - 1<<(((vm_offset_t)(offset & TWS_INDEX_MASK))>>12); + target_element->page_cache |= startup_cache_line; + target_element->page_cache |= 1<<(((vm_offset_t)(offset & TWS_INDEX_MASK))>>12); tws_unlock(tws); - if(ask_for_startup_cache_release) + + if (age_of_cache > 45) return KERN_OPERATION_TIMED_OUT; + return KERN_SUCCESS; } @@ -1057,77 +1038,64 @@ tws_build_cluster( /* When pre-heat files are not available, resort to speculation */ /* based on size of file */ - if(tws->startup_cache || object->internal || age_of_cache > 15 || - (age_of_cache > 5 && - vm_page_free_count < (vm_page_free_target * 2) )) { - pre_heat_size = 0; + if (tws->startup_cache || object->internal || age_of_cache > 45) { + pre_heat_size = 0; } else { if (object_size > (vm_object_offset_t)(1024 * 1024)) - pre_heat_size = 8 * PAGE_SIZE; + pre_heat_size = 16 * PAGE_SIZE; else if (object_size > (vm_object_offset_t)(128 * 1024)) - pre_heat_size = 4 * PAGE_SIZE; + pre_heat_size = 8 * PAGE_SIZE; else - pre_heat_size = 2 * PAGE_SIZE; + pre_heat_size = 4 * PAGE_SIZE; } - if ((age_of_cache < 10) && (tws->startup_cache)) { - if ((max_length >= ((*end - *start) - + (32 * PAGE_SIZE))) && - (tws_test_for_community(tws, object, - *start, 3, &ele_cache))) { - int expanded; - start_cache = ele_cache; + if (tws->startup_cache) { + + if (tws_test_for_community(tws, object, *start, 4, &ele_cache)) + { + start_cache = ele_cache; *start = *start & TWS_HASH_OFF_MASK; *end = *start + (32 * PAGE_SIZE_64); - if(*end > object_size) { - *end = trunc_page_64(object_size); + + if (*end > object_size) { + *end = round_page_64(object_size); max_length = 0; - if(before >= *end) { - *end = after; - } else { - end_cache = ele_cache; - } - } else { - end_cache = ele_cache; - } - while (max_length > ((*end - *start) - + (32 * PAGE_SIZE))) { + } else + end_cache = ele_cache; + + while (max_length > ((*end - *start) + (32 * PAGE_SIZE))) { + int expanded; + expanded = 0; after = *end; - before = *start - PAGE_SIZE_64; - if((*end <= (object->size - + (32 * PAGE_SIZE_64))) && - (tws_test_for_community(tws, - object, after, - 5, &ele_cache))) { - *end = after + - (32 * PAGE_SIZE_64); - if(*end > object_size) { - *end = trunc_page_64(object_size); - max_length = 0; - if(*start >= *end) { - *end = after; - } - } + + if ((after + (32 * PAGE_SIZE_64)) <= object_size && + (tws_test_for_community(tws, object, after, 8, &ele_cache))) { + + *end = after + (32 * PAGE_SIZE_64); end_cache = ele_cache; expanded = 1; } - if (max_length > ((*end - *start) - + (32 * PAGE_SIZE_64))) { + if (max_length < ((*end - *start) + (32 * PAGE_SIZE_64))) { break; } - if((*start >= (32 * PAGE_SIZE_64)) && - (tws_test_for_community(tws, object, - before, 5, &ele_cache))) { - *start = before; - start_cache = ele_cache; - expanded = 1; + if (*start) { + before = (*start - PAGE_SIZE_64) & TWS_HASH_OFF_MASK; + + if (tws_test_for_community(tws, object, before, 8, &ele_cache)) { + + *start = before; + start_cache = ele_cache; + expanded = 1; + } } - if(expanded == 0) + if (expanded == 0) break; } + if (end_cache) + *end -= PAGE_SIZE_64; - if(start_cache != 0) { + if (start_cache != 0) { unsigned int mask; for (mask = 1; mask != 0; mask = mask << 1) { @@ -1139,24 +1107,23 @@ tws_build_cluster( break; } } - if(end_cache != 0) { + if (end_cache != 0) { unsigned int mask; for (mask = 0x80000000; mask != 0; mask = mask >> 1) { if (*end == original_end) break; - if(!(end_cache & mask)) + if (!(end_cache & mask)) *end -= PAGE_SIZE_64; else break; } } - - if (*start >= *end) - panic("bad clipping occurred\n"); - tws_unlock(tws); + + if (*end < original_end) + *end = original_end; return; } } @@ -1528,7 +1495,7 @@ tws_startup_list_lookup( hash_index = do_startup_hash(addr, startup->array_size); - if(((unsigned int)&(startup->table[hash_index])) >= startup->tws_hash_size) { + if(((unsigned int)&(startup->table[hash_index])) >= ((unsigned int)startup + startup->tws_hash_size)) { return page_cache_bits = 0; } element = (tws_startup_ptr_t)((int)startup->table[hash_index] + diff --git a/osfmk/vm/task_working_set.h b/osfmk/vm/task_working_set.h index baddb7c81..fc7fda91e 100644 --- a/osfmk/vm/task_working_set.h +++ b/osfmk/vm/task_working_set.h @@ -150,7 +150,7 @@ typedef struct tws_hash_line *tws_hash_line_t; #define TWS_ADDR_HASH 1 -#define TWS_HASH_EXPANSION_MAX 5 +#define TWS_HASH_EXPANSION_MAX 10 #define TWS_MAX_REHASH 3 diff --git a/pexpert/conf/version.minor b/pexpert/conf/version.minor index b8626c4cf..7ed6ff82d 100644 --- a/pexpert/conf/version.minor +++ b/pexpert/conf/version.minor @@ -1 +1 @@ -4 +5 -- 2.45.2