From 4452a7af2eac33dbad800bcc91f2399d62c18f53 Mon Sep 17 00:00:00 2001
From: Apple <opensource@apple.com>
Date: Tue, 30 Oct 2007 17:48:46 +0000
Subject: [PATCH] xnu-792.22.5.tar.gz

---
 EXTERNAL_HEADERS/Makefile                     |    1 +
 EXTERNAL_HEADERS/architecture/Makefile        |    6 +-
 EXTERNAL_HEADERS/architecture/byte_order.h    |  531 ---
 EXTERNAL_HEADERS/architecture/i386/Makefile   |    4 +-
 .../architecture/i386/byte_order.h            |  154 -
 EXTERNAL_HEADERS/architecture/i386/fpu.h      |  158 -
 EXTERNAL_HEADERS/architecture/i386/frame.h    |  131 -
 EXTERNAL_HEADERS/architecture/i386/pio.h      |  109 +
 EXTERNAL_HEADERS/architecture/ppc/Makefile    |    1 -
 .../architecture/ppc/byte_order.h             |  205 --
 EXTERNAL_HEADERS/mach-o/fat.h                 |    3 +-
 EXTERNAL_HEADERS/mach-o/loader.h              |    9 +-
 EXTERNAL_HEADERS/stdbool.h                    |   46 +
 bsd/bsm/audit_kevents.h                       |    1 +
 bsd/conf/MASTER                               |    2 +-
 bsd/conf/MASTER.i386                          |    4 +-
 bsd/conf/Makefile.i386                        |   72 +-
 bsd/conf/files                                |    5 +
 bsd/conf/files.i386                           |    2 +
 bsd/conf/files.ppc                            |    3 -
 bsd/conf/param.c                              |    3 +-
 bsd/dev/{ppc => }/chud/chud_bsd_callback.c    |   69 +-
 bsd/dev/{ppc => }/chud/chud_process.c         |   35 +-
 bsd/dev/i386/kern_machdep.c                   |   93 +-
 bsd/dev/i386/km.c                             |    6 +-
 bsd/dev/i386/mem.c                            |   16 +-
 bsd/dev/i386/munge.s                          |  211 ++
 bsd/dev/i386/pio.h                            |  248 --
 bsd/dev/i386/stubs.c                          |   83 +-
 bsd/dev/i386/sysctl.c                         |   47 +-
 bsd/dev/i386/systemcalls.c                    |  616 ++++
 bsd/dev/i386/unix_signal.c                    |  634 +++-
 bsd/dev/ppc/kern_machdep.c                    |    7 +-
 bsd/dev/ppc/mem.c                             |   13 +-
 bsd/dev/ppc/munge.s                           |   27 +
 bsd/dev/ppc/systemcalls.c                     |   95 +-
 bsd/dev/ppc/unix_signal.c                     |   58 +-
 bsd/dev/unix_startup.c                        |   83 +-
 bsd/dev/vn/vn.c                               |   32 +-
 bsd/hfs/hfs.h                                 |   10 +-
 bsd/hfs/hfs_btreeio.c                         |   10 +-
 bsd/hfs/hfs_chash.c                           |    3 -
 bsd/hfs/hfs_cnode.c                           |   37 +-
 bsd/hfs/hfs_cnode.h                           |    3 +
 bsd/hfs/hfs_encodings.c                       |    3 -
 bsd/hfs/hfs_endian.c                          |    2 -
 bsd/hfs/hfs_endian.h                          |   16 +-
 bsd/hfs/hfs_fsctl.h                           |    3 +
 bsd/hfs/hfs_hotfiles.c                        |   10 +-
 bsd/hfs/hfs_quota.c                           |   25 +-
 bsd/hfs/hfs_readwrite.c                       |   57 +-
 bsd/hfs/hfs_vfsops.c                          |  361 +-
 bsd/hfs/hfs_vfsutils.c                        |   11 +-
 bsd/hfs/hfs_vnops.c                           |   24 +-
 bsd/hfs/hfscommon/BTree/BTreeNodeReserve.c    |    2 -
 bsd/i386/_types.h                             |    2 +-
 bsd/i386/param.h                              |    2 +-
 bsd/i386/reg.h                                |   15 -
 bsd/i386/setjmp.h                             |   15 +
 bsd/i386/types.h                              |   21 +-
 bsd/i386/ucontext.h                           |   40 +-
 bsd/i386/vmparam.h                            |    9 +-
 bsd/isofs/cd9660/cd9660_lookup.c              |    3 -
 bsd/isofs/cd9660/cd9660_mount.h               |   10 +-
 bsd/isofs/cd9660/cd9660_util.c                |   19 +-
 bsd/isofs/cd9660/cd9660_vfsops.c              |    7 +-
 bsd/isofs/cd9660/cd9660_vnops.c               |    9 +-
 bsd/kern/bsd_init.c                           |   42 +-
 bsd/kern/bsd_stubs.c                          |   67 +-
 bsd/kern/init_sysent.c                        |   33 +-
 bsd/kern/kdebug.c                             |  873 +++--
 bsd/kern/kern_acct.c                          |    5 +-
 bsd/kern/kern_aio.c                           |    2 -
 bsd/kern/kern_audit.c                         |   76 +-
 bsd/kern/kern_authorization.c                 |  126 +-
 bsd/kern/kern_control.c                       |   62 +-
 bsd/kern/kern_core.c                          |   14 +-
 bsd/kern/kern_credential.c                    |  130 +-
 bsd/kern/kern_descrip.c                       |  115 +-
 bsd/kern/kern_event.c                         |  170 +-
 bsd/kern/kern_exec.c                          |  378 ++-
 bsd/kern/kern_exit.c                          |   14 +-
 bsd/kern/kern_fork.c                          |   36 +-
 bsd/kern/kern_ktrace.c                        |    2 +-
 bsd/kern/kern_malloc.c                        |    1 +
 bsd/kern/kern_mib.c                           |  117 +-
 bsd/kern/kern_proc.c                          |    2 +-
 bsd/kern/kern_prot.c                          |  115 +-
 bsd/kern/kern_resource.c                      |    2 +-
 bsd/kern/kern_shutdown.c                      |    4 +
 bsd/kern/kern_sig.c                           |   72 +-
 bsd/kern/kern_symfile.c                       |   29 +-
 bsd/kern/kern_sysctl.c                        |  288 +-
 bsd/kern/kern_time.c                          |   53 +-
 bsd/kern/kpi_mbuf.c                           |    4 +-
 bsd/kern/kpi_socketfilter.c                   |    3 -
 bsd/kern/mach_fat.c                           |   44 +-
 bsd/kern/mach_loader.c                        |  222 +-
 bsd/kern/mach_process.c                       |   86 +-
 bsd/kern/makesyscalls.sh                      |   44 +-
 bsd/kern/md5c.c                               |    2 +-
 bsd/kern/netboot.c                            |    3 +-
 bsd/kern/posix_sem.c                          |   37 +-
 bsd/kern/posix_shm.c                          |   45 +-
 bsd/kern/proc_info.c                          | 1064 ++++++
 bsd/kern/socket_info.c                        |  247 ++
 bsd/kern/subr_prof.c                          |    1 -
 bsd/kern/sys_generic.c                        |    2 -
 bsd/kern/sys_pipe.c                           |   68 +-
 bsd/kern/syscalls.c                           |   29 +-
 bsd/kern/syscalls.master                      |   35 +-
 bsd/kern/sysctl_init.c                        |   44 +
 bsd/kern/sysv_msg.c                           |    2 -
 bsd/kern/sysv_sem.c                           |    4 +-
 bsd/kern/sysv_shm.c                           |   18 +-
 bsd/kern/tty_tty.c                            |    3 +-
 bsd/kern/ubc_subr.c                           |   16 +-
 bsd/kern/uipc_domain.c                        |    2 -
 bsd/kern/uipc_mbuf.c                          |    8 +-
 bsd/kern/uipc_mbuf2.c                         |   40 +
 bsd/kern/uipc_socket.c                        |   48 +-
 bsd/kern/uipc_socket2.c                       |   37 +-
 bsd/kern/uipc_syscalls.c                      |    5 +-
 bsd/kern/uipc_usrreq.c                        |   29 +-
 bsd/machine/_limits.h                         |    2 +-
 bsd/machine/_types.h                          |    2 +-
 bsd/machine/cons.h                            |    2 +-
 bsd/machine/disklabel.h                       |    2 +-
 bsd/machine/endian.h                          |    2 +-
 bsd/machine/exec.h                            |   13 +-
 bsd/machine/param.h                           |    2 +-
 bsd/machine/profile.h                         |    2 +-
 bsd/machine/psl.h                             |    2 +-
 bsd/machine/ptrace.h                          |    2 +-
 bsd/machine/reboot.h                          |    2 +-
 bsd/machine/reg.h                             |    2 +-
 bsd/machine/setjmp.h                          |    2 +-
 bsd/machine/signal.h                          |    2 +-
 bsd/machine/types.h                           |    2 +-
 bsd/machine/ucontext.h                        |    2 +-
 bsd/machine/vmparam.h                         |    2 +-
 bsd/man/man2/Makefile                         |    3 +-
 bsd/man/man2/i386_get_ldt.2                   |  143 +
 bsd/man/man2/minherit.2                       |    2 +-
 bsd/miscfs/devfs/devfs_tree.c                 |    6 +-
 bsd/miscfs/devfs/devfs_vfsops.c               |   18 +-
 bsd/miscfs/devfs/devfs_vnops.c                |    1 -
 bsd/miscfs/specfs/spec_vnops.c                |   12 -
 bsd/miscfs/specfs/specdev.h                   |    1 -
 bsd/miscfs/synthfs/synthfs_vfsops.c           |   12 +-
 bsd/miscfs/synthfs/synthfs_vnops.c            |    1 -
 bsd/miscfs/union/union.h                      |    9 +-
 bsd/miscfs/union/union_subr.c                 |   14 +-
 bsd/miscfs/union/union_vfsops.c               |   10 +-
 bsd/miscfs/volfs/volfs_vnops.c                |    1 -
 bsd/net/bpf.c                                 |    2 -
 bsd/net/bpf.h                                 |   10 +-
 bsd/net/dlil.c                                |    6 +-
 bsd/net/ether_at_pr_module.c                  |    2 +-
 bsd/net/if_bond.c                             |    3 -
 bsd/net/if_var.h                              |    2 +-
 bsd/net/if_vlan.c                             |    2 -
 bsd/net/kext_net.h                            |    8 +-
 bsd/net/kpi_interface.c                       |    5 +-
 bsd/net/kpi_protocol.c                        |    2 -
 bsd/net/ndrv.c                                |   43 +-
 bsd/net/raw_usrreq.c                          |    4 -
 bsd/net/route.c                               |    4 -
 bsd/net/zlib.c                                |   12 +-
 bsd/netat/adsp.c                              |   19 +-
 bsd/netat/adsp_CLDeny.c                       |    4 +-
 bsd/netat/adsp_CLListen.c                     |    9 +-
 bsd/netat/adsp_Close.c                        |   34 +-
 bsd/netat/adsp_Control.c                      |   33 +-
 bsd/netat/adsp_Init.c                         |    4 -
 bsd/netat/adsp_Open.c                         |    6 +-
 bsd/netat/adsp_Packet.c                       |   31 +-
 bsd/netat/adsp_Read.c                         |   13 -
 bsd/netat/adsp_RxAttn.c                       |    6 +-
 bsd/netat/adsp_RxData.c                       |   15 +-
 bsd/netat/adsp_Status.c                       |    3 -
 bsd/netat/adsp_Timer.c                        |    9 -
 bsd/netat/adsp_TimerElem.c                    |   63 +-
 bsd/netat/adsp_Write.c                        |    7 -
 bsd/netat/adsp_attention.c                    |   14 +-
 bsd/netat/adsp_internal.h                     |    3 -
 bsd/netat/adsp_misc.c                         |    4 -
 bsd/netat/adsp_reset.c                        |   13 +-
 bsd/netat/adsp_stream.c                       |   48 +-
 bsd/netat/appletalk.h                         |   19 +-
 bsd/netat/asp_proto.c                         |  144 +-
 bsd/netat/at.c                                |   20 +-
 bsd/netat/at_aarp.h                           |   11 +-
 bsd/netat/at_pcb.c                            |    1 -
 bsd/netat/at_proto.c                          |   47 +-
 bsd/netat/at_var.h                            |    4 +-
 bsd/netat/atp.h                               |   11 +-
 bsd/netat/atp_alloc.c                         |   17 -
 bsd/netat/atp_misc.c                          |   37 +-
 bsd/netat/atp_open.c                          |   20 +-
 bsd/netat/atp_read.c                          |   47 +-
 bsd/netat/atp_write.c                         |  139 +-
 bsd/netat/aurp.h                              |    9 +-
 bsd/netat/aurp_aurpd.c                        |   15 +-
 bsd/netat/aurp_cfg.c                          |   10 +-
 bsd/netat/aurp_gdata.c                        |    6 +-
 bsd/netat/aurp_misc.c                         |    5 +
 bsd/netat/aurp_open.c                         |    5 +
 bsd/netat/aurp_rd.c                           |    5 +
 bsd/netat/aurp_ri.c                           |   41 +-
 bsd/netat/aurp_rx.c                           |    4 +
 bsd/netat/aurp_tickle.c                       |    5 +
 bsd/netat/aurp_tx.c                           |    5 +
 bsd/netat/aurp_zi.c                           |    5 +
 bsd/netat/ddp.c                               |   27 +-
 bsd/netat/ddp.h                               |   25 +-
 bsd/netat/ddp_aarp.c                          |   59 +-
 bsd/netat/ddp_lap.c                           |   29 +-
 bsd/netat/ddp_nbp.c                           |   40 +-
 bsd/netat/ddp_r_rtmp.c                        |   21 +-
 bsd/netat/ddp_r_zip.c                         |   36 +-
 bsd/netat/ddp_rtmptable.c                     |   29 +-
 bsd/netat/ddp_sip.c                           |   14 +-
 bsd/netat/ddp_usrreq.c                        |   30 +-
 bsd/netat/drv_dep.c                           |   36 +-
 bsd/netat/nbp.h                               |   12 +-
 bsd/netat/sys_dep.c                           |   41 +-
 bsd/netat/sys_glue.c                          |   60 +-
 bsd/netat/sysglue.h                           |   14 +-
 bsd/netinet/icmp_var.h                        |    2 +-
 bsd/netinet/if_ether.h                        |    2 +-
 bsd/netinet/in.c                              |    2 +-
 bsd/netinet/in.h                              |    2 +-
 bsd/netinet/in_arp.c                          |    4 +-
 bsd/netinet/ip.h                              |    2 +-
 bsd/netinet/ip_divert.c                       |   23 +-
 bsd/netinet/ip_dummynet.c                     |   91 +-
 bsd/netinet/ip_fw.h                           |    2 +-
 bsd/netinet/ip_fw2.c                          |    7 +-
 bsd/netinet/ip_input.c                        |    3 -
 bsd/netinet/ip_mroute.h                       |    2 +-
 bsd/netinet/ip_output.c                       |   30 +
 bsd/netinet/ip_var.h                          |    4 +-
 bsd/netinet/kpi_ipfilter.c                    |    7 +-
 bsd/netinet/raw_ip.c                          |   17 +-
 bsd/netinet/tcp.h                             |    3 +-
 bsd/netinet/tcp_input.c                       |    9 +-
 bsd/netinet/tcp_subr.c                        |   26 +-
 bsd/netinet/udp_usrreq.c                      |   34 +-
 bsd/netinet/udp_var.h                         |    4 +-
 bsd/netinet6/ah.h                             |    2 +-
 bsd/netinet6/esp.h                            |    4 +-
 bsd/netinet6/in6.h                            |   15 +-
 bsd/netinet6/in6_var.h                        |    4 +-
 bsd/netinet6/ip6_input.c                      |    2 -
 bsd/netinet6/ipcomp.h                         |    6 +-
 bsd/netinet6/ipsec.h                          |    2 +-
 bsd/netinet6/pim6_var.h                       |    1 -
 bsd/netkey/key.c                              |    1 -
 bsd/nfs/nfs.h                                 |   39 +-
 bsd/nfs/nfs_bio.c                             |   93 +-
 bsd/nfs/nfs_node.c                            |    8 +-
 bsd/nfs/nfs_serv.c                            |   38 +-
 bsd/nfs/nfs_socket.c                          |    5 +-
 bsd/nfs/nfs_srvcache.c                        |    1 -
 bsd/nfs/nfs_subs.c                            |   64 +-
 bsd/nfs/nfs_syscalls.c                        |   77 +-
 bsd/nfs/nfs_vfsops.c                          |   35 +-
 bsd/nfs/nfs_vnops.c                           |   47 +-
 bsd/nfs/nfsmount.h                            |    1 +
 bsd/nfs/nfsnode.h                             |    2 +
 bsd/ppc/_types.h                              |    2 +-
 bsd/ppc/types.h                               |    3 +
 bsd/sys/Makefile                              |    5 +-
 bsd/sys/_endian.h                             |   31 +-
 bsd/sys/aio.h                                 |   12 +-
 bsd/sys/attr.h                                |   12 +-
 bsd/sys/bsdtask_info.h                        |  114 +
 bsd/sys/buf_internal.h                        |    3 +-
 bsd/sys/cdefs.h                               |   24 +-
 bsd/sys/dirent.h                              |    8 +-
 bsd/sys/disk.h                                |   98 +-
 bsd/sys/domain.h                              |    8 +-
 bsd/sys/event.h                               |    8 +-
 bsd/sys/fcntl.h                               |   20 +-
 bsd/sys/file.h                                |    8 +-
 bsd/sys/file_internal.h                       |    7 +
 bsd/sys/imgact.h                              |    6 +-
 bsd/sys/ipcs.h                                |    8 -
 bsd/sys/kauth.h                               |   22 +-
 bsd/sys/kdebug.h                              |  117 +-
 bsd/sys/kernel_types.h                        |    2 +-
 bsd/sys/kpi_socket.h                          |    1 +
 bsd/sys/ktrace.h                              |    6 +-
 bsd/sys/lock.h                                |   15 -
 bsd/sys/lockf.h                               |    8 +-
 bsd/sys/mbuf.h                                |    1 +
 bsd/sys/mount.h                               |   28 +-
 bsd/sys/mount_internal.h                      |   14 +-
 bsd/sys/namei.h                               |    1 -
 bsd/sys/proc.h                                |    3 +-
 bsd/sys/proc_info.h                           |  574 ++++
 bsd/sys/proc_internal.h                       |   23 +-
 bsd/sys/protosw.h                             |    8 +-
 bsd/sys/quota.h                               |   12 +-
 bsd/sys/reboot.h                              |    1 +
 bsd/sys/resource.h                            |   10 +-
 bsd/sys/shm_internal.h                        |   12 +-
 bsd/sys/signal.h                              |   34 +-
 bsd/sys/socket.h                              |   11 +-
 bsd/sys/socketvar.h                           |    9 +-
 bsd/sys/stat.h                                |   11 +-
 bsd/sys/sys_domain.h                          |   58 +-
 bsd/sys/syscall.h                             |   30 +-
 bsd/sys/sysctl.h                              |   70 +-
 bsd/sys/syslimits.h                           |    2 +-
 bsd/sys/sysproto.h                            |   98 +-
 bsd/sys/termios.h                             |   10 +-
 bsd/sys/time.h                                |   40 +-
 bsd/sys/ucontext.h                            |    8 -
 bsd/sys/ucred.h                               |    2 +
 bsd/sys/user.h                                |    9 +-
 bsd/sys/vm.h                                  |   12 +-
 bsd/sys/vnioctl.h                             |    8 -
 bsd/sys/vnode_if.h                            |   13 -
 bsd/sys/vnode_internal.h                      |    3 +-
 bsd/ufs/ffs/ffs_alloc.c                       |   10 +-
 bsd/ufs/ffs/ffs_balloc.c                      |   14 +-
 bsd/ufs/ffs/ffs_inode.c                       |    6 +-
 bsd/ufs/ffs/ffs_subr.c                        |    1 -
 bsd/ufs/ffs/ffs_vfsops.c                      |   29 +-
 bsd/ufs/ffs/ffs_vnops.c                       |    2 -
 bsd/ufs/ufs/ufs_attrlist.c                    |    1 -
 bsd/ufs/ufs/ufs_bmap.c                        |    8 +-
 bsd/ufs/ufs/ufs_byte_order.c                  |   88 +-
 bsd/ufs/ufs/ufs_lookup.c                      |    7 +-
 bsd/ufs/ufs/ufs_quota.c                       |   22 +-
 bsd/ufs/ufs/ufs_vnops.c                       |    1 -
 bsd/uxkern/ux_exception.c                     |    8 +-
 bsd/vfs/kpi_vfs.c                             |  223 +-
 bsd/vfs/vfs_bio.c                             |   71 +-
 bsd/vfs/vfs_cache.c                           |  213 +-
 bsd/vfs/vfs_cluster.c                         |    6 +-
 bsd/vfs/vfs_fsevents.c                        |   69 +-
 bsd/vfs/vfs_init.c                            |   13 +-
 bsd/vfs/vfs_journal.c                         |    3 -
 bsd/vfs/vfs_lookup.c                          |   36 +-
 bsd/vfs/vfs_quota.c                           |  113 +-
 bsd/vfs/vfs_subr.c                            |   60 +-
 bsd/vfs/vfs_support.c                         |   21 -
 bsd/vfs/vfs_support.h                         |    5 -
 bsd/vfs/vfs_syscalls.c                        |  118 +-
 bsd/vfs/vfs_utfconv.c                         |   18 +-
 bsd/vfs/vfs_vnops.c                           |   64 +-
 bsd/vfs/vfs_xattr.c                           |   10 +-
 bsd/vfs/vnode_if.c                            |   17 -
 bsd/vm/vm_unix.c                              |  574 +++-
 bsd/vm/vnode_pager.c                          |   26 +
 bsd/vm/vnode_pager.h                          |    9 +
 config/BSDKernel.exports                      |    3 +-
 config/IOKit.exports                          |  138 +-
 config/IOKit.ppc.exports                      |   39 +
 config/Libkern.exports                        |   18 -
 config/Libkern.ppc.exports                    |   19 +
 config/MasterVersion                          |    2 +-
 config/System6.0.exports                      |   87 +-
 config/System6.0.i386.exports                 |  319 +-
 config/System6.0.ppc.exports                  |   39 +-
 config/Unsupported.exports                    |   30 +-
 config/Unsupported.i386.exports               |   28 +-
 config/Unsupported.ppc.exports                |    2 -
 .../Drivers/platform/drvAppleNMI/AppleNMI.cpp |   16 +-
 iokit/IOKit/IOBufferMemoryDescriptor.h        |   38 +-
 iokit/IOKit/IOCommandGate.h                   |   35 +-
 iokit/IOKit/IODMACommand.h                    |  474 +++
 iokit/IOKit/IOHibernatePrivate.h              |   51 +-
 iokit/IOKit/IOKitKeys.h                       |   24 +-
 iokit/IOKit/IOKitKeysPrivate.h                |    5 +
 iokit/IOKit/IOLib.h                           |    4 +-
 iokit/IOKit/IOMapper.h                        |   20 +-
 iokit/IOKit/IOMemoryCursor.h                  |   24 +-
 iokit/IOKit/IOMemoryDescriptor.h              |  117 +-
 iokit/IOKit/IOMultiMemoryDescriptor.h         |    5 +
 iokit/IOKit/IOPlatformExpert.h                |    6 +-
 iokit/IOKit/IOReturn.h                        |    8 +
 iokit/IOKit/IOService.h                       |   16 +-
 iokit/IOKit/IOTimeStamp.h                     |   90 +-
 iokit/IOKit/IOTypes.h                         |   24 +-
 iokit/IOKit/IOUserClient.h                    |    9 +-
 iokit/IOKit/IOWorkLoop.h                      |   42 +-
 iokit/IOKit/assert.h                          |    7 +
 iokit/IOKit/nvram/IONVRAMController.h         |    5 +
 iokit/IOKit/pwr_mgt/IOPM.h                    |  555 +--
 iokit/IOKit/pwr_mgt/IOPMDeprecated.h          |  177 +
 iokit/IOKit/pwr_mgt/IOPMPowerSource.h         |  257 +-
 iokit/IOKit/pwr_mgt/IOPMPowerSourceList.h     |   42 +-
 iokit/IOKit/pwr_mgt/IOPMinformeeList.h        |   30 +-
 iokit/IOKit/pwr_mgt/Makefile                  |    2 +-
 iokit/IOKit/pwr_mgt/RootDomain.h              |  178 +-
 iokit/Kernel/IOBufferMemoryDescriptor.cpp     |  267 +-
 iokit/Kernel/IOCPU.cpp                        |   15 +-
 iokit/Kernel/IOCatalogue.cpp                  |   42 +-
 iokit/Kernel/IOCommandGate.cpp                |  122 +-
 iokit/Kernel/IOCommandPool.cpp                |   14 +-
 iokit/Kernel/IOCopyMapper.cpp                 |  426 +++
 iokit/Kernel/IOCopyMapper.h                   |   89 +
 iokit/Kernel/IODMACommand.cpp                 |  970 ++++++
 iokit/Kernel/IODeviceTreeSupport.cpp          |    4 +
 iokit/Kernel/IOHibernateIO.cpp                |  211 +-
 iokit/Kernel/IOHibernateInternal.h            |    3 +
 iokit/Kernel/IOHibernateRestoreKernel.c       |  233 +-
 iokit/Kernel/IOInterruptController.cpp        |    5 +-
 iokit/Kernel/IOKitKernelInternal.h            |   41 +-
 iokit/Kernel/{IOLib.c => IOLib.cpp}           |  231 +-
 iokit/Kernel/IOMapper.cpp                     |  130 +-
 iokit/Kernel/IOMemoryDescriptor.cpp           |  823 +++--
 iokit/Kernel/IOMultiMemoryDescriptor.cpp      |   32 +-
 iokit/Kernel/IOPMPowerSource.cpp              |  419 ++-
 iokit/Kernel/IOPMPowerSourceList.cpp          |   83 +-
 iokit/Kernel/IOPMPowerStateQueue.cpp          |   61 +-
 iokit/Kernel/IOPMPowerStateQueue.h            |    3 +
 iokit/Kernel/IOPMWorkArbiter.cpp              |  159 +
 iokit/Kernel/IOPMWorkArbiter.h                |   83 +
 iokit/Kernel/IOPMinformeeList.cpp             |  132 +-
 iokit/Kernel/IOPMrootDomain.cpp               | 1216 +++++--
 iokit/Kernel/IOPlatformExpert.cpp             |   60 +-
 iokit/Kernel/IORegistryEntry.cpp              |  183 +-
 iokit/Kernel/IOService.cpp                    |  114 +-
 iokit/Kernel/IOServicePM.cpp                  |  233 +-
 iokit/Kernel/IOStartIOKit.cpp                 |    8 +-
 iokit/Kernel/IOStringFuncs.c                  |    9 +-
 iokit/Kernel/IOUserClient.cpp                 |  118 +-
 iokit/Kernel/IOWorkLoop.cpp                   |   88 +-
 iokit/Kernel/RootDomainUserClient.cpp         |   15 +-
 iokit/Kernel/RootDomainUserClient.h           |    3 +-
 iokit/KernelConfigTables.cpp                  |    8 -
 iokit/bsddev/DINetBootHook.cpp                |    8 +-
 iokit/bsddev/DINetBootHook.h                  |    8 +-
 iokit/bsddev/IOKitBSDInit.cpp                 |   20 +-
 iokit/conf/Makefile.i386                      |   19 +-
 iokit/conf/files                              |   10 +-
 iokit/conf/files.i386                         |    3 -
 iokit/include/architecture/i386/pio.h         |  252 --
 kgmacros                                      |  688 +++-
 libkern/Makefile                              |    6 +-
 libkern/c++/OSCollectionIterator.cpp          |    2 +-
 libkern/c++/OSData.cpp                        |    6 +-
 libkern/c++/OSDictionary.cpp                  |    4 +-
 libkern/c++/OSMetaClass.cpp                   |   55 +-
 libkern/c++/OSObject.cpp                      |    5 +-
 libkern/c++/OSOrderedSet.cpp                  |    4 +-
 libkern/c++/OSString.cpp                      |    2 +-
 libkern/c++/OSSymbol.cpp                      |   14 +-
 libkern/c++/OSUnserializeXML.cpp              |    6 +-
 libkern/c++/OSUnserializeXML.y                |    6 +-
 libkern/conf/Makefile.i386                    |    3 +
 libkern/gen/OSAtomicOperations.c              |    5 +-
 libkern/gen/OSDebug.cpp                       |  131 +-
 libkern/libkern/OSAtomic.h                    |    2 +
 libkern/libkern/OSAtomic.h.save               |    2 +
 libkern/libkern/OSByteOrder.h                 |  456 +--
 libkern/libkern/OSCrossEndian.h               |   27 +-
 libkern/libkern/c++/Makefile                  |    1 +
 libkern/libkern/c++/OSEndianTypes.h           |  155 +
 libkern/libkern/c++/OSMetaClass.h             |    4 +-
 libkern/libkern/c++/OSObject.h                |    3 +
 libkern/libkern/i386/OSByteOrder.h            |   81 +-
 libkern/libkern/machine/OSByteOrder.h         |  165 +-
 libkern/libkern/ppc/OSByteOrder.h             |  135 +-
 libkern/mach-o/loader.h                       |   10 +-
 libsa/catalogue.cpp                           |   56 +-
 libsa/kext.cpp                                |  173 +-
 libsa/kld_patch.c                             |   25 +-
 makedefs/MakeInc.cmd                          |    2 +-
 makedefs/MakeInc.def                          |   71 +-
 makedefs/MakeInc.dir                          |   22 +-
 makedefs/MakeInc.rule                         |    2 +-
 osfmk/chud/chud_cpu.c                         |  134 +
 osfmk/{ppc => }/chud/chud_glue.c              |    0
 osfmk/{ppc => }/chud/chud_memory.c            |   19 +-
 osfmk/chud/chud_osfmk_callback.c              |  139 +
 osfmk/chud/chud_thread.c                      |  448 +++
 osfmk/chud/chud_xnu.h                         |  263 ++
 .../chud_xnu_glue.h}                          |   29 +-
 osfmk/chud/chud_xnu_private.h                 |   44 +
 .../i386/chud_cpu_asm.h}                      |   14 +-
 osfmk/chud/i386/chud_cpu_asm.s                |   32 +
 osfmk/chud/i386/chud_cpu_i386.c               |  166 +
 osfmk/chud/i386/chud_osfmk_callback_i386.c    |  596 ++++
 osfmk/chud/i386/chud_thread_i386.c            |  657 ++++
 osfmk/{ppc/chud => chud/i386}/chud_xnu_glue.h |    0
 osfmk/chud/i386/chud_xnu_private.h            |   60 +
 osfmk/{ppc/chud => chud/ppc}/chud_cpu_asm.h   |    0
 osfmk/{ppc/chud => chud/ppc}/chud_cpu_asm.s   |    2 +-
 .../chud_cpu.c => chud/ppc/chud_cpu_ppc.c}    |   83 +-
 .../ppc/chud_osfmk_callback_ppc.c}            |  187 +-
 osfmk/{ppc/chud => chud/ppc}/chud_spr.h       |    0
 .../ppc/chud_thread_ppc.c}                    |  390 +--
 osfmk/chud/ppc/chud_xnu_glue.h                |   28 +
 .../{ppc/chud => chud/ppc}/chud_xnu_private.h |   35 +-
 osfmk/conf/MASTER.i386                        |    9 +-
 osfmk/conf/Makefile                           |  116 +-
 osfmk/conf/Makefile.i386                      |   81 +-
 osfmk/conf/files                              |   15 +
 osfmk/conf/files.i386                         |   30 +-
 osfmk/conf/files.ppc                          |   11 +-
 osfmk/console/i386/serial_console.c           |  216 +-
 osfmk/console/i386/text_console.c             |    6 +-
 osfmk/console/i386/text_console.h             |    2 -
 osfmk/console/panic_dialog.c                  |   13 +-
 osfmk/console/ppc/serial_console.c            |   71 +-
 osfmk/console/serial_general.c                |  133 +
 osfmk/console/serial_protos.h                 |   60 +
 osfmk/console/video_console.c                 |   69 +-
 osfmk/ddb/db_command.c                        |   25 +-
 osfmk/ddb/db_run.c                            |   19 +-
 osfmk/ddb/db_task_thread.c                    |    4 +-
 osfmk/ddb/db_trap.c                           |   10 +-
 osfmk/default_pager/default_pager.c           |    2 +
 osfmk/default_pager/default_pager_internal.h  |   12 +-
 osfmk/default_pager/dp_backing_store.c        |   28 +-
 osfmk/default_pager/dp_memory_object.c        |   24 +-
 osfmk/device/device.defs                      |   11 +
 osfmk/device/device_types.defs                |    2 +-
 osfmk/device/iokit_rpc.c                      |  121 +-
 osfmk/i386/AT386/bbclock.c                    |  277 --
 osfmk/i386/AT386/conf.c                       |    6 +-
 osfmk/i386/AT386/misc_protos.h                |    8 -
 osfmk/i386/AT386/model_dep.c                  |  788 ++++-
 osfmk/i386/AT386/rtc.h                        |  232 --
 osfmk/i386/Diagnostics.c                      |  166 +
 osfmk/i386/Diagnostics.h                      |  117 +
 osfmk/i386/Makefile                           |   11 +
 osfmk/i386/acpi.c                             |  111 +-
 osfmk/i386/acpi_wakeup.s                      |   84 +-
 osfmk/i386/asm.h                              |    3 +-
 osfmk/i386/asm64.h                            |   62 +
 osfmk/i386/ast.h                              |    7 -
 osfmk/i386/bsd_i386.c                         |  910 ++---
 osfmk/i386/commpage/atomic.s                  |  160 +-
 osfmk/i386/commpage/bcopy_scalar.s            |   20 +-
 osfmk/i386/commpage/bcopy_sse3.s              |  476 +++
 osfmk/i386/commpage/bcopy_sse4.s              |  805 +++++
 osfmk/i386/commpage/bcopy_sse4_64.s           |  797 +++++
 osfmk/i386/commpage/bzero_scalar.s            |   13 +-
 osfmk/i386/commpage/bzero_sse3.s              |  164 +
 osfmk/i386/commpage/bzero_sse3_64.s           |  164 +
 osfmk/i386/commpage/cacheflush.s              |   46 +-
 osfmk/i386/commpage/commpage.c                |  382 ++-
 osfmk/i386/commpage/commpage.h                |   28 +-
 osfmk/i386/commpage/commpage_asm.s            |  196 ++
 osfmk/i386/commpage/commpage_gettimeofday.s   |   99 +-
 .../commpage/commpage_mach_absolute_time.s    |  150 +-
 osfmk/i386/commpage/commpage_sigs.c           |  189 ++
 osfmk/i386/commpage/commpage_sigs.h           |    0
 osfmk/i386/commpage/commpage_sigs.s           |    0
 osfmk/i386/commpage/longcopy_sse4.s           |  221 ++
 osfmk/i386/commpage/longcopy_sse4_64.s        |  211 ++
 osfmk/i386/commpage/memset_pattern_sse3.s     |  185 +
 osfmk/i386/commpage/memset_pattern_sse3_64.s  |  187 ++
 osfmk/i386/commpage/pthreads.s                |   30 +-
 osfmk/i386/commpage/spinlocks.s               |  156 +-
 osfmk/i386/cpu.c                              |   23 +-
 osfmk/i386/cpu_capabilities.h                 |  197 +-
 osfmk/i386/cpu_data.h                         |   98 +-
 osfmk/i386/cpu_threads.c                      |  107 +-
 osfmk/i386/cpu_threads.h                      |    2 +-
 osfmk/i386/cpuid.c                            |  349 +-
 osfmk/i386/cpuid.h                            |  139 +-
 osfmk/i386/cswitch.s                          |   36 +-
 osfmk/i386/db_interface.c                     |  145 +-
 osfmk/i386/db_machdep.h                       |   44 +-
 osfmk/i386/db_trace.c                         |  337 +-
 osfmk/i386/eflags.h                           |    7 +-
 osfmk/i386/fpu.c                              |  762 ++---
 osfmk/i386/fpu.h                              |  111 +-
 osfmk/i386/gdt.c                              |  114 +-
 osfmk/i386/genassym.c                         |  293 +-
 osfmk/i386/hibernate_i386.c                   |  187 +-
 osfmk/i386/hibernate_restore.s                |  178 +-
 osfmk/i386/hpet.c                             |  434 +++
 osfmk/i386/hpet.h                             |  108 +
 osfmk/i386/i386_init.c                        |  160 +-
 osfmk/i386/i386_lock.s                        |  403 ++-
 .../i386/i386_lowmem.h                        |   22 +-
 osfmk/i386/i386_vm_init.c                     |  583 ++--
 osfmk/i386/idt.s                              |  502 ++-
 osfmk/i386/idt64.s                            | 1167 +++++++
 osfmk/i386/io_emulate.c                       |    2 +-
 osfmk/i386/io_emulate.h                       |    2 +-
 osfmk/i386/io_map.c                           |   14 +-
 osfmk/i386/io_map_entries.h                   |    5 +-
 osfmk/i386/iopb.c                             |   10 +-
 osfmk/i386/ktss.c                             |  102 +-
 osfmk/i386/ldt.c                              |   78 +-
 osfmk/i386/lock.h                             |   88 +-
 osfmk/i386/locks.h                            |   26 +-
 osfmk/i386/locks_i386.c                       |  436 +--
 osfmk/i386/locore.s                           | 2338 +++----------
 osfmk/i386/loose_ends.c                       | 1078 ++++--
 osfmk/i386/lowglobals.h                       |   68 +
 osfmk/i386/lowmem_vectors.s                   |   89 +
 osfmk/i386/machdep_call.c                     |   26 +-
 osfmk/i386/machdep_call.h                     |   25 +-
 osfmk/i386/machine_check.c                    |  268 ++
 osfmk/i386/machine_check.h                    |  196 ++
 osfmk/i386/machine_cpu.h                      |    9 +-
 osfmk/i386/machine_routines.c                 |  259 +-
 osfmk/i386/machine_routines.h                 |   42 +
 osfmk/i386/machine_routines_asm.s             |  166 +-
 osfmk/i386/misc_protos.h                      |   39 +-
 osfmk/i386/mp.c                               |  626 +++-
 osfmk/i386/mp.h                               |   63 +-
 osfmk/i386/mp_desc.c                          |  510 ++-
 osfmk/i386/mp_desc.h                          |   37 +-
 osfmk/i386/mp_events.h                        |    2 +
 osfmk/i386/mp_slave_boot.s                    |   59 +-
 osfmk/i386/pcb.c                              | 2210 ++++++++----
 osfmk/i386/perfmon.c                          |  115 +-
 osfmk/i386/perfmon.h                          |    5 +-
 osfmk/i386/phys.c                             |   89 +-
 osfmk/i386/pio.h                              |   50 +-
 osfmk/i386/pmCPU.c                            |  475 +++
 osfmk/i386/pmCPU.h                            |  147 +
 osfmk/i386/pmap.c                             | 2982 +++++++++++------
 osfmk/i386/pmap.h                             |  607 ++--
 osfmk/i386/postcode.h                         |   78 +-
 osfmk/i386/proc_reg.h                         |   50 +-
 osfmk/i386/rtclock.c                          | 1417 ++------
 .../IntelClock.cpp => osfmk/i386/rtclock.h    |   39 +-
 osfmk/i386/seg.h                              |  154 +-
 osfmk/i386/simple_lock.h                      |    2 +
 osfmk/i386/start.s                            |  372 +-
 osfmk/i386/start64.s                          |  182 +
 osfmk/i386/startup64.c                        |  348 ++
 osfmk/i386/task.h                             |    5 +-
 osfmk/i386/thread.h                           |  129 +-
 osfmk/i386/trap.c                             | 1601 ++++-----
 osfmk/i386/trap.h                             |   47 +-
 osfmk/i386/tsc.c                              |  221 ++
 osfmk/i386/tsc.h                              |   72 +
 osfmk/i386/tss.h                              |   88 +-
 osfmk/i386/user_ldt.c                         |  570 ++--
 osfmk/i386/user_ldt.h                         |   35 +-
 osfmk/ipc/ipc_object.c                        |    7 +-
 osfmk/ipc/ipc_port.c                          |   21 +-
 osfmk/ipc/mach_msg.c                          |    8 +-
 osfmk/kdp/kdp.c                               |  154 +-
 osfmk/kdp/kdp_core.h                          |    6 +-
 osfmk/kdp/kdp_internal.h                      |    3 +-
 osfmk/kdp/kdp_udp.c                           | 1144 ++++---
 osfmk/kdp/ml/i386/kdp_machdep.c               |  213 +-
 osfmk/kdp/ml/i386/kdp_vm.c                    |  492 ++-
 osfmk/kdp/ml/ppc/kdp_machdep.c                |  169 +
 osfmk/kdp/ml/ppc/kdp_vm.c                     |   83 +-
 osfmk/kern/Makefile                           |    3 +
 osfmk/kern/ast.c                              |   11 +-
 osfmk/kern/ast.h                              |    5 +
 osfmk/kern/bsd_kern.c                         |  160 +
 osfmk/kern/clock.c                            | 1164 +++----
 osfmk/kern/clock.h                            |   95 +-
 osfmk/kern/clock_oldops.c                     |  808 +++++
 osfmk/kern/debug.h                            |    2 +-
 osfmk/kern/etimer.c                           |  200 ++
 osfmk/kern/etimer.h                           |   74 +
 osfmk/kern/hibernate.c                        |   25 +-
 osfmk/kern/ipc_kobject.h                      |   13 +-
 osfmk/kern/ipc_tt.c                           |  117 +-
 osfmk/kern/ipc_tt.h                           |    8 +
 osfmk/kern/kalloc.c                           |   63 +-
 osfmk/kern/locks.c                            |   13 +-
 osfmk/kern/locks.h                            |   11 +-
 osfmk/kern/machine.c                          |    8 +-
 osfmk/kern/misc_protos.h                      |    6 +-
 osfmk/kern/page_decrypt.c                     |   80 +
 osfmk/kern/page_decrypt.h                     |   38 +
 osfmk/{ppc => kern}/pms.c                     |  258 +-
 osfmk/kern/pms.h                              |  197 ++
 osfmk/kern/printf.c                           |    9 +-
 osfmk/kern/sched_prim.c                       |   25 +-
 osfmk/kern/stack.c                            |    2 +-
 osfmk/kern/startup.c                          |   32 +-
 osfmk/kern/syscall_sw.c                       |    6 +-
 osfmk/kern/task.c                             |   69 +-
 osfmk/kern/task.h                             |    8 +
 osfmk/kern/thread.c                           |    4 -
 osfmk/kern/thread.h                           |   10 +
 osfmk/kern/timer_call.c                       |   10 +-
 osfmk/kern/wait_queue.c                       |    9 +-
 osfmk/kern/wait_queue.h                       |   23 +-
 osfmk/kern/zalloc.c                           |   25 +-
 osfmk/mach-o/loader.h                         |    2 +-
 osfmk/mach/host_info.h                        |    8 +-
 osfmk/mach/host_reboot.h                      |    3 +-
 osfmk/mach/i386/boolean.h                     |    4 +
 osfmk/mach/i386/exception.h                   |    1 +
 osfmk/mach/i386/fp_reg.h                      |   32 +-
 osfmk/mach/i386/machine_types.defs            |   22 +-
 osfmk/mach/i386/rpc.h                         |    2 +-
 osfmk/mach/i386/syscall_sw.h                  |   95 +-
 osfmk/mach/i386/thread_state.h                |    5 +-
 osfmk/mach/i386/thread_status.h               |  742 ++--
 osfmk/mach/i386/vm_param.h                    |   52 +-
 osfmk/mach/i386/vm_types.h                    |   26 +-
 osfmk/mach/kmod.h                             |   16 +-
 osfmk/mach/mach_traps.h                       |   31 +-
 osfmk/mach/mach_types.defs                    |    8 +
 osfmk/mach/mach_types.h                       |    4 +-
 osfmk/mach/machine.h                          |   18 +-
 osfmk/mach/machine/asm.h                      |    2 +-
 osfmk/mach/machine/boolean.h                  |    2 +-
 osfmk/mach/machine/exception.h                |    2 +-
 osfmk/mach/machine/kern_return.h              |    2 +-
 osfmk/mach/machine/machine_types.defs         |    2 +-
 osfmk/mach/machine/ndr_def.h                  |    2 +-
 osfmk/mach/machine/processor_info.h           |    2 +-
 osfmk/mach/machine/rpc.h                      |    2 +-
 osfmk/mach/machine/syscall_sw.h               |    2 +-
 osfmk/mach/machine/thread_state.h             |    2 +-
 osfmk/mach/machine/thread_status.h            |    2 +-
 osfmk/mach/machine/vm_param.h                 |    2 +-
 osfmk/mach/machine/vm_types.h                 |    2 +-
 osfmk/mach/memory_object_types.h              |   66 +-
 osfmk/mach/mig.h                              |    4 +-
 osfmk/mach/port.h                             |    6 +-
 osfmk/mach/ppc/rpc.h                          |    2 +-
 osfmk/mach/ppc/syscall_sw.h                   |    2 +-
 osfmk/mach/syscall_sw.h                       |    1 +
 osfmk/mach/task.defs                          |    2 +-
 osfmk/mach/task_info.h                        |    8 +-
 osfmk/mach/task_special_ports.h               |    2 +
 osfmk/mach/thread_status.h                    |    3 +-
 osfmk/mach/vm_param.h                         |    3 +
 osfmk/mach/vm_prot.h                          |    7 +
 osfmk/mach/vm_region.h                        |    8 +-
 osfmk/mach_debug/vm_info.h                    |    8 +-
 osfmk/machine/cpu_capabilities.h              |    2 +-
 osfmk/ppc/Diagnostics.h                       |    1 +
 osfmk/ppc/Makefile                            |    2 +-
 osfmk/ppc/PPCcalls.h                          |    2 +-
 osfmk/ppc/ast.h                               |    7 +-
 osfmk/ppc/chud/chud_xnu.h                     |  268 --
 osfmk/ppc/commpage/commpage.h                 |    2 +-
 osfmk/ppc/commpage/commpage_asm.s             |   12 +-
 osfmk/ppc/commpage/gettimeofday.s             |   86 +-
 osfmk/ppc/conf.c                              |    4 +-
 osfmk/ppc/cpu.c                               |   23 +-
 osfmk/ppc/cpu_capabilities.h                  |   13 +-
 osfmk/ppc/db_interface.c                      |   12 -
 osfmk/ppc/db_machdep.h                        |    2 -
 osfmk/ppc/exception.h                         |    4 +-
 osfmk/ppc/hibernate_ppc.c                     |   12 +
 osfmk/ppc/hw_lock.s                           |   23 +-
 osfmk/ppc/hw_vm.s                             |   10 +-
 osfmk/ppc/interrupt.c                         |    4 +-
 osfmk/ppc/io_map.c                            |   20 +-
 osfmk/ppc/io_map_entries.h                    |    5 +-
 osfmk/ppc/locks.h                             |    4 +-
 osfmk/ppc/locks_ppc.c                         |   37 +-
 osfmk/ppc/lowglobals.h                        |    8 +-
 osfmk/ppc/lowmem_vectors.s                    |    9 +-
 osfmk/ppc/machine_cpu.h                       |    2 +
 osfmk/ppc/machine_routines.c                  |   12 +-
 osfmk/ppc/machine_routines.h                  |    5 +
 osfmk/ppc/machine_routines_asm.s              |   13 -
 osfmk/ppc/mappings.c                          |   94 +-
 osfmk/ppc/mappings.h                          |    2 +-
 osfmk/ppc/pmap.c                              |   61 +-
 osfmk/ppc/pmap.h                              |   12 +-
 osfmk/ppc/pms.h                               |  140 +-
 osfmk/ppc/pmsCPU.c                            |   79 +-
 osfmk/ppc/ppc_init.c                          |    2 +-
 osfmk/ppc/ppc_vm_init.c                       |   18 +-
 osfmk/ppc/rtclock.c                           |  878 +----
 osfmk/ppc/rtclock.h                           |    3 +-
 osfmk/ppc/serial_io.c                         |   63 +-
 osfmk/ppc/serial_io.h                         |   17 +-
 osfmk/ppc/skiplists.s                         |    6 +-
 osfmk/ppc/status.c                            |   13 +
 osfmk/ppc/trap.c                              |    9 +-
 osfmk/ppc/vmachmon.c                          |    6 +-
 osfmk/vm/bsd_vm.c                             |  261 +-
 osfmk/vm/device_vm.c                          |   35 +-
 osfmk/vm/memory_object.c                      |  417 +--
 osfmk/vm/memory_object.h                      |    6 +-
 osfmk/vm/pmap.h                               |    7 +-
 osfmk/vm/task_working_set.c                   |    4 +
 osfmk/vm/vm_apple_protect.c                   | 1077 ++++++
 osfmk/vm/vm_fault.c                           |  122 +-
 osfmk/vm/vm_init.c                            |   13 +-
 osfmk/vm/vm_kern.c                            |   50 +-
 osfmk/vm/vm_kern.h                            |    1 +
 osfmk/vm/vm_map.c                             |  587 +++-
 osfmk/vm/vm_map.h                             |   26 +
 osfmk/vm/vm_object.c                          |  274 +-
 osfmk/vm/vm_object.h                          |   49 +-
 osfmk/vm/vm_page.h                            |   14 +
 osfmk/vm/vm_pageout.c                         |  214 +-
 osfmk/vm/vm_pageout.h                         |    4 +
 osfmk/vm/vm_protos.h                          |   42 +-
 osfmk/vm/vm_resident.c                        |  274 +-
 osfmk/vm/vm_shared_memory_server.c            |  524 ++-
 osfmk/vm/vm_shared_memory_server.h            |   38 +-
 osfmk/vm/vm_user.c                            |  107 +-
 pexpert/conf/Makefile.i386                    |    1 +
 pexpert/gen/bootargs.c                        |   44 +-
 pexpert/i386/kd.c                             |   31 -
 pexpert/i386/pe_bootargs.c                    |    2 +-
 pexpert/i386/pe_identify_machine.c            |    3 -
 pexpert/i386/pe_init.c                        |  193 +-
 pexpert/i386/pe_interrupt.c                   |   36 +-
 pexpert/i386/pe_kprintf.c                     |    7 +
 pexpert/i386/pe_serial.c                      |   55 +-
 pexpert/pexpert/i386/Makefile                 |    1 +
 pexpert/pexpert/i386/boot.h                   |  298 +-
 pexpert/pexpert/i386/efi.h                    |  549 +++
 pexpert/pexpert/ppc/protos.h                  |    5 +
 pexpert/ppc/pe_identify_machine.c             |    3 +-
 pexpert/ppc/pe_init.c                         |    1 +
 pexpert/ppc/pe_kprintf.c                      |    3 +-
 820 files changed, 53732 insertions(+), 25838 deletions(-)
 delete mode 100644 EXTERNAL_HEADERS/architecture/byte_order.h
 delete mode 100644 EXTERNAL_HEADERS/architecture/i386/byte_order.h
 delete mode 100644 EXTERNAL_HEADERS/architecture/i386/fpu.h
 delete mode 100644 EXTERNAL_HEADERS/architecture/i386/frame.h
 create mode 100644 EXTERNAL_HEADERS/architecture/i386/pio.h
 delete mode 100644 EXTERNAL_HEADERS/architecture/ppc/byte_order.h
 create mode 100644 EXTERNAL_HEADERS/stdbool.h
 rename bsd/dev/{ppc => }/chud/chud_bsd_callback.c (57%)
 rename bsd/dev/{ppc => }/chud/chud_process.c (70%)
 create mode 100644 bsd/dev/i386/munge.s
 delete mode 100644 bsd/dev/i386/pio.h
 create mode 100644 bsd/dev/i386/systemcalls.c
 create mode 100644 bsd/kern/proc_info.c
 create mode 100644 bsd/kern/socket_info.c
 create mode 100644 bsd/man/man2/i386_get_ldt.2
 create mode 100644 bsd/sys/bsdtask_info.h
 create mode 100644 bsd/sys/proc_info.h
 create mode 100644 iokit/IOKit/IODMACommand.h
 create mode 100644 iokit/IOKit/pwr_mgt/IOPMDeprecated.h
 create mode 100644 iokit/Kernel/IOCopyMapper.cpp
 create mode 100644 iokit/Kernel/IOCopyMapper.h
 create mode 100644 iokit/Kernel/IODMACommand.cpp
 rename iokit/Kernel/{IOLib.c => IOLib.cpp} (81%)
 create mode 100644 iokit/Kernel/IOPMWorkArbiter.cpp
 create mode 100644 iokit/Kernel/IOPMWorkArbiter.h
 delete mode 100644 iokit/include/architecture/i386/pio.h
 create mode 100644 libkern/libkern/c++/OSEndianTypes.h
 create mode 100644 osfmk/chud/chud_cpu.c
 rename osfmk/{ppc => }/chud/chud_glue.c (100%)
 rename osfmk/{ppc => }/chud/chud_memory.c (79%)
 create mode 100644 osfmk/chud/chud_osfmk_callback.c
 create mode 100644 osfmk/chud/chud_thread.c
 create mode 100644 osfmk/chud/chud_xnu.h
 rename osfmk/{i386/rtclock_entries.h => chud/chud_xnu_glue.h} (66%)
 create mode 100644 osfmk/chud/chud_xnu_private.h
 rename osfmk/{i386/AT386/bbclock_entries.h => chud/i386/chud_cpu_asm.h} (80%)
 create mode 100644 osfmk/chud/i386/chud_cpu_asm.s
 create mode 100644 osfmk/chud/i386/chud_cpu_i386.c
 create mode 100644 osfmk/chud/i386/chud_osfmk_callback_i386.c
 create mode 100644 osfmk/chud/i386/chud_thread_i386.c
 rename osfmk/{ppc/chud => chud/i386}/chud_xnu_glue.h (100%)
 create mode 100644 osfmk/chud/i386/chud_xnu_private.h
 rename osfmk/{ppc/chud => chud/ppc}/chud_cpu_asm.h (100%)
 rename osfmk/{ppc/chud => chud/ppc}/chud_cpu_asm.s (99%)
 rename osfmk/{ppc/chud/chud_cpu.c => chud/ppc/chud_cpu_ppc.c} (96%)
 rename osfmk/{ppc/chud/chud_osfmk_callback.c => chud/ppc/chud_osfmk_callback_ppc.c} (72%)
 rename osfmk/{ppc/chud => chud/ppc}/chud_spr.h (100%)
 rename osfmk/{ppc/chud/chud_thread.c => chud/ppc/chud_thread_ppc.c} (75%)
 create mode 100644 osfmk/chud/ppc/chud_xnu_glue.h
 rename osfmk/{ppc/chud => chud/ppc}/chud_xnu_private.h (71%)
 create mode 100644 osfmk/console/serial_general.c
 create mode 100644 osfmk/console/serial_protos.h
 delete mode 100644 osfmk/i386/AT386/bbclock.c
 delete mode 100644 osfmk/i386/AT386/rtc.h
 create mode 100644 osfmk/i386/Diagnostics.c
 create mode 100644 osfmk/i386/Diagnostics.h
 create mode 100644 osfmk/i386/asm64.h
 create mode 100644 osfmk/i386/commpage/bcopy_sse3.s
 create mode 100644 osfmk/i386/commpage/bcopy_sse4.s
 create mode 100644 osfmk/i386/commpage/bcopy_sse4_64.s
 create mode 100644 osfmk/i386/commpage/bzero_sse3.s
 create mode 100644 osfmk/i386/commpage/bzero_sse3_64.s
 create mode 100644 osfmk/i386/commpage/commpage_asm.s
 create mode 100644 osfmk/i386/commpage/commpage_sigs.c
 delete mode 100644 osfmk/i386/commpage/commpage_sigs.h
 delete mode 100644 osfmk/i386/commpage/commpage_sigs.s
 create mode 100644 osfmk/i386/commpage/longcopy_sse4.s
 create mode 100644 osfmk/i386/commpage/longcopy_sse4_64.s
 create mode 100644 osfmk/i386/commpage/memset_pattern_sse3.s
 create mode 100644 osfmk/i386/commpage/memset_pattern_sse3_64.s
 create mode 100644 osfmk/i386/hpet.c
 create mode 100644 osfmk/i386/hpet.h
 rename iokit/Drivers/platform/drvAppleIntelClock/AppleIntelClock.h => osfmk/i386/i386_lowmem.h (79%)
 create mode 100644 osfmk/i386/idt64.s
 create mode 100644 osfmk/i386/lowglobals.h
 create mode 100644 osfmk/i386/lowmem_vectors.s
 create mode 100644 osfmk/i386/machine_check.c
 create mode 100644 osfmk/i386/machine_check.h
 create mode 100644 osfmk/i386/pmCPU.c
 create mode 100644 osfmk/i386/pmCPU.h
 rename iokit/Drivers/platform/drvAppleIntelClock/IntelClock.cpp => osfmk/i386/rtclock.h (71%)
 create mode 100644 osfmk/i386/start64.s
 create mode 100644 osfmk/i386/startup64.c
 create mode 100644 osfmk/i386/tsc.c
 create mode 100644 osfmk/i386/tsc.h
 create mode 100644 osfmk/kern/clock_oldops.c
 create mode 100644 osfmk/kern/etimer.c
 create mode 100644 osfmk/kern/etimer.h
 create mode 100644 osfmk/kern/page_decrypt.c
 create mode 100644 osfmk/kern/page_decrypt.h
 rename osfmk/{ppc => kern}/pms.c (82%)
 create mode 100644 osfmk/kern/pms.h
 delete mode 100644 osfmk/ppc/chud/chud_xnu.h
 create mode 100644 osfmk/vm/vm_apple_protect.c
 create mode 100644 pexpert/pexpert/i386/efi.h

diff --git a/EXTERNAL_HEADERS/Makefile b/EXTERNAL_HEADERS/Makefile
index 0e05710a4..fceef154e 100644
--- a/EXTERNAL_HEADERS/Makefile
+++ b/EXTERNAL_HEADERS/Makefile
@@ -23,6 +23,7 @@ INSTINC_SUBDIRS_I386 =	\
 EXPORT_FILES = \
 	ar.h		\
 	stdarg.h	\
+	stdbool.h	\
 	stdint.h
 
 INSTALL_MI_LIST = 
diff --git a/EXTERNAL_HEADERS/architecture/Makefile b/EXTERNAL_HEADERS/architecture/Makefile
index fd64ab197..41a4fa6d4 100644
--- a/EXTERNAL_HEADERS/architecture/Makefile
+++ b/EXTERNAL_HEADERS/architecture/Makefile
@@ -15,8 +15,7 @@ INSTINC_SUBDIRS_PPC = 	\
 INSTINC_SUBDIRS_I386 =	\
 	i386
 
-EXPORT_FILES = \
-	byte_order.h
+EXPORT_FILES = 
 
 INSTALL_MI_LIST = 
 
@@ -30,3 +29,6 @@ include $(MakeInc_rule)
 include $(MakeInc_dir)
 
 
+build_installhdrs_mi::
+	$(MKDIR) $(DSTROOT)/$(KINCDIR)/$(EXPORT_MI_DIR)
+	$(LN) ../../../../../../../../usr/include/architecture/byte_order.h $(DSTROOT)/$(KINCDIR)/$(EXPORT_MI_DIR)/byte_order.h 
diff --git a/EXTERNAL_HEADERS/architecture/byte_order.h b/EXTERNAL_HEADERS/architecture/byte_order.h
deleted file mode 100644
index 6b6a08ea3..000000000
--- a/EXTERNAL_HEADERS/architecture/byte_order.h
+++ /dev/null
@@ -1,531 +0,0 @@
-/*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the License
- * may not be used to create, or enable the creation or redistribution of,
- * unlawful or unlicensed copies of an Apple operating system, or to
- * circumvent, violate, or enable the circumvention or violation of, any
- * terms of an Apple operating system software license agreement.
- * 
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- * 
- * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
- */
-/*
- * Copyright (c) 1992 NeXT Computer, Inc.
- *
- * Byte ordering conversion.
- */
-
-#ifndef	_ARCHITECTURE_BYTE_ORDER_H_
-#define _ARCHITECTURE_BYTE_ORDER_H_
- 
-typedef unsigned long NXSwappedFloat;
-typedef unsigned long long NXSwappedDouble;
-
-#if defined (__ppc__) || defined(__ppc64__)
-#include "architecture/ppc/byte_order.h"
-#elif defined (__i386__)
-#include "architecture/i386/byte_order.h"
-#else
-#error architecture not supported
-#endif
-
-/*
- * Identify the byte order
- * of the current host.
- */
-
-enum NXByteOrder {
-    NX_UnknownByteOrder,
-    NX_LittleEndian,
-    NX_BigEndian
-};
-
-static __inline__
-enum NXByteOrder
-NXHostByteOrder(void)
-{
-    unsigned int	_x;
-    
-    _x = (NX_BigEndian << 24) | NX_LittleEndian;
-        
-    return ((enum NXByteOrder)*((unsigned char *)&_x));
-}
-
-/*
- * The predicated versions
- * are defined here in terms
- * of the unpredicated ones.
- */
-
-#if	__BIG_ENDIAN__
-
-static __inline__
-unsigned short
-NXSwapBigShortToHost(
-    unsigned short	x
-)
-{
-    return (x);
-}
-
-static __inline__
-unsigned int
-NXSwapBigIntToHost(
-    unsigned int	x
-)
-{
-    return (x);
-}
-
-static __inline__
-unsigned long
-NXSwapBigLongToHost(
-    unsigned long	x
-)
-{
-    return (x);
-}
-
-static __inline__
-unsigned long long
-NXSwapBigLongLongToHost(
-    unsigned long long	x
-)
-{
-    return (x);
-}
-
-#ifndef KERNEL
-
-static __inline__
-double
-NXSwapBigDoubleToHost(
-    NXSwappedDouble	x
-)
-{
-    return NXConvertSwappedDoubleToHost(x);
-}
-
-static __inline__
-float
-NXSwapBigFloatToHost(
-    NXSwappedFloat	x
-)
-{
-    return NXConvertSwappedFloatToHost(x);
-}
-
-#endif /* KERNEL */
-
-static __inline__
-unsigned short
-NXSwapHostShortToBig(
-    unsigned short	x
-)
-{
-    return (x);
-}
-
-static __inline__
-unsigned int
-NXSwapHostIntToBig(
-    unsigned int	x
-)
-{
-    return (x);
-}
-
-static __inline__
-unsigned long
-NXSwapHostLongToBig(
-    unsigned long	x
-)
-{
-    return (x);
-}
-
-static __inline__
-unsigned long long
-NXSwapHostLongLongToBig(
-    unsigned long long	x
-)
-{
-    return (x);
-}
-
-#ifndef KERNEL
-
-static __inline__
-NXSwappedDouble
-NXSwapHostDoubleToBig(
-    double		x
-)
-{
-    return NXConvertHostDoubleToSwapped(x);
-}
-
-static __inline__
-NXSwappedFloat
-NXSwapHostFloatToBig(
-    float		x
-)
-{
-    return NXConvertHostFloatToSwapped(x);
-}
-
-#endif /* KERNEL */
-
-static __inline__
-unsigned short
-NXSwapLittleShortToHost(
-    unsigned short	x
-)
-{
-    return (NXSwapShort(x));
-}
-
-static __inline__
-unsigned int
-NXSwapLittleIntToHost(
-    unsigned int	x
-)
-{
-    return (NXSwapInt(x));
-}
-
-static __inline__
-unsigned long
-NXSwapLittleLongToHost(
-    unsigned long	x
-)
-{
-    return (NXSwapLong(x));
-}
-
-static __inline__
-unsigned long long
-NXSwapLittleLongLongToHost(
-    unsigned long long	x
-)
-{
-    return (NXSwapLongLong(x));
-}
-
-#ifndef KERNEL
-
-static __inline__
-double
-NXSwapLittleDoubleToHost(
-    NXSwappedDouble	x
-)
-{
-    return NXConvertSwappedDoubleToHost(NXSwapDouble(x));
-}
-
-static __inline__
-float
-NXSwapLittleFloatToHost(
-    NXSwappedFloat	x
-)
-{
-    return NXConvertSwappedFloatToHost(NXSwapFloat(x));
-}
-
-#endif /* KERNEL */
-
-static __inline__
-unsigned short
-NXSwapHostShortToLittle(
-    unsigned short	x
-)
-{
-    return (NXSwapShort(x));
-}
-
-static __inline__
-unsigned int
-NXSwapHostIntToLittle(
-    unsigned int	x
-)
-{
-    return (NXSwapInt(x));
-}
-
-static __inline__
-unsigned long
-NXSwapHostLongToLittle(
-    unsigned long	x
-)
-{
-    return (NXSwapLong(x));
-}
-
-static __inline__
-unsigned long long
-NXSwapHostLongLongToLittle(
-    unsigned long long	x
-)
-{
-    return (NXSwapLongLong(x));
-}
-
-#ifndef KERNEL
-
-static __inline__
-NXSwappedDouble
-NXSwapHostDoubleToLittle(
-    double		x
-)
-{
-    return NXSwapDouble(NXConvertHostDoubleToSwapped(x));
-}
-
-static __inline__
-NXSwappedFloat
-NXSwapHostFloatToLittle(
-    float		x
-)
-{
-    return NXSwapFloat(NXConvertHostFloatToSwapped(x));
-}
-
-#endif /* KERNEL */
-#endif /*__BIG_ENDIAN__ */
-
-#if	__LITTLE_ENDIAN__
-
-static __inline__
-unsigned short
-NXSwapBigShortToHost(
-    unsigned short	x
-)
-{
-    return (NXSwapShort(x));
-}
-
-static __inline__
-unsigned int
-NXSwapBigIntToHost(
-    unsigned int	x
-)
-{
-    return (NXSwapInt(x));
-}
-
-static __inline__
-unsigned long
-NXSwapBigLongToHost(
-    unsigned long	x
-)
-{
-    return (NXSwapLong(x));
-}
-
-static __inline__
-unsigned long long
-NXSwapBigLongLongToHost(
-    unsigned long long	x
-)
-{
-    return (NXSwapLongLong(x));
-}
-
-static __inline__
-double
-NXSwapBigDoubleToHost(
-    NXSwappedDouble	x
-)
-{
-    return NXConvertSwappedDoubleToHost(NXSwapDouble(x));
-}
-
-static __inline__
-float
-NXSwapBigFloatToHost(
-    NXSwappedFloat	x
-)
-{
-    return NXConvertSwappedFloatToHost(NXSwapFloat(x));
-}
-
-static __inline__
-unsigned short
-NXSwapHostShortToBig(
-    unsigned short	x
-)
-{
-    return (NXSwapShort(x));
-}
-
-static __inline__
-unsigned int
-NXSwapHostIntToBig(
-    unsigned int	x
-)
-{
-    return (NXSwapInt(x));
-}
-
-static __inline__
-unsigned long
-NXSwapHostLongToBig(
-    unsigned long	x
-)
-{
-    return (NXSwapLong(x));
-}
-
-static __inline__
-unsigned long long
-NXSwapHostLongLongToBig(
-    unsigned long long	x
-)
-{
-    return (NXSwapLongLong(x));
-}
-
-static __inline__
-NXSwappedDouble
-NXSwapHostDoubleToBig(
-    double		x
-)
-{
-    return (NXSwapDouble(NXConvertHostDoubleToSwapped(x)));
-}
-
-static __inline__
-NXSwappedFloat
-NXSwapHostFloatToBig(
-    float		x
-)
-{
-    return (NXSwapFloat(NXConvertHostFloatToSwapped(x)));
-}
-
-static __inline__
-unsigned short
-NXSwapLittleShortToHost(
-    unsigned short	x
-)
-{
-    return (x);
-}
-
-static __inline__
-unsigned int
-NXSwapLittleIntToHost(
-    unsigned int	x
-)
-{
-    return (x);
-}
-
-static __inline__
-unsigned long
-NXSwapLittleLongToHost(
-    unsigned long	x
-)
-{
-    return (x);
-}
-
-static __inline__
-unsigned long long
-NXSwapLittleLongLongToHost(
-    unsigned long long	x
-)
-{
-    return (x);
-}
-
-static __inline__
-double
-NXSwapLittleDoubleToHost(
-    NXSwappedDouble	x
-)
-{
-    return NXConvertSwappedDoubleToHost(x);
-}
-
-static __inline__
-float
-NXSwapLittleFloatToHost(
-    NXSwappedFloat	x
-)
-{
-    return NXConvertSwappedFloatToHost(x);
-}
-
-static __inline__
-unsigned short
-NXSwapHostShortToLittle(
-    unsigned short	x
-)
-{
-    return (x);
-}
-
-static __inline__
-unsigned int
-NXSwapHostIntToLittle(
-    unsigned int	x
-)
-{
-    return (x);
-}
-
-static __inline__
-unsigned long
-NXSwapHostLongToLittle(
-    unsigned long	x
-)
-{
-    return (x);
-}
-
-static __inline__
-unsigned long long
-NXSwapHostLongLongToLittle(
-    unsigned long long	x
-)
-{
-    return (x);
-}
-
-static __inline__
-NXSwappedDouble
-NXSwapHostDoubleToLittle(
-    double		x
-)
-{
-    return NXConvertHostDoubleToSwapped(x);
-}
-
-static __inline__
-NXSwappedFloat
-NXSwapHostFloatToLittle(
-    float		x
-)
-{
-    return NXConvertHostFloatToSwapped(x);
-}
-
-#endif /* __LITTLE_ENDIAN__ */
-
-#endif	/* _ARCHITECTURE_BYTE_ORDER_H_ */
diff --git a/EXTERNAL_HEADERS/architecture/i386/Makefile b/EXTERNAL_HEADERS/architecture/i386/Makefile
index 5140ed922..5ab63fe3e 100644
--- a/EXTERNAL_HEADERS/architecture/i386/Makefile
+++ b/EXTERNAL_HEADERS/architecture/i386/Makefile
@@ -12,13 +12,11 @@ INSTINC_SUBDIRS_I386 =
 EXPORT_FILES = 		\
 	asm_help.h	\
 	cpu.h		\
-	fpu.h		\
 	io.h		\
+	pio.h		\
 	sel.h		\
 	tss.h		\
-	byte_order.h	\
 	desc.h		\
-	frame.h		\
 	reg_help.h	\
 	table.h
 
diff --git a/EXTERNAL_HEADERS/architecture/i386/byte_order.h b/EXTERNAL_HEADERS/architecture/i386/byte_order.h
deleted file mode 100644
index fb0cb4a74..000000000
--- a/EXTERNAL_HEADERS/architecture/i386/byte_order.h
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the License
- * may not be used to create, or enable the creation or redistribution of,
- * unlawful or unlicensed copies of an Apple operating system, or to
- * circumvent, violate, or enable the circumvention or violation of, any
- * terms of an Apple operating system software license agreement.
- * 
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- * 
- * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
- */
-/*
- * Copyright (c) 1992 NeXT Computer, Inc.
- *
- * Byte ordering conversion (for i386).
- *
- * HISTORY
- *
- * 8 October 1992 ? at NeXT
- *	Converted to NXxxx versions.  Condensed history.
- *
- * 18 May 1992 ? at NeXT
- *	Created.
- */
- 
-static __inline__
-unsigned short
-NXSwapShort(
-    unsigned short	inv
-)
-{
-    register unsigned short value = inv;
-    
-    __asm__ volatile( "xchgb %h1, %b1" : "=q" (value) : "0" (value));
-
-    return (value);
-}
- 
-static __inline__
-unsigned long
-NXSwapInt(
-    unsigned long inv
-)
-{
-    register unsigned int outv = inv;
-    
-    __asm__ volatile( "bswap %0" : "=r"	(outv) : "0" (outv));
-	
-    return (outv);
-}
- 
-static __inline__
-unsigned long
-NXSwapLong(
-    unsigned long	inv
-)
-{
-    unsigned long	outv;
-    
-    __asm__ volatile(
-    	"bswap %0"
-	
-	: "=r"	(outv)
-	: "0"	(inv));
-	
-    return (outv);
-}
-
-static __inline__
-unsigned long long
-NXSwapLongLong(
-    unsigned long long	inv
-)
-{
-    union llconv {
-	unsigned long long	ull;
-	unsigned long		ul[2];
-    } *inp, outv;
-    
-    inp = (union llconv *)&inv;
-    
-    outv.ul[0] = NXSwapLong(inp->ul[1]);
-    outv.ul[1] = NXSwapLong(inp->ul[0]);
-    	
-    return (outv.ull);
-}
-
-static __inline__ NXSwappedFloat
-NXConvertHostFloatToSwapped(float x)
-{
-    union fconv {
-	float number;
-	NXSwappedFloat sf;
-    };
-    return ((union fconv *)&x)->sf;
-}
-
-static __inline__ float
-NXConvertSwappedFloatToHost(NXSwappedFloat x)
-{
-    union fconv {
-	float number;
-	NXSwappedFloat sf;
-    };
-    return ((union fconv *)&x)->number;
-}
-
-static __inline__ NXSwappedDouble
-NXConvertHostDoubleToSwapped(double x)
-{
-    union dconv {
-	double number;
-	NXSwappedDouble sd;
-    };
-    return ((union dconv *)&x)->sd;
-}
-
-static __inline__ double
-NXConvertSwappedDoubleToHost(NXSwappedDouble x)
-{
-    union dconv {
-	double number;
-	NXSwappedDouble sd;
-    };
-    return ((union dconv *)&x)->number;
-}
-
-static __inline__ NXSwappedFloat
-NXSwapFloat(NXSwappedFloat x)
-{
-    return NXSwapLong(x);
-}
-
-static __inline__ NXSwappedDouble
-NXSwapDouble(NXSwappedDouble x)
-{
-    return NXSwapLongLong(x);
-}
diff --git a/EXTERNAL_HEADERS/architecture/i386/fpu.h b/EXTERNAL_HEADERS/architecture/i386/fpu.h
deleted file mode 100644
index 82edeacb2..000000000
--- a/EXTERNAL_HEADERS/architecture/i386/fpu.h
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the License
- * may not be used to create, or enable the creation or redistribution of,
- * unlawful or unlicensed copies of an Apple operating system, or to
- * circumvent, violate, or enable the circumvention or violation of, any
- * terms of an Apple operating system software license agreement.
- * 
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- * 
- * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
- */
-/*
- * Copyright (c) 1992 NeXT Computer, Inc.
- *
- * Intel386 Family:	Floating Point unit.
- *
- * HISTORY
- *
- * 5 October 1992 ? at NeXT
- *	Added names to previously unamed fields in the mantissa.
- *
- * 5 April 1992 ? at NeXT
- *	Created.
- */
-
-/*
- * Data register.
- */
-
-typedef struct fp_data_reg {
-    unsigned short		mant;
-    unsigned short		mant1	:16,
-				mant2	:16,
-				mant3	:16;
-    unsigned short		exp	:15,
-			    	sign	:1;
-} fp_data_reg_t;
-
-/*
- * Data register stack.
- */
-
-typedef struct fp_stack {
-    fp_data_reg_t		ST[8];
-} fp_stack_t;
-
-/*
- * Register stack tag word.
- */
-
-typedef struct fp_tag {
-    unsigned short		tag0	:2,
-    				tag1	:2,
-				tag2	:2,
-				tag3	:2,
-    				tag4	:2,
-				tag5	:2,
-				tag6	:2,
-				tag7	:2;
-#define FP_TAG_VALID		0
-#define FP_TAG_ZERO		1
-#define FP_TAG_SPEC		2
-#define FP_TAG_EMPTY		3
-} fp_tag_t;
-
-/*
- * Status word.
- */
-
-typedef struct fp_status {
-    unsigned short		invalid	:1,
-    				denorm	:1,
-				zdiv	:1,
-				ovrfl	:1,
-				undfl	:1,
-				precis	:1,
-				stkflt	:1,
-				errsumm	:1,
-				c0	:1,
-				c1	:1,
-				c2	:1,
-				tos	:3,
-				c3	:1,
-				busy	:1;
-} fp_status_t;
-
-/*
- * Control word.
- */
-
-typedef struct fp_control {
-    unsigned short		invalid	:1,
-    				denorm	:1,
-				zdiv	:1,
-				ovrfl	:1,
-				undfl	:1,
-				precis	:1,
-					:2,
-				pc	:2,
-#define FP_PREC_24B		0
-#define	FP_PREC_53B		2
-#define FP_PREC_64B		3
-				rc	:2,
-#define FP_RND_NEAR		0
-#define FP_RND_DOWN		1
-#define FP_RND_UP		2
-#define FP_CHOP			3
-				/*inf*/	:1,
-					:3;
-} fp_control_t;
-
-#include <architecture/i386/sel.h>
-
-/*
- * Floating point 'environment'
- * used by FSTENV/FLDENV instructions.
- */
-
-typedef struct fp_env {
-    fp_control_t		control;
-    unsigned short			:16;
-    fp_status_t			status;
-    unsigned short			:16;
-    fp_tag_t			tag;
-    unsigned short			:16;
-    unsigned int		ip;
-    sel_t			cs;
-    unsigned short		opcode;
-    unsigned int		dp;
-    sel_t			ds;
-    unsigned short			:16;
-} fp_env_t;
-
-/*
- * Floating point state
- * used by FSAVE/FRSTOR instructions.
- */
- 
-typedef struct fp_state {
-    fp_env_t			environ;
-    fp_stack_t			stack;
-} fp_state_t;
diff --git a/EXTERNAL_HEADERS/architecture/i386/frame.h b/EXTERNAL_HEADERS/architecture/i386/frame.h
deleted file mode 100644
index 5e280bfbb..000000000
--- a/EXTERNAL_HEADERS/architecture/i386/frame.h
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the License
- * may not be used to create, or enable the creation or redistribution of,
- * unlawful or unlicensed copies of an Apple operating system, or to
- * circumvent, violate, or enable the circumvention or violation of, any
- * terms of an Apple operating system software license agreement.
- * 
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- * 
- * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
- */
-/*
- * Copyright (c) 1992 NeXT Computer, Inc.
- *
- * Intel386 Family:	Processor exception frame.
- *
- * HISTORY
- *
- * 31 August 1992 ? at NeXT
- *	Added v86 mode stuff.
- *
- * 8 June 1992 ? at NeXT
- *	Changed name of write field in err_code_t
- *	which collided with write() in shlib.
- *
- * 30 March 1992 ? at NeXT
- *	Created.
- */
-
-/*
- * Format of the error code
- * generated by the hardware
- * for certain exceptions.
- */
- 
-typedef union err_code {
-    struct err_code_normal {
-	unsigned int	ext	:1,
-			tbl	:2,
-#define ERR_GDT		0
-#define ERR_IDT		1
-#define ERR_LDT		2
-			index	:13,
-				:16;
-    } normal;
-    struct err_code_pgfault {
-	unsigned int	prot	:1,
-			wrtflt	:1,
-			user	:1,
-				:29;
-    } pgfault;
-} err_code_t;
-
-#include <architecture/i386/sel.h>
-
-/*
- * The actual hardware exception frame
- * is variable in size.  An error code is
- * only pushed for certain exceptions.
- * Previous stack information is only
- * pushed for exceptions that cause a
- * change in privilege level.  The dpl
- * field of the saved CS selector can be
- * used to determine whether this is the
- * case.  If the interrupted task was
- * executing in v86 mode, then the data
- * segment registers are also present in
- * the exception frame (in addition to
- * previous stack information).  This
- * case can be determined by examining
- * eflags.
- */
-
-typedef struct except_frame {
-    err_code_t		err;
-    unsigned int	eip;
-    sel_t		cs;
-    unsigned int		:0;
-    unsigned int	eflags;
-    unsigned int	esp;
-    sel_t		ss;
-    unsigned int		:0;
-    unsigned short	v_es;
-    unsigned int		:0;
-    unsigned short	v_ds;
-    unsigned int		:0;
-    unsigned short	v_fs;
-    unsigned int		:0;
-    unsigned short	v_gs;
-    unsigned int		:0;
-} except_frame_t;
-
-/*
- * Values in eflags.
- */
-
-#ifndef	EFL_CF	/* FIXME */
-#define EFL_CF		0x00001
-#define EFL_PF		0x00004
-#define EFL_AF		0x00010
-#define EFL_ZF		0x00040
-#define EFL_SF		0x00080
-#define EFL_TF		0x00100
-#define EFL_IF		0x00200
-#define EFL_DF		0x00400
-#define EFL_OF		0x00800
-#define EFL_IOPL	0x03000
-#define EFL_NT		0x04000
-#define EFL_RF		0x10000
-#define EFL_VM		0x20000
-#define EFL_AC		0x40000
-#endif
-
-#define EFL_CLR		0xfff88028
-#define EFL_SET		0x00000002
diff --git a/EXTERNAL_HEADERS/architecture/i386/pio.h b/EXTERNAL_HEADERS/architecture/i386/pio.h
new file mode 100644
index 000000000..1c9586d62
--- /dev/null
+++ b/EXTERNAL_HEADERS/architecture/i386/pio.h
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+/* 
+ */
+#ifndef _ARCH_I386_PIO_H_
+#define _ARCH_I386_PIO_H_
+
+typedef unsigned short i386_ioport_t;
+
+#if defined(__GNUC__)
+static __inline__ unsigned long	inl(
+				i386_ioport_t port)
+{
+	unsigned long datum;
+	__asm__ volatile("inl %w1, %0" : "=a" (datum) : "Nd" (port));
+	return(datum);
+}
+
+static __inline__ unsigned short inw(
+				i386_ioport_t port)
+{
+	unsigned short datum;
+	__asm__ volatile("inw %w1, %w0" : "=a" (datum) : "Nd" (port));
+	return(datum);
+}
+
+static __inline__ unsigned char inb(
+				i386_ioport_t port)
+{
+	unsigned char datum;
+	__asm__ volatile("inb %w1, %b0" : "=a" (datum) : "Nd" (port));
+	return(datum);
+}
+
+static __inline__ void outl(
+				i386_ioport_t port,
+				unsigned long datum)
+{
+	__asm__ volatile("outl %0, %w1" : : "a" (datum), "Nd" (port));
+}
+
+static __inline__ void outw(
+				i386_ioport_t port,
+				unsigned short datum)
+{
+	__asm__ volatile("outw %w0, %w1" : : "a" (datum), "Nd" (port));
+}
+
+static __inline__ void outb(
+				i386_ioport_t port,
+				unsigned char datum)
+{
+	__asm__ volatile("outb %b0, %w1" : : "a" (datum), "Nd" (port));
+}
+#endif /* defined(__GNUC__) */
+#endif /* _ARCH_I386_PIO_H_ */
diff --git a/EXTERNAL_HEADERS/architecture/ppc/Makefile b/EXTERNAL_HEADERS/architecture/ppc/Makefile
index 8cf3b09c2..374f3bd9a 100644
--- a/EXTERNAL_HEADERS/architecture/ppc/Makefile
+++ b/EXTERNAL_HEADERS/architecture/ppc/Makefile
@@ -12,7 +12,6 @@ INSTINC_SUBDIRS_PPC =
 EXPORT_FILES = 		\
 	asm_help.h	\
 	basic_regs.h	\
-	byte_order.h	\
 	cframe.h	\
 	fp_regs.h	\
 	macro_help.h	\
diff --git a/EXTERNAL_HEADERS/architecture/ppc/byte_order.h b/EXTERNAL_HEADERS/architecture/ppc/byte_order.h
deleted file mode 100644
index f271cc6fb..000000000
--- a/EXTERNAL_HEADERS/architecture/ppc/byte_order.h
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the License
- * may not be used to create, or enable the creation or redistribution of,
- * unlawful or unlicensed copies of an Apple operating system, or to
- * circumvent, violate, or enable the circumvention or violation of, any
- * terms of an Apple operating system software license agreement.
- * 
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- * 
- * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
- */
-/*
- * Copyright (c) 1996 NeXT Software, Inc.
- *
- * Byte ordering conversion (for ppc).
- */
-
-static __inline__
-unsigned short
-NXSwapShort(
-    unsigned short	inv
-)
-{
-    union sconv {
-	unsigned short	us;
-	unsigned char	uc[2];
-    } *inp, outv;
-    
-    inp = (union sconv *)&inv;
-    
-    outv.uc[0] = inp->uc[1];
-    outv.uc[1] = inp->uc[0];
-    
-    return (outv.us);
-}
-
-static __inline__
-unsigned int
-NXSwapInt(
-    unsigned int	inv
-)
-{
-    union iconv {
-	unsigned int	ui;
-	unsigned char	uc[4];
-    } *inp, outv;
-    
-    inp = (union iconv *)&inv;
-    
-    outv.uc[0] = inp->uc[3];
-    outv.uc[1] = inp->uc[2];
-    outv.uc[2] = inp->uc[1];
-    outv.uc[3] = inp->uc[0];
-    
-    return (outv.ui);
-}
-
-static __inline__
-unsigned long long
-NXSwapLongLong(
-    unsigned long long	inv
-)
-{
-    union llconv {
-	unsigned long long	ull;
-	unsigned char		uc[8];
-    } *inp, outv;
-    
-    inp = (union llconv *)&inv;
-    
-    outv.uc[0] = inp->uc[7];
-    outv.uc[1] = inp->uc[6];
-    outv.uc[2] = inp->uc[5];
-    outv.uc[3] = inp->uc[4];
-    outv.uc[4] = inp->uc[3];
-    outv.uc[5] = inp->uc[2];
-    outv.uc[6] = inp->uc[1];
-    outv.uc[7] = inp->uc[0];
-    
-    return (outv.ull);
-}
-
-#if defined(__LP64__)
-
-static __inline__
-unsigned long 
-NXSwapLong(
-    unsigned long inv
-)
-{
-    union llconv {
-	unsigned long		ul;
-	unsigned char		uc[8];
-    } *inp, outv;
-    
-    inp = (union llconv *)&inv;
-    
-    outv.uc[0] = inp->uc[7];
-    outv.uc[1] = inp->uc[6];
-    outv.uc[2] = inp->uc[5];
-    outv.uc[3] = inp->uc[4];
-    outv.uc[4] = inp->uc[3];
-    outv.uc[5] = inp->uc[2];
-    outv.uc[6] = inp->uc[1];
-    outv.uc[7] = inp->uc[0];
-    
-    return (outv.ul);
-}
-
-#else 
-
-static __inline__
-unsigned long
-NXSwapLong(
-    unsigned long	inv
-)
-{
-    union lconv {
-	unsigned long	ul;
-	unsigned char	uc[4];
-    } *inp, outv;
-    
-    inp = (union lconv *)&inv;
-    
-    outv.uc[0] = inp->uc[3];
-    outv.uc[1] = inp->uc[2];
-    outv.uc[2] = inp->uc[1];
-    outv.uc[3] = inp->uc[0];
-    
-    return (outv.ul);
-}
-
-#endif /* __LP64__ */
-
-#ifndef KERNEL
-
-static __inline__ NXSwappedFloat
-NXConvertHostFloatToSwapped(float x)
-{
-    union fconv {
-	float number;
-	NXSwappedFloat sf;
-    };
-    return ((union fconv *)&x)->sf;
-}
-
-static __inline__ float
-NXConvertSwappedFloatToHost(NXSwappedFloat x)
-{
-    union fconv {
-	float number;
-	NXSwappedFloat sf;
-    };
-    return ((union fconv *)&x)->number;
-}
-
-static __inline__ NXSwappedDouble
-NXConvertHostDoubleToSwapped(double x)
-{
-    union dconv {
-	double number;
-	NXSwappedDouble sd;
-    };
-    return ((union dconv *)&x)->sd;
-}
-
-static __inline__ double
-NXConvertSwappedDoubleToHost(NXSwappedDouble x)
-{
-    union dconv {
-	double number;
-	NXSwappedDouble sd;
-    };
-    return ((union dconv *)&x)->number;
-}
-
-static __inline__ NXSwappedFloat
-NXSwapFloat(NXSwappedFloat x)
-{
-    return NXSwapLong(x);
-}
-
-static __inline__ NXSwappedDouble
-NXSwapDouble(NXSwappedDouble x)
-{
-    return NXSwapLongLong(x);
-}
-
-#endif /* ! KERNEL */
diff --git a/EXTERNAL_HEADERS/mach-o/fat.h b/EXTERNAL_HEADERS/mach-o/fat.h
index f305ece2a..6e724325f 100644
--- a/EXTERNAL_HEADERS/mach-o/fat.h
+++ b/EXTERNAL_HEADERS/mach-o/fat.h
@@ -45,10 +45,9 @@
  * and contains the constants for the possible values of these types.
  */
 #include <mach/machine.h>
-#include <architecture/byte_order.h>
 
 #define FAT_MAGIC	0xcafebabe
-#define FAT_CIGAM	NXSwapLong(FAT_MAGIC)
+#define FAT_CIGAM	0xbebafeca	
 
 struct fat_header {
 	unsigned long	magic;		/* FAT_MAGIC */
diff --git a/EXTERNAL_HEADERS/mach-o/loader.h b/EXTERNAL_HEADERS/mach-o/loader.h
index 74d770269..9536cadf7 100644
--- a/EXTERNAL_HEADERS/mach-o/loader.h
+++ b/EXTERNAL_HEADERS/mach-o/loader.h
@@ -49,7 +49,6 @@
  * states and the structures of those flavors for each machine.
  */
 #include <mach/machine/thread_status.h>
-#include <architecture/byte_order.h>
 
 /*
  * The mach header appears at the very beginning of the object file; it
@@ -82,11 +81,11 @@ struct mach_header_64 {
 
 /* Constant for the magic field of the mach_header (32-bit architectures) */
 #define	MH_MAGIC	0xfeedface	/* the mach magic number */
-#define MH_CIGAM	NXSwapInt(MH_MAGIC)
+#define MH_CIGAM	0xcefaedfe
 
 /* Constant for the magic field of the mach_header_64 (64-bit architectures) */
 #define MH_MAGIC_64	0xfeedfacf	/* the 64-bit mach magic number */
-#define MH_CIGAM_64	NXSwapInt(MH_MAGIC_64)
+#define MH_CIGAM_64	0xcffaedfe	
 
 /* Constants for the cmd field of new load commands, the type */
 #define LC_SEGMENT_64	0x19	/* 64-bit segment of this file to be mapped */
@@ -139,6 +138,10 @@ struct mach_header_64 {
 					   linker when loaded. */
 #define MH_PREBOUND	0x10		/* the file has it's dynamic undefined
 					   references prebound. */
+#define MH_ALLOW_STACK_EXECUTION 0x20000/* When this bit is set, all stacks 
+					   in the task will be given stack
+					   execution privilege.  Only used in
+					   MH_EXECUTE filetypes. */
 
 /*
  * The load commands directly follow the mach_header.  The total size of all
diff --git a/EXTERNAL_HEADERS/stdbool.h b/EXTERNAL_HEADERS/stdbool.h
new file mode 100644
index 000000000..be545ce1a
--- /dev/null
+++ b/EXTERNAL_HEADERS/stdbool.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2000 Jeroen Ruigrok van der Werven <asmodai@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD: src/include/stdbool.h,v 1.6 2002/08/16 07:33:14 alfred Exp $
+ */
+
+#ifndef _STDBOOL_H_
+#define	_STDBOOL_H_	
+
+#define	__bool_true_false_are_defined	1
+
+#ifndef __cplusplus
+
+#define	false	0
+#define	true	1
+
+#define	bool	_Bool
+#if __STDC_VERSION__ < 199901L && __GNUC__ < 3
+typedef	int	_Bool;
+#endif
+
+#endif /* !__cplusplus */
+
+#endif /* !_STDBOOL_H_ */
diff --git a/bsd/bsm/audit_kevents.h b/bsd/bsm/audit_kevents.h
index 5dcb5ef13..699ef143c 100644
--- a/bsd/bsm/audit_kevents.h
+++ b/bsd/bsm/audit_kevents.h
@@ -327,6 +327,7 @@
 #define AUE_SWAPOFF	355
 #define AUE_INITPROCESS	356
 #define AUE_MAPFD	357
+#define AUE_TASKNAMEFORPID AUE_NULL
 #define AUE_TASKFORPID	358
 #define AUE_PIDFORTASK	359
 #define AUE_SYSCTL_NONADMIN	360
diff --git a/bsd/conf/MASTER b/bsd/conf/MASTER
index 90cb6ba5c..9ccacbf12 100644
--- a/bsd/conf/MASTER
+++ b/bsd/conf/MASTER
@@ -182,7 +182,7 @@ options		NFSSERVER	# Be an NFS server		# <nfsclient>
 # AppleTalk Support
 #
 options		NETAT		# AppleTalk support		# <netat>
-options		AURP_SUPPORT    # AppleTalk Update Routing      # <netat>
+#options		AURP_SUPPORT    # AppleTalk Update Routing      # <netat>
 #
 # Machine Independent Apple Features
 #
diff --git a/bsd/conf/MASTER.i386 b/bsd/conf/MASTER.i386
index 6c9d460bc..d7d200932 100644
--- a/bsd/conf/MASTER.i386
+++ b/bsd/conf/MASTER.i386
@@ -45,9 +45,9 @@
 #  Standard Apple Research Configurations:
 #  -------- ----- -------- ---------------
 #
-#  RELEASE = [intel pc mach medium event vol pst gdb kernobjc libdriver fixpri simple_clock mdebug kernserv driverkit uxpr kernstack ipc_compat ipc_debug nfsclient nfsserver quota fifo fdesc union ffs cd9660 compat_oldsock volfs devfs revfs hfs mrouting ipdivert ipfirewall ipfw2 dummynet ipv6firewall inet6 ipsec gif tcpdrop_synfin ktrace stf compat_43_tty compat_43_socket vlan bond netmibs]
+#  RELEASE = [intel pc mach medium event vol pst gdb kernobjc libdriver fixpri simple_clock mdebug kernserv driverkit uxpr kernstack ipc_compat ipc_debug nfsclient nfsserver quota fifo fdesc union ffs cd9660 compat_oldsock volfs devfs revfs hfs mrouting ipdivert ipfirewall ipfw2 dummynet ipv6firewall inet6 ipsec gif tcpdrop_synfin ktrace stf compat_43_tty compat_43_socket vlan bond netmibs netat]
 #  PROFILE = [RELEASE profile]
-#  DEBUG = [intel pc mach medium event vol pst gdb kernobjc libdriver_g fixpri debug simple_clock mdebug kernserv driverkit xpr_debug uxpr kernstack ipc_compat ipc_debug nfsclient nfsserver quota fifo fdesc union ffs cd9660 compat_oldsock revfs hfs volfs devfs mach_assert mrouting ipdivert ipfirewall ipfw2 dummynet ipv6firewall inet6 ipsec gif tcpdrop_synfin ktrace stf compat_43_tty compat_43_socket vlan bond netmibs]
+#  DEBUG = [intel pc mach medium event vol pst gdb kernobjc libdriver_g fixpri debug simple_clock mdebug kernserv driverkit xpr_debug uxpr kernstack ipc_compat ipc_debug nfsclient nfsserver quota fifo fdesc union ffs cd9660 compat_oldsock revfs hfs volfs devfs mach_assert mrouting ipdivert ipfirewall ipfw2 dummynet ipv6firewall inet6 ipsec gif tcpdrop_synfin ktrace stf compat_43_tty compat_43_socket vlan bond netmibs netat]
 #
 ######################################################################
 #
diff --git a/bsd/conf/Makefile.i386 b/bsd/conf/Makefile.i386
index c6a7619c4..cbe71f0a8 100644
--- a/bsd/conf/Makefile.i386
+++ b/bsd/conf/Makefile.i386
@@ -199,8 +199,8 @@ OBJS_NO_WERROR =		\
 	tcp_output.o		\
 	tcp_subr.o		\
 	tcp_timer.o		\
-	tcp_usrreq.o		\
 	tcp_sack.o		\
+	tcp_usrreq.o		\
 	udp_usrreq.o		\
 	ah_core.o		\
 	ah_input.o		\
@@ -291,7 +291,75 @@ OBJS_NO_WERROR =		\
 	sysctl.o		\
 	unix_startup.o 		\
 	memdev.o		\
-	init_sysent.o
+	randomdev.o		\
+	sha1mod.o		\
+	inet_ntop.o		\
+	vfs_attrlist.o	\
+	volfs_vnops.o	\
+	bpf_filter.o	\
+	devtimer.o		\
+	in_arp.o		\
+	ip_ecn.o		\
+	key_debug.o		\
+	nfs_serv.o		\
+	nfs_lock.o		\
+	kern_lockf.o	\
+	kern_subr.o		\
+	sysv_msg.o		\
+	vnode_pager.o	\
+	dp_backing_file.o \
+	vm_unix.o		\
+	param.o			\
+	mem.o			\
+	km.o			\
+	init_sysent.o		\
+	at.o			\
+	adsp.o			\
+	adsp_CLDeny.o		\
+	adsp_CLListen.o		\
+	adsp_Close.o		\
+	adsp_Control.o		\
+	adsp_Init.o		\
+	adsp_InitGlobals.o	\
+	adsp_NewCID.o		\
+	adsp_Open.o		\
+	adsp_Options.o		\
+	adsp_Packet.o		\
+	adsp_Read.o		\
+	adsp_RxAttn.o		\
+	adsp_RxData.o		\
+	adsp_Status.o		\
+	adsp_Timer.o		\
+	adsp_TimerElem.o	\
+	adsp_Write.o		\
+	adsp_attention.o	\
+	adsp_misc.o		\
+	adsp_reset.o		\
+	adsp_stream.o		\
+	at_proto.o		\
+	at_pcb.o		\
+	atp_alloc.o		\
+	atp_misc.o		\
+	atp_open.o		\
+	atp_read.o		\
+	atp_write.o		\
+	ddp_aarp.o		\
+	ddp_aep.o		\
+	ddp_brt.o		\
+	ddp_lap.o		\
+	ddp_nbp.o		\
+	ddp_proto.o		\
+	ddp_r_rtmp.o		\
+	ddp_r_zip.o		\
+	ddp_rtmp.o		\
+	ddp_rtmptable.o		\
+	ddp_sip.o		\
+	ddp.o			\
+	sys_dep.o		\
+	sys_glue.o		\
+	drv_dep.o		\
+	ddp_usrreq.o		\
+	asp_proto.o	
 
 OBJS_WERROR=$(filter-out $(OBJS_NO_WERROR),$(OBJS))
 
diff --git a/bsd/conf/files b/bsd/conf/files
index 109f2b106..268cb3175 100644
--- a/bsd/conf/files
+++ b/bsd/conf/files
@@ -548,6 +548,8 @@ bsd/kern/posix_shm.c			standard
 bsd/kern/qsort.c			standard
 bsd/kern/kpi_socket.c		standard
 bsd/kern/kpi_socketfilter.c	standard
+bsd/kern/proc_info.c			standard
+bsd/kern/socket_info.c			standard
 
 bsd/vm/vnode_pager.c			standard
 bsd/vm/vm_unix.c			standard
@@ -557,3 +559,6 @@ bsd/uxkern/ux_exception.c		standard
 
 bsd/conf/param.c			standard
 ./ioconf.c				standard
+
+bsd/dev/chud/chud_bsd_callback.c	standard
+bsd/dev/chud/chud_process.c		standard
diff --git a/bsd/conf/files.i386 b/bsd/conf/files.i386
index cc998565b..3eb14dd0d 100644
--- a/bsd/conf/files.i386
+++ b/bsd/conf/files.i386
@@ -10,9 +10,11 @@ bsd/dev/i386/km.c		standard
 bsd/dev/i386/kern_machdep.c	standard
 bsd/dev/i386/memmove.c		standard
 bsd/dev/i386/stubs.c		standard
+bsd/dev/i386/systemcalls.c	standard
 bsd/dev/i386/lock_stubs.c	standard
 bsd/dev/i386/sysctl.c           standard
 bsd/dev/i386/unix_signal.c	standard
+bsd/dev/i386/munge.s		standard
 
 
 bsd/kern/bsd_stubs.c		standard
diff --git a/bsd/conf/files.ppc b/bsd/conf/files.ppc
index 36adc9a65..c24d147e5 100644
--- a/bsd/conf/files.ppc
+++ b/bsd/conf/files.ppc
@@ -19,8 +19,5 @@ bsd/dev/ppc/xsumas.s		standard
 bsd/dev/ppc/sysctl.c		standard	
 bsd/dev/ppc/munge.s		standard
 
-bsd/dev/ppc/chud/chud_bsd_callback.c	standard
-bsd/dev/ppc/chud/chud_process.c		standard
-
 bsd/kern/bsd_stubs.c		standard
 
diff --git a/bsd/conf/param.c b/bsd/conf/param.c
index b1812db65..97b008e15 100644
--- a/bsd/conf/param.c
+++ b/bsd/conf/param.c
@@ -85,8 +85,9 @@
 struct	timezone tz = { TIMEZONE, PST };
 
 #define	NPROC (20 + 16 * MAXUSERS)
-#define HNPROC (20 + 64 * MAXUSERS)
+#define HNPROC 2500	/* based on thread_max */
 int	maxproc = NPROC;
+int	maxprocperuid = NPROC/2;
 __private_extern__ int hard_maxproc = HNPROC;	/* hardcoded limit */
 int nprocs = 0; /* XXX */
 
diff --git a/bsd/dev/ppc/chud/chud_bsd_callback.c b/bsd/dev/chud/chud_bsd_callback.c
similarity index 57%
rename from bsd/dev/ppc/chud/chud_bsd_callback.c
rename to bsd/dev/chud/chud_bsd_callback.c
index 21d85209e..68daf186d 100644
--- a/bsd/dev/ppc/chud/chud_bsd_callback.c
+++ b/bsd/dev/chud/chud_bsd_callback.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -40,18 +40,30 @@
 typedef void (*chudxnu_kdebug_callback_func_t)(uint32_t debugid, uint32_t arg0, uint32_t arg1, uint32_t arg2, uint32_t arg3, uint32_t arg4);
 static chudxnu_kdebug_callback_func_t kdebug_callback_fn = NULL;
 
-extern void kdbg_control_chudxnu(int val, void *fn);
+kern_return_t chudxnu_kdebug_callback_enter(chudxnu_kdebug_callback_func_t);
+kern_return_t chudxnu_kdebug_callback_cancel(void);
+
+extern void kdbg_control_chud(int val, void *fn);
 extern unsigned int kdebug_enable;
 
-static void chudxnu_private_kdebug_callback(unsigned int debugid, unsigned int arg0, unsigned int arg1, unsigned int arg2, unsigned int arg3, unsigned int arg4)
+static void
+chudxnu_private_kdebug_callback(
+	unsigned int debugid,
+	unsigned int arg0,
+	unsigned int arg1,
+	unsigned int arg2,
+	unsigned int arg3,
+	unsigned int arg4)
 {
-    if(kdebug_callback_fn) {
-        (kdebug_callback_fn)(debugid, arg0, arg1, arg2, arg3, arg4);
+    chudxnu_kdebug_callback_func_t fn = kdebug_callback_fn;
+    
+    if(fn) {
+        (fn)(debugid, arg0, arg1, arg2, arg3, arg4);
     }
 }
 
-__private_extern__
-kern_return_t chudxnu_kdebug_callback_enter(chudxnu_kdebug_callback_func_t func)
+__private_extern__ kern_return_t
+chudxnu_kdebug_callback_enter(chudxnu_kdebug_callback_func_t func)
 {
     kdebug_callback_fn = func;
 
@@ -61,8 +73,8 @@ kern_return_t chudxnu_kdebug_callback_enter(chudxnu_kdebug_callback_func_t func)
     return KERN_SUCCESS;
 }
 
-__private_extern__
-kern_return_t chudxnu_kdebug_callback_cancel(void)
+__private_extern__ kern_return_t
+chudxnu_kdebug_callback_cancel(void)
 {
     kdebug_callback_fn = NULL;
     kdbg_control_chud(FALSE, NULL);
@@ -70,3 +82,42 @@ kern_return_t chudxnu_kdebug_callback_cancel(void)
 
     return KERN_SUCCESS;
 }
+
+#pragma mark **** CHUD syscall ****
+typedef kern_return_t (*chudxnu_syscall_callback_func_t)(uint32_t code, uint32_t arg0, uint32_t arg1, uint32_t arg2, uint32_t arg3, uint32_t arg4);
+static chudxnu_syscall_callback_func_t syscall_callback_fn = NULL;
+
+kern_return_t chudxnu_syscall_callback_enter(chudxnu_syscall_callback_func_t func);
+kern_return_t chudxnu_syscall_callback_cancel(void);
+
+int chud(p, uap, retval)
+     struct proc *p;
+     struct chud_args *uap;
+     register_t *retval;
+{
+#pragma unused (p)
+
+    chudxnu_syscall_callback_func_t fn = syscall_callback_fn;
+    
+	if(!fn) {
+		return EINVAL;
+	}
+	
+	*retval = fn(uap->code, uap->arg1, uap->arg2, uap->arg3, uap->arg4, uap->arg5);
+		
+	return 0;
+}
+
+__private_extern__
+kern_return_t chudxnu_syscall_callback_enter(chudxnu_syscall_callback_func_t func)
+{
+	syscall_callback_fn = func;
+    return KERN_SUCCESS;
+}
+
+__private_extern__
+kern_return_t chudxnu_syscall_callback_cancel(void)
+{
+	syscall_callback_fn = NULL;
+    return KERN_SUCCESS;
+}
diff --git a/bsd/dev/ppc/chud/chud_process.c b/bsd/dev/chud/chud_process.c
similarity index 70%
rename from bsd/dev/ppc/chud/chud_process.c
rename to bsd/dev/chud/chud_process.c
index e8963c597..a27928e1a 100644
--- a/bsd/dev/ppc/chud/chud_process.c
+++ b/bsd/dev/chud/chud_process.c
@@ -29,9 +29,15 @@
 #include <sys/systm.h>
 #include <sys/proc_internal.h>
 #include <sys/vnode_internal.h> // vn_getpath()
+#include <kern/task.h>
+#include <sys/user.h>
 
-__private_extern__
-int chudxnu_pid_for_task(task_t task)
+int	chudxnu_pid_for_task(task_t task);
+task_t	chudxnu_task_for_pid(int pid);
+int	chudxnu_current_pid(void);
+
+__private_extern__ int
+chudxnu_pid_for_task(task_t task)
 {
     struct proc *p;
 
@@ -44,8 +50,8 @@ int chudxnu_pid_for_task(task_t task)
     return -1;
 }
 
-__private_extern__
-task_t chudxnu_task_for_pid(int pid)
+__private_extern__ task_t
+chudxnu_task_for_pid(int pid)
 {
     struct proc *p = pfind(pid);
     if(p) {
@@ -54,8 +60,23 @@ task_t chudxnu_task_for_pid(int pid)
     return TASK_NULL;
 }
 
-__private_extern__
-int chudxnu_current_pid(void)
+__private_extern__ int
+chudxnu_current_pid(void)
 {
-    return current_proc()->p_pid;
+    int pid = -1;
+    struct uthread *ut = get_bsdthread_info(current_thread());
+    task_t t = current_task();
+
+    if(t != TASK_NULL) {
+        pid = chudxnu_pid_for_task(t);
+    } else {
+        // no task, so try looking in the uthread and/or proc
+        pid = current_proc()->p_pid;
+
+        if(ut && ut->uu_proc) {
+            pid = ut->uu_proc->p_pid;
+        }
+    }
+    
+    return pid;
 }
diff --git a/bsd/dev/i386/kern_machdep.c b/bsd/dev/i386/kern_machdep.c
index 7bdd46a3e..125b62efc 100644
--- a/bsd/dev/i386/kern_machdep.c
+++ b/bsd/dev/i386/kern_machdep.c
@@ -37,92 +37,27 @@
 #include	<sys/types.h>
 #include	<mach/machine.h>
 #include	<kern/cpu_number.h>
-
-extern int grade_binary(cpu_type_t exectype, cpu_subtype_t execsubtype);
+#include	<machine/exec.h>
+#include	<machine/machine_routines.h>
 
 /**********************************************************************
  * Routine:	grade_binary()
  *
- * Function:	Return a relative preference for exectypes and
- *		execsubtypes in fat executable files.  The higher the
- *		grade, the higher the preference.  A grade of 0 means
- *		not acceptable.
+ * Function:	Keep the API the same between PPC and X86; always say
+ *		any CPU subtype is OK with us, but only OK CPU types
+ *		for which we are actually capable of executing the
+ *		binary, either directly or via an imputed interpreter.
  **********************************************************************/
 int
-grade_binary(__unused cpu_type_t exectype, cpu_subtype_t execsubtype)
+grade_binary(cpu_type_t exectype, cpu_subtype_t execsubtype)
 {
-	int		cpusubtype = cpu_subtype();
-
-	switch (cpusubtype) {
-	    case CPU_SUBTYPE_386:
-		switch (execsubtype) {
-		    case CPU_SUBTYPE_386:
-			return 1;
-		    default:
-			return 0;
-		}
-
-	    case CPU_SUBTYPE_486:
-		switch (execsubtype) {
-		    case CPU_SUBTYPE_386:
-			return 1;
-
-		    case CPU_SUBTYPE_486SX:
-			return 2;
-
-		    case CPU_SUBTYPE_486:
-			return 3;
-
-		    default:
-			return 0;
-		}
-
-	    case CPU_SUBTYPE_486SX:
-		switch (execsubtype) {
-		    case CPU_SUBTYPE_386:
-			return 1;
-
-		    case CPU_SUBTYPE_486:
-			return 2;
-
-		    case CPU_SUBTYPE_486SX:
-			return 3;
-
-		    default:
-			return 0;
-		}
-
-	    case CPU_SUBTYPE_586:
-		switch (execsubtype) {
-		    case CPU_SUBTYPE_386:
-			return 1;
-
-		    case CPU_SUBTYPE_486SX:
-			return 2;
-
-		    case CPU_SUBTYPE_486:
-			return 3;
-
-		    case CPU_SUBTYPE_586:
-			return 4;
-
-		    default:
-			return 0;
-		}
-
-	    default:
-		if (	CPU_SUBTYPE_INTEL_MODEL(execsubtype) ==
-		    	CPU_SUBTYPE_INTEL_MODEL_ALL) {
-		    if (	CPU_SUBTYPE_INTEL_FAMILY(cpusubtype) >=
-				CPU_SUBTYPE_INTEL_FAMILY(execsubtype))
-			return CPU_SUBTYPE_INTEL_FAMILY_MAX - 
-			    CPU_SUBTYPE_INTEL_FAMILY(cpusubtype) -
-			    CPU_SUBTYPE_INTEL_FAMILY(execsubtype);
-		}
-		else {
-		    if (	cpusubtype == execsubtype)
-			return CPU_SUBTYPE_INTEL_FAMILY_MAX + 1;
-		}
+	switch(exectype) {
+	case CPU_TYPE_X86:		/* native */
+	case CPU_TYPE_POWERPC:		/* via translator */
+		return 1;
+	case CPU_TYPE_X86_64:		/* native 64-bit */
+		return (ml_is64bit() && execsubtype == CPU_SUBTYPE_X86_64_ALL) ? 2 : 0;
+	default:			/* all other binary types */
 		return 0;
 	}
 }
diff --git a/bsd/dev/i386/km.c b/bsd/dev/i386/km.c
index 7b53090ff..d5ecffc29 100644
--- a/bsd/dev/i386/km.c
+++ b/bsd/dev/i386/km.c
@@ -46,6 +46,7 @@
 #include <sys/time.h>
 #include <dev/kmreg_com.h>
 #include <pexpert/pexpert.h>
+#include <pexpert/i386/boot.h>
 
 extern int hz;
 
@@ -141,7 +142,10 @@ kmopen(
 
 		bzero(&video, sizeof(video));
 		PE_current_console(&video);
-		if( video.v_width != 0 && video.v_height != 0 ) {
+                if( video.v_display == VGA_TEXT_MODE ) {
+                        wp->ws_col = video.v_width;
+                        wp->ws_row = video.v_height;
+                } else if( video.v_width != 0 && video.v_height != 0 ) {
 			wp->ws_col = video.v_width / wp->ws_xpixel;
 			wp->ws_row = video.v_height / wp->ws_ypixel;
 		} else {
diff --git a/bsd/dev/i386/mem.c b/bsd/dev/i386/mem.c
index ee9be2dba..9f4eeddde 100644
--- a/bsd/dev/i386/mem.c
+++ b/bsd/dev/i386/mem.c
@@ -85,8 +85,9 @@
 #include <mach/vm_param.h>
 #include <vm/vm_kern.h>		/* for kernel_map */
 
-extern vm_offset_t kvtophys(vm_offset_t va); 
+extern addr64_t  kvtophys(vm_offset_t va); 
 extern boolean_t kernacc(off_t, size_t );
+extern int setup_kmem;
 
 static caddr_t devzerobuf;
 
@@ -110,9 +111,14 @@ mmwrite(dev_t dev, struct uio *uio)
 }
 
 int
-mmioctl(__unused dev_t dev, u_long cmd, __unused caddr_t data, 
+mmioctl(dev_t dev, u_long cmd, __unused caddr_t data, 
 		__unused int flag, __unused struct proc *p)
 {
+	int minnum = minor(dev);
+
+	if ((setup_kmem == 0) && ((minnum == 0) || (minnum == 1)))
+		return(EINVAL);
+
 	switch (cmd) {
 	case FIONBIO:
 	case FIOASYNC:
@@ -134,6 +140,7 @@ mmrw(dev_t dev, struct uio *uio, enum uio_rw rw)
 	vm_offset_t	where;
 	vm_size_t size;
 
+
 	while (uio_resid(uio) > 0 && error == 0) {
 		if (uio_iov_len(uio) == 0) {
 			uio_next_iov(uio);
@@ -146,6 +153,9 @@ mmrw(dev_t dev, struct uio *uio, enum uio_rw rw)
 
 		/* minor device 0 is physical memory */
 		case 0:
+			if (setup_kmem == 0)
+				return(ENODEV);
+
 			v = trunc_page(uio->uio_offset);
 			if (uio->uio_offset >= mem_size)
 				goto fault;
@@ -164,6 +174,8 @@ mmrw(dev_t dev, struct uio *uio, enum uio_rw rw)
 
 		/* minor device 1 is kernel memory */
 		case 1:
+			if (setup_kmem == 0)
+				return(ENODEV);
 			/* Do some sanity checking */
 			if (((vm_address_t)uio->uio_offset >= VM_MAX_KERNEL_ADDRESS) ||
 				((vm_address_t)uio->uio_offset <= VM_MIN_KERNEL_ADDRESS))
diff --git a/bsd/dev/i386/munge.s b/bsd/dev/i386/munge.s
new file mode 100644
index 000000000..414dd6fe1
--- /dev/null
+++ b/bsd/dev/i386/munge.s
@@ -0,0 +1,211 @@
+/*
+ * Coyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * Syscall argument mungers.
+ *
+ * The data to be munged has been explicitly copied in to the arguement area,
+ * and will be munged in place in the uu_arg[] array.  Because of this, the
+ * functions all take the same arguments as their PPC equivalents, but the
+ * first argument is ignored.  These mungers are for 32-bit app's syscalls,
+ * since 64-bit args are stored into the save area (which overlays the
+ * uu_args) in the order the syscall ABI calls for.
+ *
+ * The issue is that the incoming args are 32-bit, but we must expand
+ * them in place into 64-bit args, as if they were from a 64-bit process.
+ *
+ * There are several functions in this file.  Each takes two parameters:
+ *
+ *	void	munge_XXXX( const void *regs, void *uu_args);
+ *
+ * The name of the function encodes the number and type of the parameters,
+ * as follows:
+ *
+ *	w = a 32-bit value such as an int or a 32-bit ptr, that does not
+ *	    require sign extension.  These are handled by zeroing a word
+ *          of output, and copying a word from input to output.
+ *
+ *	s = a 32-bit value such as a long, which must be sign-extended to
+ *	    a 64-bit long-long in the uu_args.  These are handled by
+ *	    loading a word of input and sign extending it to a double,
+ *          and storing two words of output.
+ *
+ *	l = a 64-bit long-long.  These are handled by copying two words
+ *          of input to the output.
+ *
+ * For example, "munge_wls" takes a word, a long-long, and a word.  This
+ * takes four words in the uu_arg[] area: the first word is in one, the
+ * long-long takes two, and the final word is in the fourth.  We store six
+ * words: the low word is left in place, followed by a 0, followed by the
+ * two words of the long-long, followed by the low word and the sign extended
+ * high word of the preceeding low word.
+ *
+ * Because this is an in-place modification, we actually start at the end
+ * of uu_arg[] and work our way back to the beginning of the array.
+ *
+ * As you can see, we save a lot of code by collapsing mungers that are
+ * prefixes or suffixes of each other.
+ */
+#include <i386/asm.h>
+
+ENTRY(munge_w)
+	movl	8(%esp),%ecx	// get &uu_args
+	movl	$0,4(%ecx)
+	ret
+	
+ENTRY(munge_ww)
+	movl	8(%esp),%ecx	// get &uu_args
+	xorl	%edx,%edx
+	jmp	Lw2
+ENTRY(munge_www)
+	movl	8(%esp),%ecx	// get &uu_args
+	xorl	%edx,%edx
+	jmp	Lw3
+ENTRY(munge_wwww)
+	movl	8(%esp),%ecx	// get &uu_args
+	xorl	%edx,%edx
+	jmp	Lw4
+ENTRY(munge_wwwww)
+	movl	8(%esp),%ecx	// get &uu_args
+	xorl	%edx,%edx
+	jmp	Lw5
+ENTRY(munge_wwwwww)
+	movl	8(%esp),%ecx	// get &uu_args
+	xorl	%edx,%edx
+	jmp	Lw6
+ENTRY(munge_wwwwwww)
+	movl	8(%esp),%ecx	// get &uu_args
+	xorl	%edx,%edx
+	jmp	Lw7
+ENTRY(munge_wwwwwwww)
+	movl	8(%esp),%ecx	// get &uu_args
+	xorl	%edx,%edx
+	movl	28(%ecx),%eax
+	movl	%eax,56(%ecx)
+	movl	%edx,60(%ecx)
+Lw7:
+	movl	24(%ecx),%eax
+	movl	%eax,48(%ecx)
+	movl	%edx,52(%ecx)
+Lw6:
+	movl	20(%ecx),%eax
+	movl	%eax,40(%ecx)
+	movl	%edx,44(%ecx)
+Lw5:
+	movl	16(%ecx),%eax
+	movl	%eax,32(%ecx)
+	movl	%edx,36(%ecx)
+Lw4:
+	movl	12(%ecx),%eax
+	movl	%eax,24(%ecx)
+	movl	%edx,28(%ecx)
+Lw3:
+	movl	8(%ecx),%eax
+	movl	%eax,16(%ecx)
+	movl	%edx,20(%ecx)
+Lw2:
+	movl	4(%ecx),%eax
+	movl	%eax,8(%ecx)
+	movl	%edx,12(%ecx)
+	movl	%edx,4(%ecx)
+	ret
+
+
+Entry(munge_wl)			/* Costs an extra w move to do this */
+ENTRY(munge_wlw)
+	movl	8(%esp),%ecx	// get &uu_args
+	xorl	%edx,%edx
+	movl	12(%ecx),%eax
+	movl	%eax,16(%ecx)
+	movl	%edx,20(%ecx)
+	movl	8(%ecx),%eax
+	movl	%eax,12(%ecx)
+	movl	4(%ecx),%eax
+	movl	%eax,8(%ecx)
+	movl	%edx,4(%ecx)
+	ret
+
+ENTRY(munge_wwwl)
+	movl	8(%esp),%ecx	// get &uu_args
+	xorl	%edx,%edx
+	movl	12(%ecx),%eax
+	movl	%eax,24(%ecx)
+	movl	16(%ecx),%eax
+	movl	%eax,28(%ecx)
+	jmp	Lw3
+
+ENTRY(munge_wwwwl)
+	movl	8(%esp),%ecx	// get &uu_args
+	xorl	%edx,%edx
+	movl	16(%ecx),%eax
+	movl	%eax,32(%ecx)
+	movl	20(%ecx),%eax
+	movl	%eax,36(%ecx)
+	jmp	Lw4
+
+ENTRY(munge_wwwwwl)
+	movl	8(%esp),%ecx	// get &uu_args
+	xorl	%edx,%edx
+	movl	20(%ecx),%eax
+	movl	%eax,40(%ecx)
+	movl	24(%ecx),%eax
+	movl	%eax,44(%ecx)
+	jmp	Lw5
+
+ENTRY(munge_wsw)
+	movl	8(%esp),%ecx	// get &uu_args
+	movl	8(%ecx),%eax
+	movl	%eax,16(%ecx)
+	movl	$0,20(%ecx)
+	movl	4(%ecx),%eax
+	cltd
+	movl	%eax,8(%ecx)
+	movl	%edx,12(%ecx)
+	movl	$0,4(%ecx)
+	ret
+
+ENTRY(munge_wws)
+	movl	8(%esp),%ecx	// get &uu_args
+	movl	8(%ecx),%eax
+	cltd
+	movl	%eax,16(%ecx)
+	movl	%edx,20(%ecx)
+	xorl	%edx,%edx
+	jmp	Lw2
+
+ENTRY(munge_wwwsw)
+	movl	8(%esp),%ecx	// get &uu_args
+	movl	16(%ecx),%eax
+	movl	%eax,32(%ecx)
+	movl	$0,36(%ecx)
+	movl	12(%ecx),%eax
+	cltd
+	movl	%eax,24(%ecx)
+	movl	%edx,28(%ecx)
+	xorl	%edx,%edx
+	jmp	Lw3
diff --git a/bsd/dev/i386/pio.h b/bsd/dev/i386/pio.h
deleted file mode 100644
index c91422208..000000000
--- a/bsd/dev/i386/pio.h
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the License
- * may not be used to create, or enable the creation or redistribution of,
- * unlawful or unlicensed copies of an Apple operating system, or to
- * circumvent, violate, or enable the circumvention or violation of, any
- * terms of an Apple operating system software license agreement.
- * 
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- * 
- * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/*
- * HISTORY
- * 
- * Revision 1.2  1998/09/30 21:20:45  wsanchez
- * Merged in IntelMerge1 (mburg: Intel support)
- *
- * Revision 1.1.2.1  1998/09/30 18:18:50  mburg
- * Changes for Intel port
- *
- * Revision 1.1.1.1  1998/03/07 02:25:38  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.1.8.2  1996/07/31  09:46:36  paire
- * 	Merged with nmk20b7_shared (1.1.11.2 -> 1.1.11.1)
- * 	[96/06/10            paire]
- *
- * Revision 1.1.11.2  1996/06/13  12:38:25  bernadat
- * 	Do not use inline macros when MACH_ASSERT is configured.
- * 	[96/05/24            bernadat]
- * 
- * Revision 1.1.11.1  1996/05/14  13:50:23  paire
- * 	Added new linl and loutl __inline__.
- * 	Added conditional compilation for [l]{in|oub}[bwl]() __inline__.
- * 	[95/11/24            paire]
- * 
- * Revision 1.1.8.1  1994/09/23  02:00:28  ezf
- * 	change marker to not FREE
- * 	[1994/09/22  21:25:52  ezf]
- * 
- * Revision 1.1.4.5  1993/08/09  19:40:41  dswartz
- * 	Add ANSI prototypes - CR#9523
- * 	[1993/08/06  17:45:57  dswartz]
- * 
- * Revision 1.1.4.4  1993/06/11  15:17:37  jeffc
- * 	CR9176 - ANSI C violations: inb/outb macros must be changed from
- * 	({ ... }) to inline functions, with proper type definitions. Callers
- * 	must pass proper types to these functions: 386 I/O port addresses
- * 	are unsigned shorts (not pointers).
- * 	[1993/06/10  14:26:10  jeffc]
- * 
- * Revision 1.1.4.3  1993/06/07  22:09:28  jeffc
- * 	CR9176 - ANSI C violations: trailing tokens on CPP
- * 	directives, extra semicolons after decl_ ..., asm keywords
- * 	[1993/06/07  19:00:26  jeffc]
- * 
- * Revision 1.1.4.2  1993/06/04  15:28:45  jeffc
- * 	CR9176 - ANSI problems -
- * 	Added casts to get macros to take caddr_t as an I/O space address.
- * 	[1993/06/04  13:45:55  jeffc]
- * 
- * Revision 1.1  1992/09/30  02:25:51  robert
- * 	Initial revision
- * 
- * $EndLog$
- */
-/* CMU_HIST */
-/*
- * Revision 2.5  91/05/14  16:14:20  mrt
- * 	Correcting copyright
- * 
- * Revision 2.4  91/02/05  17:13:56  mrt
- * 	Changed to new Mach copyright
- * 	[91/02/01  17:37:08  mrt]
- * 
- * Revision 2.3  90/12/20  16:36:37  jeffreyh
- * 	changes for __STDC__
- * 	[90/12/07            jeffreyh]
- * 
- * Revision 2.2  90/11/26  14:48:41  rvb
- * 	Pulled from 2.5
- * 	[90/11/22  10:09:38  rvb]
- * 
- * 	[90/08/14            mg32]
- * 
- * 	Now we know how types are factor in.
- * 	Cleaned up a bunch: eliminated ({ for output and flushed unused
- * 	output variables.
- * 	[90/08/14            rvb]
- * 
- * 	This is how its done in gcc:
- * 		Created.
- * 	[90/03/26            rvb]
- * 
- */
-/* CMU_ENDHIST */
-/* 
- * Mach Operating System
- * Copyright (c) 1991,1990 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
-/* 
- */
-#ifndef I386_PIO_H
-#define I386_PIO_H
-
-typedef unsigned short i386_ioport_t;
-
-/* read a longword */
-extern unsigned long	inl(
-				i386_ioport_t	port);
-/* read a shortword */
-extern unsigned short	inw(
-				i386_ioport_t	port);
-/* read a byte */
-extern unsigned char	inb(
-				i386_ioport_t	port);
-/* write a longword */
-extern void		outl(
-				i386_ioport_t	port,
-				unsigned long	datum);
-/* write a word */
-extern void		outw(
-				i386_ioport_t	port,
-				unsigned short	datum);
-/* write a longword */
-extern void		outb(
-				i386_ioport_t	port,
-				unsigned char	datum);
-
-/* input an array of longwords */
-extern void		linl(
-				i386_ioport_t	port,
-				int		* data,
-				int		count);
-/* output an array of longwords */
-extern void		loutl(
-				i386_ioport_t	port,
-				int		* data,
-				int		count);
-
-/* input an array of words */
-extern void		linw(
-				i386_ioport_t	port,
-				int		* data,
-				int		count);
-/* output an array of words */
-extern void		loutw(
-				i386_ioport_t	port,
-				int		* data,
-				int		count);
-
-/* input an array of bytes */
-extern void		linb(
-				i386_ioport_t	port,
-				char		* data,
-				int		count);
-/* output an array of bytes */
-extern void		loutb(
-				i386_ioport_t	port,
-				char		* data,
-				int		count);
-
-extern __inline__ unsigned long	inl(
-				i386_ioport_t port)
-{
-	unsigned long datum;
-	__asm__ volatile("inl %1, %0" : "=a" (datum) : "d" (port));
-	return(datum);
-}
-
-extern __inline__ unsigned short inw(
-				i386_ioport_t port)
-{
-	unsigned short datum;
-	__asm__ volatile(".byte 0x66; inl %1, %0" : "=a" (datum) : "d" (port));
-	return(datum);
-}
-
-extern __inline__ unsigned char inb(
-				i386_ioport_t port)
-{
-	unsigned char datum;
-	__asm__ volatile("inb %1, %0" : "=a" (datum) : "d" (port));
-	return(datum);
-}
-
-extern __inline__ void outl(
-				i386_ioport_t port,
-				unsigned long datum)
-{
-	__asm__ volatile("outl %0, %1" : : "a" (datum), "d" (port));
-}
-
-extern __inline__ void outw(
-				i386_ioport_t port,
-				unsigned short datum)
-{
-	__asm__ volatile(".byte 0x66; outl %0, %1" : : "a" (datum), "d" (port));
-}
-
-extern __inline__ void outb(
-				i386_ioport_t port,
-				unsigned char datum)
-{
-	__asm__ volatile("outb %0, %1" : : "a" (datum), "d" (port));
-}
-
-#endif /* I386_PIO_H */
diff --git a/bsd/dev/i386/stubs.c b/bsd/dev/i386/stubs.c
index f4804dad7..fa75f2b89 100644
--- a/bsd/dev/i386/stubs.c
+++ b/bsd/dev/i386/stubs.c
@@ -43,12 +43,13 @@
 #include <kern/task.h>
 #include <kern/thread.h>
 #include <vm/vm_map.h>
+#include <machine/machine_routines.h>
 
 /* XXX should be elsewhere (cpeak) */
-extern int	set_bsduthreadargs(thread_t, void *, void *);
+extern struct proc *i386_current_proc(void);
 extern void	*get_bsduthreadarg(thread_t);
 extern int	*get_bsduthreadrval(thread_t);
-extern int	*get_bsduthreadlowpridelay(thread_t);
+extern void	*find_user_regs(thread_t);
 
 /* 
  * copy a null terminated string from the kernel address space into
@@ -117,56 +118,62 @@ copywithin(void *src, void *dst, size_t count)
 	return 0;
 }
 
-int
-set_bsduthreadargs(thread_t th, void * pcb, __unused void *ignored_arg)
-{
+/*
+ * This is just current_proc() from bsd/kern/bsd_stubs.c, but instead of
+ * returning kernproc in the non-vfork() case, it can return NULL.  This is
+ * needed because the system call entry point is in osfmk/i386/bsd_i386.c
+ * instead of bsd/dev/i386, and therefore cannot see some BSD thread
+ * internals.  We need to distinguish kernproc defaulting in the vfork and
+ * non-vfork cases vs. actually being the real process context.
+ */     
+struct proc *
+i386_current_proc(void)
+{       
 	struct uthread * ut;
-	struct proc *p = current_proc();
-
-	ut = get_bsdthread_info(th);
-	ut->uu_ar0 = (int *)pcb;
-
-	/*
-	 * Delayed binding of thread credential to process credential.
-	 *
-	 * XXX This doesn't really belong here, but the i386 code has a
-	 * XXX number of seemingly gratuitous structural differences that
-	 * XXX make this the most appropriate place to do the work.
-	 */
-	if (ut->uu_ucred != p->p_ucred &&
-	    (ut->uu_flag & UT_SETUID) == 0) {
-		kauth_cred_t old = ut->uu_ucred;
-		proc_lock(p);
-		ut->uu_ucred = p->p_ucred;
-		kauth_cred_ref(ut->uu_ucred);
-		proc_unlock(p);
-		if (old != NOCRED)
-			kauth_cred_rele(old);
+	struct proc *p; 
+	thread_t thr_act = current_thread();
+
+	ut = (struct uthread *)get_bsdthread_info(thr_act);
+	if (ut &&  (ut->uu_flag & UT_VFORK)) { 
+		if (ut->uu_proc) {
+			p = ut->uu_proc; 
+			if ((p->p_flag & P_INVFORK) == 0)
+				panic("returning child proc not under vfork");
+			if (p->p_vforkact != (void *)thr_act)
+				panic("returning child proc which is not cur_act");
+			return(p);
+		} else {
+			return (kernproc);
+		}
 	}
 
-	return(1);
+	/* Not in vfork - may return NULL */
+	p = (struct proc *)get_bsdtask_info(current_task());
+
+	return (p);
 }
 
 void *
 get_bsduthreadarg(thread_t th)
 {
-struct uthread *ut;
+        void	*arg_ptr;
+	struct uthread *ut;
+  
 	ut = get_bsdthread_info(th);
-	return((void *)(ut->uu_arg));
+
+	if (ml_thread_is64bit(th) == TRUE)
+	        arg_ptr = (void *)saved_state64(find_user_regs(th));
+	else
+		arg_ptr = (void *)(ut->uu_arg);
+
+	return(arg_ptr);
 }
 
 int *
 get_bsduthreadrval(thread_t th)
 {
-struct uthread *ut;
-	ut = get_bsdthread_info(th);
-	return(&ut->uu_rval[0]);
-}
+        struct uthread *ut;
 
-int *
-get_bsduthreadlowpridelay(thread_t th)
-{
-struct uthread *ut;
 	ut = get_bsdthread_info(th);
-	return(&ut->uu_lowpri_delay);
+	return(&ut->uu_rval[0]);
 }
diff --git a/bsd/dev/i386/sysctl.c b/bsd/dev/i386/sysctl.c
index e3594c22a..8c1b741c6 100644
--- a/bsd/dev/i386/sysctl.c
+++ b/bsd/dev/i386/sysctl.c
@@ -71,6 +71,20 @@ hw_cpu_features SYSCTL_HANDLER_ARGS
     return SYSCTL_OUT(req, buf, strlen(buf) + 1);
 }
 
+static int
+hw_cpu_extfeatures SYSCTL_HANDLER_ARGS
+{
+    __unused struct sysctl_oid *unused_oidp = oidp;
+    __unused void *unused_arg1 = arg1;
+    __unused int unused_arg2 = arg2; 
+    char buf[256];
+
+    buf[0] = '\0';
+    cpuid_get_extfeature_names(cpuid_extfeatures(), buf, sizeof(buf));
+
+    return SYSCTL_OUT(req, buf, strlen(buf) + 1);
+}
+
 SYSCTL_NODE(_machdep, OID_AUTO, cpu, CTLFLAG_RW, 0,
 	"CPU info");
 
@@ -86,10 +100,6 @@ SYSCTL_PROC(_machdep_cpu, OID_AUTO, model_string, CTLTYPE_STRING | CTLFLAG_RD,
 	    (void *)offsetof(i386_cpu_info_t, cpuid_model_string), -1,
 	    hw_cpu_sysctl, "A", "CPU model string");
 
-SYSCTL_PROC(_machdep_cpu, OID_AUTO, value, CTLTYPE_INT | CTLFLAG_RD, 
-	    (void *)offsetof(i386_cpu_info_t, cpuid_value), sizeof(uint32_t),
-	    hw_cpu_sysctl, "I", "CPU value");
-
 SYSCTL_PROC(_machdep_cpu, OID_AUTO, family, CTLTYPE_INT | CTLFLAG_RD, 
 	    (void *)offsetof(i386_cpu_info_t, cpuid_family), sizeof(uint8_t),
 	    hw_cpu_sysctl, "I", "CPU family");
@@ -110,10 +120,14 @@ SYSCTL_PROC(_machdep_cpu, OID_AUTO, stepping, CTLTYPE_INT | CTLFLAG_RD,
 	    (void *)offsetof(i386_cpu_info_t, cpuid_stepping), sizeof(uint8_t),
 	    hw_cpu_sysctl, "I", "CPU stepping");
 
-SYSCTL_PROC(_machdep_cpu, OID_AUTO, feature_bits, CTLTYPE_INT | CTLFLAG_RD, 
-	    (void *)offsetof(i386_cpu_info_t, cpuid_features), sizeof(uint32_t),
+SYSCTL_PROC(_machdep_cpu, OID_AUTO, feature_bits, CTLTYPE_QUAD | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_features), sizeof(uint64_t),
 	    hw_cpu_sysctl, "I", "CPU features");
 
+SYSCTL_PROC(_machdep_cpu, OID_AUTO, extfeature_bits, CTLTYPE_QUAD | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_extfeatures), sizeof(uint64_t),
+	    hw_cpu_sysctl, "I", "CPU extended features");
+
 SYSCTL_PROC(_machdep_cpu, OID_AUTO, signature, CTLTYPE_INT | CTLFLAG_RD, 
 	    (void *)offsetof(i386_cpu_info_t, cpuid_signature), sizeof(uint32_t),
 	    hw_cpu_sysctl, "I", "CPU signature");
@@ -126,6 +140,22 @@ SYSCTL_PROC(_machdep_cpu, OID_AUTO, features, CTLTYPE_STRING | CTLFLAG_RD,
 	    0, 0,
 	    hw_cpu_features, "A", "CPU feature names");
 
+SYSCTL_PROC(_machdep_cpu, OID_AUTO, extfeatures, CTLTYPE_STRING | CTLFLAG_RD, 
+	    0, 0,
+	    hw_cpu_extfeatures, "A", "CPU extended feature names");
+
+SYSCTL_PROC(_machdep_cpu, OID_AUTO, logical_per_package,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_logical_per_package),
+	    sizeof(uint32_t),
+	    hw_cpu_sysctl, "I", "CPU logical cpus per package");
+
+SYSCTL_PROC(_machdep_cpu, OID_AUTO, cores_per_package,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_cores_per_package),
+	    sizeof(uint32_t),
+	    hw_cpu_sysctl, "I", "CPU cores per package");
+
 
 struct sysctl_oid *machdep_sysctl_list[] =
 {
@@ -133,16 +163,19 @@ struct sysctl_oid *machdep_sysctl_list[] =
     &sysctl__machdep_cpu_vendor,
     &sysctl__machdep_cpu_brand_string,
     &sysctl__machdep_cpu_model_string,
-    &sysctl__machdep_cpu_value,
     &sysctl__machdep_cpu_family,
     &sysctl__machdep_cpu_model,
     &sysctl__machdep_cpu_extmodel,
     &sysctl__machdep_cpu_extfamily,
     &sysctl__machdep_cpu_feature_bits,
+    &sysctl__machdep_cpu_extfeature_bits,
     &sysctl__machdep_cpu_stepping,
     &sysctl__machdep_cpu_signature,
     &sysctl__machdep_cpu_brand,
     &sysctl__machdep_cpu_features,
+    &sysctl__machdep_cpu_extfeatures,
+    &sysctl__machdep_cpu_logical_per_package,
+    &sysctl__machdep_cpu_cores_per_package,
     (struct sysctl_oid *) 0
 };
 
diff --git a/bsd/dev/i386/systemcalls.c b/bsd/dev/i386/systemcalls.c
new file mode 100644
index 000000000..9f5e9d914
--- /dev/null
+++ b/bsd/dev/i386/systemcalls.c
@@ -0,0 +1,616 @@
+/*
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <kern/task.h>
+#include <kern/thread.h>
+#include <kern/assert.h>
+#include <kern/clock.h>
+#include <kern/locks.h>
+#include <kern/sched_prim.h>
+#include <mach/machine/thread_status.h>
+#include <mach/thread_act.h>
+
+#include <sys/kernel.h>
+#include <sys/vm.h>
+#include <sys/proc_internal.h>
+#include <sys/syscall.h>
+#include <sys/systm.h>
+#include <sys/user.h>
+#include <sys/errno.h>
+#include <sys/ktrace.h>
+#include <sys/kdebug.h>
+#include <sys/sysent.h>
+#include <sys/sysproto.h>
+#include <sys/kauth.h>
+#include <sys/systm.h>
+
+#include <bsm/audit_kernel.h>
+
+#include <i386/seg.h>
+#include <i386/machine_routines.h>
+#include <mach/i386/syscall_sw.h>
+
+extern void unix_syscall(x86_saved_state_t *);
+extern void unix_syscall64(x86_saved_state_t *);
+extern void unix_syscall_return(int);
+extern void *find_user_regs(thread_t);
+extern void IOSleep(int);
+extern void exit_funnel_section(void);
+
+extern void Debugger(const char      * message);
+
+/*
+ * Function:	unix_syscall
+ *
+ * Inputs:	regs	- pointer to i386 save area
+ *
+ * Outputs:	none
+ */
+void
+unix_syscall(x86_saved_state_t *state)
+{
+	thread_t	thread;
+	void		*vt; 
+	unsigned short	code;
+	struct sysent	*callp;
+	int		nargs;
+	int		error;
+	int		funnel_type;
+	vm_offset_t	params;
+	struct proc	*p;
+	struct uthread	*uthread;
+	unsigned int cancel_enable;
+	x86_saved_state32_t	*regs;
+
+	assert(is_saved_state32(state));
+	regs = saved_state32(state);
+
+	if (regs->eax == 0x800)
+		thread_exception_return();
+
+	thread = current_thread();
+	uthread = get_bsdthread_info(thread);
+
+	/* Get the approriate proc; may be different from task's for vfork() */
+	if (!(uthread->uu_flag & UT_VFORK))
+		p = (struct proc *)get_bsdtask_info(current_task());
+	else 
+		p = current_proc();
+
+	/* Verify that we are not being called from a task without a proc */
+	if (p == NULL) {
+		regs->eax = EPERM;
+		regs->efl |= EFL_CF;
+		task_terminate_internal(current_task());
+		thread_exception_return();
+		/* NOTREACHED */
+	}
+
+	//printf("[scall : eax %x]",  regs->eax);
+	code = regs->eax;
+	params = (vm_offset_t) ((caddr_t)regs->uesp + sizeof (int));
+	callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
+
+	if (callp == sysent) {
+		code = fuword(params);
+		params += sizeof (int);
+		callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
+	}
+	vt = (void *)uthread->uu_arg;
+
+	nargs = callp->sy_narg * sizeof (syscall_arg_t);
+	if (nargs != 0) {
+		sy_munge_t	*mungerp;
+
+		assert(nargs <= 8);
+
+		error = copyin((user_addr_t) params, (char *) vt, nargs);
+		if (error) {
+			regs->eax = error;
+			regs->efl |= EFL_CF;
+			thread_exception_return();
+			/* NOTREACHED */
+		}
+		if (code != 180) {
+		        int *ip = (int *)vt;
+
+			KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
+					      *ip, *(ip+1), *(ip+2), *(ip+3), 0);
+		}
+		mungerp = callp->sy_arg_munge32;
+
+		/*
+		 * If non-NULL, then call the syscall argument munger to
+		 * copy in arguments (see xnu/bsd/dev/i386/munge.s); the
+		 * first argument is NULL because we are munging in place
+		 * after a copyin because the ABI currently doesn't use
+		 * registers to pass system call arguments.
+		 */
+		if (mungerp != NULL)
+			(*mungerp)(NULL, vt);
+	} else
+	        KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
+				      0, 0, 0, 0, 0);
+	/*
+	 * Delayed binding of thread credential to process credential, if we
+	 * are not running with an explicitly set thread credential.
+	 */
+	if (uthread->uu_ucred != p->p_ucred &&
+	    (uthread->uu_flag & UT_SETUID) == 0) {
+		kauth_cred_t old = uthread->uu_ucred;
+		uthread->uu_ucred = kauth_cred_proc_ref(p);
+		if (IS_VALID_CRED(old))
+			kauth_cred_unref(&old);
+	}
+
+	uthread->uu_rval[0] = 0;
+	uthread->uu_rval[1] = regs->edx;
+
+	cancel_enable = callp->sy_cancel;
+	
+	if (cancel_enable == _SYSCALL_CANCEL_NONE) {
+	        uthread->uu_flag |= UT_NOTCANCELPT;
+	} else {
+		if ((uthread->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL) {
+			if (cancel_enable == _SYSCALL_CANCEL_PRE) {
+					/* system call cancelled; return to handle cancellation */
+					regs->eax = (long long)EINTR;
+					regs->efl |= EFL_CF;
+					thread_exception_return();
+					/* NOTREACHED */
+ 			} else {
+			        thread_abort_safely(thread);
+			}
+		}
+	}
+
+	funnel_type = (callp->sy_funnel & FUNNEL_MASK);
+	if (funnel_type == KERNEL_FUNNEL)
+		thread_funnel_set(kernel_flock, TRUE);
+
+	if (KTRPOINT(p, KTR_SYSCALL))
+	        ktrsyscall(p, code, callp->sy_narg, vt);
+
+	AUDIT_SYSCALL_ENTER(code, p, uthread);
+	error = (*(callp->sy_call))((void *) p, (void *) vt, &(uthread->uu_rval[0]));
+	AUDIT_SYSCALL_EXIT(error, p, uthread);
+	
+	if (error == ERESTART) {
+		/*
+		 * Move the user's pc back to repeat the syscall:
+		 * 5 bytes for a sysenter, or 2 for an int 8x.
+		 * The SYSENTER_TF_CS covers single-stepping over a sysenter
+		 * - see debug trap handler in idt.s/idt64.s
+		 */
+		if (regs->cs == SYSENTER_CS || regs->cs == SYSENTER_TF_CS)
+			regs->eip -= 5;
+		else
+			regs->eip -= 2;
+	}
+	else if (error != EJUSTRETURN) {
+		if (error) {
+		    regs->eax = error;
+		    regs->efl |= EFL_CF;	/* carry bit */
+		} else { /* (not error) */
+		    regs->eax = uthread->uu_rval[0];
+		    regs->edx = uthread->uu_rval[1];
+		    regs->efl &= ~EFL_CF;
+		} 
+	}
+
+	if (KTRPOINT(p, KTR_SYSRET))
+	        ktrsysret(p, code, error, uthread->uu_rval[0]);
+
+	if (cancel_enable == _SYSCALL_CANCEL_NONE)
+                uthread->uu_flag &= ~UT_NOTCANCELPT;
+
+	/*
+	 * if we're holding the funnel
+	 * than drop it regardless of whether
+	 * we took it on system call entry
+	 */
+	exit_funnel_section();
+
+	if (uthread->uu_lowpri_delay) {
+	        /*
+		 * task is marked as a low priority I/O type
+		 * and the I/O we issued while in this system call
+		 * collided with normal I/O operations... we'll
+		 * delay in order to mitigate the impact of this
+		 * task on the normal operation of the system
+		 */
+		IOSleep(uthread->uu_lowpri_delay);
+	        uthread->uu_lowpri_delay = 0;
+	}
+	if (code != 180)
+	        KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
+				      error, uthread->uu_rval[0], uthread->uu_rval[1], 0, 0);
+
+	thread_exception_return();
+	/* NOTREACHED */
+}
+
+
+void
+unix_syscall64(x86_saved_state_t *state)
+{
+	thread_t	thread;
+	unsigned short	code;
+	struct sysent	*callp;
+	void		*uargp;
+	int		args_in_regs;
+	int		error;
+	int		funnel_type;
+	struct proc	*p;
+	struct uthread	*uthread;
+	unsigned int cancel_enable;
+	x86_saved_state64_t *regs;
+
+	assert(is_saved_state64(state));
+	regs = saved_state64(state);
+
+	if (regs->rax == 0x2000800)
+		thread_exception_return();
+
+	thread = current_thread();
+	uthread = get_bsdthread_info(thread);
+
+	/* Get the approriate proc; may be different from task's for vfork() */
+	if (!(uthread->uu_flag & UT_VFORK))
+		p = (struct proc *)get_bsdtask_info(current_task());
+	else 
+		p = current_proc();
+
+	/* Verify that we are not being called from a task without a proc */
+	if (p == NULL) {
+		regs->rax = EPERM;
+		regs->isf.rflags |= EFL_CF;
+		task_terminate_internal(current_task());
+		thread_exception_return();
+		/* NOTREACHED */
+	}
+	args_in_regs = 6;
+
+	code = regs->rax & SYSCALL_NUMBER_MASK;
+	callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
+	uargp = (void *)(&regs->rdi);
+
+	if (callp == sysent) {
+	        /*
+		 * indirect system call... system call number
+		 * passed as 'arg0'
+		 */
+	        code = regs->rdi;
+		callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
+		uargp = (void *)(&regs->rsi);
+		args_in_regs = 5;
+	}
+
+	if (callp->sy_narg != 0) {
+		if (code != 180) {
+		        uint64_t *ip = (uint64_t *)uargp;
+
+			KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
+					      (int)(*ip), (int)(*(ip+1)), (int)(*(ip+2)), (int)(*(ip+3)), 0);
+		}
+	        assert(callp->sy_narg <= 8);
+
+		if (callp->sy_narg > args_in_regs) {
+		        int copyin_count;
+
+		        copyin_count = (callp->sy_narg - args_in_regs) * sizeof(uint64_t);
+
+		        error = copyin((user_addr_t)(regs->isf.rsp + sizeof(user_addr_t)), (char *)&regs->v_arg6, copyin_count);
+			if (error) {
+			        regs->rax = error;
+				regs->isf.rflags |= EFL_CF;
+				thread_exception_return();
+				/* NOTREACHED */
+			}
+		}
+		/*
+		 * XXX Turn 64 bit unsafe calls into nosys()
+		 */
+		if (callp->sy_funnel & UNSAFE_64BIT) {
+		        callp = &sysent[63];
+			goto unsafe;
+		}
+
+	} else
+	        KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
+				      0, 0, 0, 0, 0);
+unsafe:
+
+	/*
+	 * Delayed binding of thread credential to process credential, if we
+	 * are not running with an explicitly set thread credential.
+	 */
+	if (uthread->uu_ucred != p->p_ucred &&
+	    (uthread->uu_flag & UT_SETUID) == 0) {
+		kauth_cred_t old = uthread->uu_ucred;
+		uthread->uu_ucred = kauth_cred_proc_ref(p);
+		if (IS_VALID_CRED(old))
+			kauth_cred_unref(&old);
+	}
+
+	uthread->uu_rval[0] = 0;
+	uthread->uu_rval[1] = 0;
+
+	cancel_enable = callp->sy_cancel;
+	
+	if (cancel_enable == _SYSCALL_CANCEL_NONE) {
+	        uthread->uu_flag |= UT_NOTCANCELPT;
+	} else {
+		if ((uthread->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL) {
+			if (cancel_enable == _SYSCALL_CANCEL_PRE) {
+					/* system call cancelled; return to handle cancellation */
+					regs->rax = EINTR;
+					regs->isf.rflags |= EFL_CF;
+					thread_exception_return();
+					/* NOTREACHED */
+ 			} else {
+			        thread_abort_safely(thread);
+			}
+		}
+	}
+
+	funnel_type = (callp->sy_funnel & FUNNEL_MASK);
+	if (funnel_type == KERNEL_FUNNEL)
+		thread_funnel_set(kernel_flock, TRUE);
+
+	if (KTRPOINT(p, KTR_SYSCALL))
+	        ktrsyscall(p, code, callp->sy_narg, uargp);
+
+	AUDIT_SYSCALL_ENTER(code, p, uthread);
+	error = (*(callp->sy_call))((void *) p, uargp, &(uthread->uu_rval[0]));
+	AUDIT_SYSCALL_EXIT(error, p, uthread);
+	
+	if (error == ERESTART) {
+		/*
+		 * all system calls come through via the syscall instruction
+		 * in 64 bit mode... its 2 bytes in length
+		 * move the user's pc back to repeat the syscall:
+		 */
+	        regs->isf.rip -= 2;
+	}
+	else if (error != EJUSTRETURN) {
+		if (error) {
+		        regs->rax = error;
+			regs->isf.rflags |= EFL_CF;	/* carry bit */
+		} else { /* (not error) */
+
+			switch (callp->sy_return_type) {
+			case _SYSCALL_RET_INT_T:
+				regs->rax = uthread->uu_rval[0];
+				regs->rdx = uthread->uu_rval[1];
+				break;
+			case _SYSCALL_RET_UINT_T:
+				regs->rax = ((u_int)uthread->uu_rval[0]);
+				regs->rdx = ((u_int)uthread->uu_rval[1]);
+				break;
+			case _SYSCALL_RET_OFF_T:
+			case _SYSCALL_RET_ADDR_T:
+			case _SYSCALL_RET_SIZE_T:
+			case _SYSCALL_RET_SSIZE_T:
+			        regs->rax = *((uint64_t *)(&uthread->uu_rval[0]));
+				regs->rdx = 0;
+				break;
+			case _SYSCALL_RET_NONE:
+				break;
+			default:
+				panic("unix_syscall: unknown return type");
+				break;
+			}
+			regs->isf.rflags &= ~EFL_CF;
+		} 
+	}
+
+	if (KTRPOINT(p, KTR_SYSRET))
+	        ktrsysret(p, code, error, uthread->uu_rval[0]);
+
+	if (cancel_enable == _SYSCALL_CANCEL_NONE)
+                uthread->uu_flag &= ~UT_NOTCANCELPT;
+
+	/*
+	 * if we're holding the funnel
+	 * than drop it regardless of whether
+	 * we took it on system call entry
+	 */
+	exit_funnel_section();
+
+	if (uthread->uu_lowpri_delay) {
+	        /*
+		 * task is marked as a low priority I/O type
+		 * and the I/O we issued while in this system call
+		 * collided with normal I/O operations... we'll
+		 * delay in order to mitigate the impact of this
+		 * task on the normal operation of the system
+		 */
+		IOSleep(uthread->uu_lowpri_delay);
+	        uthread->uu_lowpri_delay = 0;
+	}
+	if (code != 180)
+	        KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
+				      error, uthread->uu_rval[0], uthread->uu_rval[1], 0, 0);
+
+	thread_exception_return();
+	/* NOTREACHED */
+}
+
+
+void
+unix_syscall_return(int error)
+{
+	thread_t		thread;
+	struct uthread		*uthread;
+	struct proc *p;
+	unsigned short code;
+	vm_offset_t params;
+	struct sysent *callp;
+	unsigned int cancel_enable;
+
+	thread = current_thread();
+	uthread = get_bsdthread_info(thread);
+
+	p = current_proc();
+
+	if (proc_is64bit(p)) {
+	        x86_saved_state64_t *regs;
+
+		regs = saved_state64(find_user_regs(thread));
+
+	        /* reconstruct code for tracing before blasting rax */
+		code = regs->rax & SYSCALL_NUMBER_MASK;
+		callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
+
+		if (callp == sysent)
+		        /*
+			 * indirect system call... system call number
+			 * passed as 'arg0'
+			 */
+		        code = regs->rdi;
+
+		if (error == ERESTART) {
+		        /*
+			 * all system calls come through via the syscall instruction
+			 * in 64 bit mode... its 2 bytes in length
+			 * move the user's pc back to repeat the syscall:
+			 */
+		        regs->isf.rip -= 2;
+		}
+		else if (error != EJUSTRETURN) {
+		        if (error) {
+			        regs->rax = error;
+				regs->isf.rflags |= EFL_CF;	/* carry bit */
+			} else { /* (not error) */
+
+			        switch (callp->sy_return_type) {
+				case _SYSCALL_RET_INT_T:
+				        regs->rax = uthread->uu_rval[0];
+					regs->rdx = uthread->uu_rval[1];
+					break;
+				case _SYSCALL_RET_UINT_T:
+				        regs->rax = ((u_int)uthread->uu_rval[0]);
+					regs->rdx = ((u_int)uthread->uu_rval[1]);
+					break;
+				case _SYSCALL_RET_OFF_T:
+				case _SYSCALL_RET_ADDR_T:
+				case _SYSCALL_RET_SIZE_T:
+				case _SYSCALL_RET_SSIZE_T:
+				        regs->rax = *((uint64_t *)(&uthread->uu_rval[0]));
+					regs->rdx = 0;
+					break;
+				case _SYSCALL_RET_NONE:
+				        break;
+				default:
+				        panic("unix_syscall: unknown return type");
+					break;
+				}
+				regs->isf.rflags &= ~EFL_CF;
+			} 
+		}
+	} else {
+	        x86_saved_state32_t	*regs;
+
+		regs = saved_state32(find_user_regs(thread));
+
+		/* reconstruct code for tracing before blasting eax */
+		code = regs->eax;
+		callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
+
+		if (callp == sysent) {
+		        params = (vm_offset_t) ((caddr_t)regs->uesp + sizeof (int));
+		        code = fuword(params);
+		}
+		if (error == ERESTART) {
+		        regs->eip -= ((regs->cs & 0xffff) == SYSENTER_CS) ? 5 : 2;
+		}
+		else if (error != EJUSTRETURN) {
+		        if (error) {
+			        regs->eax = error;
+				regs->efl |= EFL_CF;	/* carry bit */
+			} else { /* (not error) */
+			        regs->eax = uthread->uu_rval[0];
+				regs->edx = uthread->uu_rval[1];
+				regs->efl &= ~EFL_CF;
+			} 
+		}
+	}
+	if (KTRPOINT(p, KTR_SYSRET))
+	        ktrsysret(p, code, error, uthread->uu_rval[0]);
+
+	cancel_enable = callp->sy_cancel;
+
+	if (cancel_enable == _SYSCALL_CANCEL_NONE)
+                uthread->uu_flag &= ~UT_NOTCANCELPT;
+
+	/*
+	 * if we're holding the funnel
+	 * than drop it regardless of whether
+	 * we took it on system call entry
+	 */
+	exit_funnel_section();
+
+	if (uthread->uu_lowpri_delay) {
+	        /*
+		 * task is marked as a low priority I/O type
+		 * and the I/O we issued while in this system call
+		 * collided with normal I/O operations... we'll
+		 * delay in order to mitigate the impact of this
+		 * task on the normal operation of the system
+		 */
+		IOSleep(uthread->uu_lowpri_delay);
+	        uthread->uu_lowpri_delay = 0;
+	}
+	if (code != 180)
+	        KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
+				      error, uthread->uu_rval[0], uthread->uu_rval[1], 0, 0);
+
+	thread_exception_return();
+	/* NOTREACHED */
+}
+
+void
+munge_wwwlww(
+	__unused const void	*in32,
+	void			*out64)
+{
+	uint32_t	*arg32;
+	uint64_t	*arg64;
+
+	/* we convert in place in out64 */
+	arg32 = (uint32_t *) out64;
+	arg64 = (uint64_t *) out64;
+
+	arg64[5] = arg32[6];	/* wwwlwW */
+	arg64[4] = arg32[5];	/* wwwlWw */
+	arg32[7] = arg32[4];	/* wwwLww (hi) */
+	arg32[6] = arg32[3];	/* wwwLww (lo) */
+	arg64[2] = arg32[2];	/* wwWlww */
+	arg64[1] = arg32[1];	/* wWwlww */
+	arg64[0] = arg32[0];	/* Wwwlww */
+}	
diff --git a/bsd/dev/i386/unix_signal.c b/bsd/dev/i386/unix_signal.c
index 74bf9fc3b..69e3dc720 100644
--- a/bsd/dev/i386/unix_signal.c
+++ b/bsd/dev/i386/unix_signal.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -44,28 +44,35 @@
 #include <sys/user.h>
 #include <sys/sysproto.h>
 #include <sys/sysent.h>
+#include <sys/ucontext.h>
+#include <sys/wait.h>
 #include <mach/thread_act.h>	/* for thread_abort_safely */
+#include <mach/thread_status.h>	
+#include <i386/machine_routines.h>
 
+#include <i386/eflags.h>
 #include <i386/psl.h>
 #include <i386/seg.h>
 
-#include <mach/i386/thread_status.h>
-
-extern struct i386_saved_state *get_user_regs(thread_t);
-
-extern boolean_t valid_user_segment_selectors(uint16_t cs,
-					      uint16_t ss,
-					      uint16_t ds,
-					      uint16_t es,
-					      uint16_t fs,
-					      uint16_t gs);
+#include <sys/kdebug.h>
 
 /* Forward: */
 extern boolean_t machine_exception(int, int, int, int *, int *);
+extern kern_return_t thread_getstatus(register thread_t act, int flavor,
+			thread_state_t tstate, mach_msg_type_number_t *count);
+extern kern_return_t thread_setstatus(thread_t thread, int flavor,
+			thread_state_t tstate, mach_msg_type_number_t count);
 
 /* Signal handler flavors supported */
 /* These defns should match the Libc implmn */
 #define UC_TRAD			1
+#define UC_FLAVOR		30
+
+#define	C_32_STK_ALIGN		16
+#define	C_64_STK_ALIGN		16
+#define	C_64_REDZONE_LEN	128
+#define TRUNC_DOWN32(a,c)	((((uint32_t)a)-(c)) & ((uint32_t)(-(c))))
+#define TRUNC_DOWN64(a,c)	((((uint64_t)a)-(c)) & ((uint64_t)(-(c))))
 
 /*
  * Send an interrupt to process.
@@ -77,101 +84,408 @@ extern boolean_t machine_exception(int, int, int, int *, int *);
  * pointer, and the argument pointer, it returns
  * to the user specified pc, psl.
  */
+struct sigframe32 {
+        int		  retaddr;
+	sig_t		  catcher;
+	int		  sigstyle;
+	int		  sig;
+	siginfo_t	* sinfo;
+	struct ucontext * uctx;
+};
+
+
+
 void
-sendsig(p, catcher, sig, mask, code)
- 	struct proc *p;
-	user_addr_t catcher;	/* sig_t */
-	int sig, mask;
-	u_long code;
+sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused u_long code)
 {
-	struct sigframe {
-		int			retaddr;
-		sig_t			catcher;
-		int			sigstyle;
-		int			sig;
-		int			code;
-		struct sigcontext *	scp;
-	} frame, *fp;
-	struct sigcontext		context, *scp;
+        union {
+	    struct mcontext32	mctx32;
+	    struct mcontext64	mctx64;
+	} mctx;
+	user_addr_t	ua_sp;
+	user_addr_t	ua_fp;
+	user_addr_t	ua_cr2;
+	user_addr_t	ua_sip;
+	user_addr_t 	ua_uctxp;
+	user_addr_t	ua_mctxp;
+	user_siginfo_t	sinfo64;
+
 	struct sigacts *ps = p->p_sigacts;
-	int oonstack;
+	int oonstack, flavor; 
+	void * state;
+	mach_msg_type_number_t state_count;
+	int uthsigaltstack = 0;
+	int altstack = 0;
+
 	thread_t thread = current_thread();
 	struct uthread * ut;
-	struct i386_saved_state * saved_state = get_user_regs(thread);
-	sig_t trampact;
+	int stack_size = 0;
+	int infostyle = UC_TRAD;
     
+	if (p->p_sigacts->ps_siginfo & sigmask(sig))
+		infostyle = UC_FLAVOR;
+
 	ut = get_bsdthread_info(thread);
-	oonstack = ps->ps_sigstk.ss_flags & SA_ONSTACK;
-	if ((ps->ps_flags & SAS_ALTSTACK) && !oonstack &&
-		(ps->ps_sigonstack & sigmask(sig))) {
-			scp = ((struct sigcontext *)ps->ps_sigstk.ss_sp) - 1;
-			ps->ps_sigstk.ss_flags |= SA_ONSTACK;
-	} else
-		scp = ((struct sigcontext *)saved_state->uesp) - 1;
-	fp = ((struct sigframe *)scp) - 1;
-
-	/* 
-	 * Build the argument list for the signal handler.
-	 */
 
-	trampact = (sig_t)ps->ps_trampact[sig];
-	/* Handler should call sigreturn to get out of it */
-	frame.retaddr = 0xffffffff;	
-	frame.catcher = CAST_DOWN(sig_t,catcher);	/* XXX LP64 */
-	frame.sigstyle = UC_TRAD;
-	frame.sig = sig;
-
-	if (sig == SIGILL || sig == SIGFPE) {
-		frame.code = code;
-	} else
-		frame.code = 0;
-	frame.scp = scp;
-	if (copyout((caddr_t)&frame, (user_addr_t)fp, sizeof (frame))) 
-		goto bad;
+	uthsigaltstack = p->p_lflag & P_LTHSIGSTACK;
 
+	if (uthsigaltstack != 0 )  {
+		oonstack = ut->uu_sigstk.ss_flags & SA_ONSTACK;
+		altstack = ut->uu_flag & UT_ALTSTACK;
+	} else {
+		oonstack = ps->ps_sigstk.ss_flags & SA_ONSTACK;
+		altstack = ps->ps_flags & SAS_ALTSTACK;
+	}
 	/*
-	 * Build the signal context to be used by sigreturn.
+	 * init siginfo
 	 */
-	context.sc_onstack = oonstack;
-	context.sc_mask = mask;
-	context.sc_eax = saved_state->eax;
-	context.sc_ebx = saved_state->ebx;
-	context.sc_ecx = saved_state->ecx;
-	context.sc_edx = saved_state->edx;
-	context.sc_edi = saved_state->edi;
-	context.sc_esi = saved_state->esi;
-	context.sc_ebp = saved_state->ebp;
-	context.sc_esp = saved_state->uesp;
-	context.sc_ss = saved_state->ss;
-	context.sc_eflags = saved_state->efl;
-	context.sc_eip = saved_state->eip;
-	context.sc_cs = saved_state->cs;
-	if (saved_state->efl & EFL_VM) {
-		context.sc_ds = saved_state->v86_segs.v86_ds;
-		context.sc_es = saved_state->v86_segs.v86_es;
-		context.sc_fs = saved_state->v86_segs.v86_fs;
-		context.sc_gs = saved_state->v86_segs.v86_gs;
-
-		saved_state->efl &= ~EFL_VM;
+	bzero((caddr_t)&sinfo64, sizeof(user_siginfo_t));
+	sinfo64.si_signo = sig;
+		
+	if (proc_is64bit(p)) {
+	        x86_thread_state64_t	*tstate64;
+	        struct user_ucontext64 	uctx64;
+
+	        flavor = x86_THREAD_STATE64;
+		state_count = x86_THREAD_STATE64_COUNT;
+		state = (void *)&mctx.mctx64.ss;
+		if (thread_getstatus(thread, flavor, (thread_state_t)state, &state_count) != KERN_SUCCESS)
+		        goto bad;
+
+		flavor = x86_FLOAT_STATE64;
+		state_count = x86_FLOAT_STATE64_COUNT;
+		state = (void *)&mctx.mctx64.fs;
+		if (thread_getstatus(thread, flavor, (thread_state_t)state, &state_count) != KERN_SUCCESS)
+		        goto bad;
+
+		flavor = x86_EXCEPTION_STATE64;
+		state_count = x86_EXCEPTION_STATE64_COUNT;
+		state = (void *)&mctx.mctx64.es;
+		if (thread_getstatus(thread, flavor, (thread_state_t)state, &state_count) != KERN_SUCCESS)
+		        goto bad;
+
+		tstate64 = &mctx.mctx64.ss;
+
+		if (altstack && !oonstack && (ps->ps_sigonstack & sigmask(sig))) {
+		        if (uthsigaltstack != 0) {
+			        ua_sp = ut->uu_sigstk.ss_sp;
+				stack_size = ut->uu_sigstk.ss_size;
+				ua_sp += stack_size;
+				ut->uu_sigstk.ss_flags |= SA_ONSTACK;
+			} else {
+			        ua_sp = ps->ps_sigstk.ss_sp;
+				stack_size = ps->ps_sigstk.ss_size;
+				ua_sp += stack_size;
+				ps->ps_sigstk.ss_flags |= SA_ONSTACK;
+			}
+		} else
+		        ua_sp = tstate64->rsp;
+		ua_cr2 = mctx.mctx64.es.faultvaddr;
+
+		/* The x86_64 ABI defines a 128-byte red zone. */
+		ua_sp -= C_64_REDZONE_LEN;
+
+		ua_sp -= sizeof (struct user_ucontext64);
+		ua_uctxp = ua_sp;			 // someone tramples the first word!
+
+		ua_sp -= sizeof (user_siginfo_t);
+		ua_sip = ua_sp;
+
+	        ua_sp -= sizeof (struct mcontext64);
+		ua_mctxp = ua_sp;
+
+		/*
+		 * Align the frame and stack pointers to 16 bytes for SSE.
+		 * (Note that we use 'ua_fp' as the base of the stack going forward)
+		 */
+		ua_fp = TRUNC_DOWN64(ua_sp, C_64_STK_ALIGN);
+
+		/*
+		 * But we need to account for the return address so the alignment is
+		 * truly "correct" at _sigtramp
+		 */
+		ua_fp -= sizeof(user_addr_t);
+
+		/*
+		 * Build the signal context to be used by sigreturn.
+		 */
+		bzero(&uctx64, sizeof(uctx64));
+
+		uctx64.uc_onstack = oonstack;
+		uctx64.uc_sigmask = mask;
+		uctx64.uc_stack.ss_sp = ua_fp;
+		uctx64.uc_stack.ss_size = stack_size;
+
+		if (oonstack)
+		        uctx64.uc_stack.ss_flags |= SS_ONSTACK;	
+		uctx64.uc_link = 0;
+
+		uctx64.uc_mcsize = sizeof(struct mcontext64);
+		uctx64.uc_mcontext64 = ua_mctxp;
+		
+		if (copyout((caddr_t)&uctx64, ua_uctxp, sizeof (uctx64))) 
+		        goto bad;
+
+	        if (copyout((caddr_t)&mctx.mctx64, ua_mctxp, sizeof (struct mcontext64))) 
+		        goto bad;
+
+		sinfo64.pad[0]  = tstate64->rsp;
+		sinfo64.si_addr = tstate64->rip;
+
+		tstate64->rip = ps->ps_trampact[sig];
+		tstate64->rsp = ua_fp;
+		tstate64->rflags = get_eflags_exportmask();
+		/*
+		 * JOE - might not need to set these
+		 */
+		tstate64->cs = USER64_CS;
+		tstate64->fs = NULL_SEG;
+		tstate64->gs = USER_CTHREAD;
+
+		/* 
+		 * Build the argument list for the signal handler.
+		 * Handler should call sigreturn to get out of it
+		 */
+		tstate64->rdi = ua_catcher;
+		tstate64->rsi = infostyle;
+		tstate64->rdx = sig;
+		tstate64->rcx = ua_sip;
+		tstate64->r8  = ua_uctxp;
+
 	} else {
-		context.sc_ds = saved_state->ds;
-		context.sc_es = saved_state->es;
-		context.sc_fs = saved_state->fs;
-		context.sc_gs = saved_state->gs;
+	        x86_thread_state32_t	*tstate32;
+	        struct ucontext 	uctx32;
+		struct sigframe32	frame32;
+
+	        flavor = x86_THREAD_STATE32;
+		state_count = x86_THREAD_STATE32_COUNT;
+		state = (void *)&mctx.mctx32.ss;
+		if (thread_getstatus(thread, flavor, (thread_state_t)state, &state_count) != KERN_SUCCESS)
+		        goto bad;
+
+		flavor = x86_FLOAT_STATE32;
+		state_count = x86_FLOAT_STATE32_COUNT;
+		state = (void *)&mctx.mctx32.fs;
+		if (thread_getstatus(thread, flavor, (thread_state_t)state, &state_count) != KERN_SUCCESS)
+		        goto bad;
+
+		flavor = x86_EXCEPTION_STATE32;
+		state_count = x86_EXCEPTION_STATE32_COUNT;
+		state = (void *)&mctx.mctx32.es;
+		if (thread_getstatus(thread, flavor, (thread_state_t)state, &state_count) != KERN_SUCCESS)
+		        goto bad;
+
+		tstate32 = &mctx.mctx32.ss;
+
+		if (altstack && !oonstack && (ps->ps_sigonstack & sigmask(sig))) {
+		        if (uthsigaltstack != 0) {
+			        ua_sp = ut->uu_sigstk.ss_sp;
+				stack_size = ut->uu_sigstk.ss_size;
+				ua_sp += stack_size;
+				ut->uu_sigstk.ss_flags |= SA_ONSTACK;
+			} else {
+			        ua_sp = ps->ps_sigstk.ss_sp;
+				stack_size = ps->ps_sigstk.ss_size;
+				ua_sp += stack_size;
+				ps->ps_sigstk.ss_flags |= SA_ONSTACK;
+			}
+		} else
+		        ua_sp = tstate32->esp;
+		ua_cr2 = mctx.mctx32.es.faultvaddr;
+
+		ua_sp -= sizeof (struct ucontext);
+		ua_uctxp = ua_sp;			 // someone tramples the first word!
+
+		ua_sp -= sizeof (siginfo_t);
+		ua_sip = ua_sp;
+
+	        ua_sp -= sizeof (struct mcontext32);
+		ua_mctxp = ua_sp;
+
+		ua_sp -= sizeof (struct sigframe32);
+		ua_fp = ua_sp;
+
+		/*
+		 * Align the frame and stack pointers to 16 bytes for SSE.
+		 * (Note that we use 'fp' as the base of the stack going forward)
+		 */
+		ua_fp = TRUNC_DOWN32(ua_fp, C_32_STK_ALIGN);
+
+		/*
+		 * But we need to account for the return address so the alignment is
+		 * truly "correct" at _sigtramp
+		 */
+		ua_fp -= sizeof(frame32.retaddr);
+
+		/* 
+		 * Build the argument list for the signal handler.
+		 * Handler should call sigreturn to get out of it
+		 */
+		frame32.retaddr = -1;	
+		frame32.sigstyle = infostyle;
+		frame32.sig = sig;
+		frame32.catcher = CAST_DOWN(sig_t, ua_catcher);
+		frame32.sinfo = CAST_DOWN(siginfo_t *, ua_sip);
+		frame32.uctx = CAST_DOWN(struct ucontext *, ua_uctxp);
+
+		if (copyout((caddr_t)&frame32, ua_fp, sizeof (frame32))) 
+		        goto bad;
+
+		/*
+		 * Build the signal context to be used by sigreturn.
+		 */
+		bzero(&uctx32, sizeof(uctx32));
+
+		uctx32.uc_onstack = oonstack;
+		uctx32.uc_sigmask = mask;
+		uctx32.uc_stack.ss_sp = CAST_DOWN(char *, ua_fp);
+		uctx32.uc_stack.ss_size = stack_size;
+
+		if (oonstack)
+		        uctx32.uc_stack.ss_flags |= SS_ONSTACK;	
+		uctx32.uc_link = 0;
+
+		uctx32.uc_mcsize = sizeof(struct mcontext32);
+
+		uctx32.uc_mcontext = CAST_DOWN(struct mcontext *, ua_mctxp);
+		
+		if (copyout((caddr_t)&uctx32, ua_uctxp, sizeof (uctx32))) 
+		        goto bad;
+
+		if (copyout((caddr_t)&mctx.mctx32, ua_mctxp, sizeof (struct mcontext32))) 
+		        goto bad;
+
+		sinfo64.pad[0]  = tstate32->esp;
+		sinfo64.si_addr = tstate32->eip;
+	}
+
+	switch (sig) {
+		case SIGCHLD:
+			sinfo64.si_pid = p->si_pid;
+			p->si_pid =0;
+			sinfo64.si_status = p->si_status;
+			p->si_status = 0;
+			sinfo64.si_uid = p->si_uid;
+			p->si_uid =0;
+			sinfo64.si_code = p->si_code;
+			p->si_code = 0;
+			if (sinfo64.si_code == CLD_EXITED) {
+				if (WIFEXITED(sinfo64.si_status)) 
+					sinfo64.si_code = CLD_EXITED;
+				else if (WIFSIGNALED(sinfo64.si_status)) {
+					if (WCOREDUMP(sinfo64.si_status))
+						sinfo64.si_code = CLD_DUMPED;
+					else	
+						sinfo64.si_code = CLD_KILLED;
+				}
+			}
+			break;
+		case SIGILL:
+			switch (ut->uu_code) {
+				case EXC_I386_INVOP:
+					sinfo64.si_code = ILL_ILLOPC;
+					break;
+				case EXC_I386_GPFLT:
+					sinfo64.si_code = ILL_PRVOPC;
+					break;
+				default:
+					printf("unknown SIGILL code %d\n", ut->uu_code);
+					sinfo64.si_code = ILL_NOOP;
+			}
+			break;
+		case SIGFPE:
+#define FP_IE 0 /* Invalid operation */
+#define FP_DE 1 /* Denormalized operand */
+#define FP_ZE 2 /* Zero divide */
+#define FP_OE 3 /* overflow */
+#define FP_UE 4 /* underflow */
+#define FP_PE 5 /* precision */
+			if (ut->uu_subcode & (1 << FP_ZE)) {
+				sinfo64.si_code = FPE_FLTDIV;
+			} else if (ut->uu_subcode & (1 << FP_OE)) {
+				sinfo64.si_code = FPE_FLTOVF;
+			} else if (ut->uu_subcode & (1 << FP_UE)) {
+				sinfo64.si_code = FPE_FLTUND;
+			} else if (ut->uu_subcode & (1 << FP_PE)) {
+				sinfo64.si_code = FPE_FLTRES;
+			} else if (ut->uu_subcode & (1 << FP_IE)) {
+				sinfo64.si_code = FPE_FLTINV;
+			} else {
+				printf("unknown SIGFPE code %d, subcode %x\n",
+				       ut->uu_code, ut->uu_subcode);
+				sinfo64.si_code = FPE_NOOP;
+			}
+			break;
+		case SIGBUS:
+			sinfo64.si_code = BUS_ADRERR;
+			sinfo64.si_addr = ua_cr2;
+			break;
+		case SIGTRAP:
+			sinfo64.si_code = TRAP_BRKPT;
+			break;
+		case SIGSEGV:
+		        sinfo64.si_addr = ua_cr2;
+
+			switch (ut->uu_code) {
+				case KERN_PROTECTION_FAILURE:
+					sinfo64.si_code = SEGV_ACCERR;
+					break;
+				case KERN_INVALID_ADDRESS:
+					sinfo64.si_code = SEGV_MAPERR;
+					break;
+				default:
+					printf("unknown SIGSEGV code %d\n", ut->uu_code);
+					sinfo64.si_code = FPE_NOOP;
+			}
+				break;
+		default:
+			break;
 	}
-	if (copyout((caddr_t)&context, (user_addr_t)scp, sizeof (context))) 
-		goto bad;
+	if (proc_is64bit(p)) {
+	        if (copyout((caddr_t)&sinfo64, ua_sip, sizeof (sinfo64))) 
+		        goto bad;
+
+		flavor = x86_THREAD_STATE64;
+		state_count = x86_THREAD_STATE64_COUNT;
+		state = (void *)&mctx.mctx64.ss;
+	} else {
+	        x86_thread_state32_t	*tstate32;
+		siginfo_t		sinfo32;
 
-	saved_state->eip = (unsigned int)trampact;
-	saved_state->cs = USER_CS;
+		bzero((caddr_t)&sinfo32, sizeof(siginfo_t));
 
-	saved_state->uesp = (unsigned int)fp;
-	saved_state->ss = USER_DS;
+		sinfo32.si_signo  = sinfo64.si_signo;
+		sinfo32.si_code   = sinfo64.si_code;
+		sinfo32.si_pid    = sinfo64.si_pid;
+		sinfo32.si_uid    = sinfo64.si_uid;
+		sinfo32.si_status = sinfo64.si_status;
+		sinfo32.si_addr   = CAST_DOWN(void *, sinfo64.si_addr);
+		sinfo32.pad[0]    = sinfo64.pad[0];
+
+	        if (copyout((caddr_t)&sinfo32, ua_sip, sizeof (sinfo32))) 
+		        goto bad;
+	
+		tstate32 = &mctx.mctx32.ss;
+		tstate32->eip = CAST_DOWN(unsigned int, ps->ps_trampact[sig]);
+		tstate32->esp = CAST_DOWN(unsigned int, ua_fp);
+		
+		tstate32->eflags = get_eflags_exportmask();
+
+		tstate32->cs = USER_CS;
+		tstate32->ss = USER_DS;
+		tstate32->ds = USER_DS;
+		tstate32->es = USER_DS;
+		tstate32->fs = NULL_SEG;
+		tstate32->gs = USER_CTHREAD;
+
+		flavor = x86_THREAD_STATE32;
+		state_count = x86_THREAD_STATE32_COUNT;
+		state = (void *)tstate32;
+	}
+	if (thread_setstatus(thread, flavor, (thread_state_t)state, state_count) != KERN_SUCCESS)
+	        goto bad;
+	ml_fp_setvalid(FALSE);
 
-	saved_state->ds = USER_DS;
-	saved_state->es = USER_DS;
-	saved_state->fs = NULL_SEG;
-	saved_state->gs = USER_CTHREAD;
 	return;
 
 bad:
@@ -195,92 +509,102 @@ bad:
  * psl to gain improper priviledges or to cause
  * a machine fault.
  */
-/* ARGSUSED */
+
 int
 sigreturn(
 	struct proc *p,
 	struct sigreturn_args *uap,
 	__unused int *retval)
 {
-    struct sigcontext		context;
-    thread_t			thread = current_thread();
-	int error;
-    struct i386_saved_state*	saved_state = (struct i386_saved_state*)
-							get_user_regs(thread);	
+        union {
+	    struct mcontext32	mctx32;
+	    struct mcontext64	mctx64;
+	} mctx;
+	thread_t thread = current_thread();
 	struct uthread * ut;
+	int	error;
+	int	uthsigaltstack = 0;
+	int	onstack = 0;
 
+	mach_msg_type_number_t ts_count;
+	unsigned int           ts_flavor;
+	void		    *  ts;
+	mach_msg_type_number_t fs_count;
+	unsigned int           fs_flavor;
+	void		    *  fs;
 
-    
-	if (saved_state == NULL) 
-		return EINVAL;
+	ut = (struct uthread *)get_bsdthread_info(thread);
+	uthsigaltstack = p->p_lflag & P_LTHSIGSTACK;
 
-    if ((error = copyin(CAST_USER_ADDR_T(uap->sigcntxp), (void *)&context, 
-				sizeof (context)))) 
-		return(error);
+	if (proc_is64bit(p)) {
+	        struct user_ucontext64	uctx64;
 
-	/*
-	 * Validate segment selectors.
-	 * Bad values would result in kernel exception at context switch
-	 * back to user mode. If other state is invalid an exception will
-	 * occur in user context.
-	 */
-	if (!valid_user_segment_selectors(context.sc_cs,
-					  context.sc_ss,
-					  context.sc_ds,
-					  context.sc_es,
-					  context.sc_fs,
-					  context.sc_gs)) {
-		return EINVAL;
-	}
+	        if ((error = copyin(uap->uctx, (void *)&uctx64, sizeof (uctx64))))
+		        return(error);
 
-	ut = (struct uthread *)get_bsdthread_info(thread);
+		if ((error = copyin(uctx64.uc_mcontext64, (void *)&mctx.mctx64, sizeof (struct mcontext64))))
+		        return(error);
 
-	if (context.sc_onstack & 01)
-		p->p_sigacts->ps_sigstk.ss_flags |= SA_ONSTACK;
-	else
-		p->p_sigacts->ps_sigstk.ss_flags &= ~SA_ONSTACK;
+		onstack = uctx64.uc_onstack & 01;
+		ut->uu_sigmask = uctx64.uc_sigmask & ~sigcantmask;
 
-	ut->uu_sigmask = context.sc_mask &~ sigcantmask;
-	if(ut->uu_siglist & ~ut->uu_sigmask)
-		signal_setast(thread);
+	        ts_flavor = x86_THREAD_STATE64;
+		ts_count  = x86_THREAD_STATE64_COUNT;
+		ts = (void *)&mctx.mctx64.ss;
+
+		fs_flavor = x86_FLOAT_STATE64;
+		fs_count  = x86_FLOAT_STATE64_COUNT;
+		fs = (void *)&mctx.mctx64.fs;
+
+      } else {
+	        struct ucontext		uctx32;
+
+	        if ((error = copyin(uap->uctx, (void *)&uctx32, sizeof (uctx32)))) 
+		        return(error);
+
+		if ((error = copyin(CAST_USER_ADDR_T(uctx32.uc_mcontext), (void *)&mctx.mctx32, sizeof (struct mcontext32)))) 
+		        return(error);
 
-	saved_state->eax = context.sc_eax;
-	saved_state->ebx = context.sc_ebx;
-	saved_state->ecx = context.sc_ecx;
-	saved_state->edx = context.sc_edx;
-	saved_state->edi = context.sc_edi;
-	saved_state->esi = context.sc_esi;
-	saved_state->ebp = context.sc_ebp;
-	saved_state->uesp = context.sc_esp;
-	saved_state->ss = context.sc_ss;
-	saved_state->efl = context.sc_eflags;
-	saved_state->efl &= ~EFL_USERCLR;
-	saved_state->efl |= EFL_USERSET;
-	saved_state->eip = context.sc_eip;
-	saved_state->cs = context.sc_cs;
-
-	if (context.sc_eflags & EFL_VM) {
-		saved_state->ds = NULL_SEG;
-    		saved_state->es = NULL_SEG;
-    		saved_state->fs = NULL_SEG;
-    		saved_state->gs = NULL_SEG;
-		saved_state->v86_segs.v86_ds = context.sc_ds;
-		saved_state->v86_segs.v86_es = context.sc_es;
-		saved_state->v86_segs.v86_fs = context.sc_fs;
-		saved_state->v86_segs.v86_gs = context.sc_gs;
-
-		saved_state->efl |= EFL_VM;
+		onstack = uctx32.uc_onstack & 01;
+		ut->uu_sigmask = uctx32.uc_sigmask & ~sigcantmask;
+
+	        ts_flavor = x86_THREAD_STATE32;
+		ts_count  = x86_THREAD_STATE32_COUNT;
+		ts = (void *)&mctx.mctx32.ss;
+
+		fs_flavor = x86_FLOAT_STATE32;
+		fs_count  = x86_FLOAT_STATE32_COUNT;
+		fs = (void *)&mctx.mctx32.fs;
 	}
-	else {
-		saved_state->ds = context.sc_ds;
-		saved_state->es = context.sc_es;
-		saved_state->fs = context.sc_fs;
-		saved_state->gs = context.sc_gs;
+	if (onstack) {
+		if (uthsigaltstack != 0)
+			ut->uu_sigstk.ss_flags |= SA_ONSTACK;
+		else
+			p->p_sigacts->ps_sigstk.ss_flags |= SA_ONSTACK;
+	} else {
+		if (uthsigaltstack != 0)
+			ut->uu_sigstk.ss_flags &= ~SA_ONSTACK;
+		else
+			p->p_sigacts->ps_sigstk.ss_flags &= ~SA_ONSTACK;
 	}
+	if (ut->uu_siglist & ~ut->uu_sigmask)
+		signal_setast(thread);
+
+	/*
+	 * thread_set_state() does all the needed checks for the passed in content
+	 */
+	if (thread_setstatus(thread, ts_flavor, ts, ts_count) != KERN_SUCCESS)
+	        return(EINVAL);
+
+	ml_fp_setvalid(TRUE);
+
+	if (thread_setstatus(thread, fs_flavor, fs, fs_count)  != KERN_SUCCESS)
+	        return(EINVAL);
 
 	return (EJUSTRETURN);
 }
 
+
 /*
  * machine_exception() performs MD translation
  * of a mach exception to a unix signal and code.
diff --git a/bsd/dev/ppc/kern_machdep.c b/bsd/dev/ppc/kern_machdep.c
index 6aeffbe20..833069e9c 100644
--- a/bsd/dev/ppc/kern_machdep.c
+++ b/bsd/dev/ppc/kern_machdep.c
@@ -41,8 +41,7 @@
 #include	<mach/boolean.h>
 #include	<mach/vm_param.h>
 #include	<kern/cpu_number.h>
-
-int grade_binary(cpu_type_t exectype, cpu_subtype_t execsubtype);
+#include	<machine/exec.h>
 
 /*
  * Routine: grade_binary()
@@ -227,8 +226,6 @@ grade_binary(cpu_type_t exectype, cpu_subtype_t execsubtype)
 	/* NOTREACHED */
 }
 
-extern vm_map_offset_t kvtophys64(vm_map_offset_t);
-
 boolean_t
 kernacc(
     off_t 	start,
@@ -242,7 +239,7 @@ kernacc(
 	end = start + len;
 	
 	while (base < end) {
-		if(kvtophys64((vm_map_offset_t)base) == (vm_map_offset_t)0)
+		if(kvtophys((vm_offset_t)base) == NULL)
 			return(FALSE);
 		base += page_size;
 	}   
diff --git a/bsd/dev/ppc/mem.c b/bsd/dev/ppc/mem.c
index c1a56f757..b4d2a860c 100644
--- a/bsd/dev/ppc/mem.c
+++ b/bsd/dev/ppc/mem.c
@@ -93,6 +93,7 @@ static caddr_t devzerobuf;
 
 extern pmap_t kernel_pmap;
 extern boolean_t kernacc(off_t, size_t );
+extern int setup_kmem;
 
 int mmread(dev_t dev, struct uio *uio);
 int mmrw(dev_t dev, struct uio *uio, enum uio_rw rw);
@@ -118,9 +119,14 @@ mmwrite(dev, uio)
 }
 
 int
-mmioctl(__unused dev_t dev, u_long cmd, __unused caddr_t data, 
+mmioctl(dev_t dev, u_long cmd, __unused caddr_t data, 
 		__unused int flag, __unused struct proc *p)
 {
+	int minnum = minor(dev);
+
+	if ((setup_kmem == 0) && ((minnum == 0) || (minnum == 1)))
+		return(EINVAL);
+
 	switch (cmd) {
 	case FIONBIO:
 	case FIOASYNC:
@@ -149,6 +155,7 @@ mmrw(dev, uio, rw)
 	int error = 0;
 	vm_offset_t	where;
 
+
 	while (uio_resid(uio) > 0 && error == 0) {
 		if (uio_iov_len(uio) == 0) {
 			uio_next_iov(uio);
@@ -161,6 +168,8 @@ mmrw(dev, uio, rw)
 
 /* minor device 0 is physical memory */
 		case 0:
+			if (setup_kmem == 0)
+				return(ENODEV);
 			vll = trunc_page_64(uio->uio_offset);
 			if(((vll >> 31) == 1) || vll >= ((dgWork.dgFlags & enaDiagDM) ? mem_actual : max_mem))
 				goto fault;
@@ -196,6 +205,8 @@ mmrw(dev, uio, rw)
 
 		/* minor device 1 is kernel memory */
 		case 1:
+			if (setup_kmem == 0)
+				return(ENODEV);
 			/* Do some sanity checking */
 			if (((addr64_t)uio->uio_offset > vm_last_addr) ||
 				((addr64_t)uio->uio_offset < VM_MIN_KERNEL_ADDRESS))
diff --git a/bsd/dev/ppc/munge.s b/bsd/dev/ppc/munge.s
index 53a11d7ef..9759b04ad 100644
--- a/bsd/dev/ppc/munge.s
+++ b/bsd/dev/ppc/munge.s
@@ -246,6 +246,33 @@ _munge_wwwl:
         blr
 
 
+        .align  5
+        .globl  _munge_wwwlww
+_munge_wwwlww:
+        li      r0,0
+        lwz     r5,0*8+4(r3)
+        lwz     r6,1*8+4(r3)
+        lwz     r7,2*8+4(r3)
+        lwz     r8,3*8+4(r3)
+        lwz     r9,4*8+4(r3)
+        lwz     r10,5*8+4(r3)
+        lwz     r11,6*8+4(r3)
+        
+        stw     r0,0*8+0(r4)
+        stw     r5,0*8+4(r4)
+        stw     r0,1*8+0(r4)
+        stw     r6,1*8+4(r4)
+        stw     r0,2*8+0(r4)
+        stw     r7,2*8+4(r4)
+        stw     r8,3*8+0(r4)
+        stw     r9,3*8+4(r4)
+        stw     r0,4*8+0(r4)
+        stw     r10,4*8+4(r4)
+        stw     r0,5*8+0(r4)
+        stw     r11,5*8+4(r4)
+        
+        blr
+
         .align  5
         .globl  _munge_wwwwl	// 4 'w's and an l
 _munge_wwwwl:
diff --git a/bsd/dev/ppc/systemcalls.c b/bsd/dev/ppc/systemcalls.c
index 623485306..39ae16682 100644
--- a/bsd/dev/ppc/systemcalls.c
+++ b/bsd/dev/ppc/systemcalls.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -122,16 +122,11 @@ unix_syscall(struct savearea	*regs)
 	if (uthread->uu_ucred != proc->p_ucred &&
 	    (uthread->uu_flag & UT_SETUID) == 0) {
 		kauth_cred_t old = uthread->uu_ucred;
-		proc_lock(proc);
-		uthread->uu_ucred = proc->p_ucred;
-		kauth_cred_ref(uthread->uu_ucred);
-		proc_unlock(proc);
-		if (old != NOCRED)
-			kauth_cred_rele(old);
+		uthread->uu_ucred = kauth_cred_proc_ref(proc);
+		if (IS_VALID_CRED(old))
+			kauth_cred_unref(&old);
 	}
 
-	uthread->uu_ar0 = (int *)regs;
-
 	callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
 
 	if (callp->sy_narg != 0) {
@@ -329,7 +324,6 @@ unix_syscall_return(int error)
 	struct savearea				*regs;
 	unsigned short				code;
 	struct sysent				*callp;
-	int funnel_type;
 	unsigned int cancel_enable;
 
 	thread_act = current_thread();
@@ -447,90 +441,9 @@ unix_syscall_return(int error)
 	/* NOTREACHED */
 }
 
-/* 
- * Time of day and interval timer support.
- *
- * These routines provide the kernel entry points to get and set
- * the time-of-day and per-process interval timers.  Subroutines
- * here provide support for adding and subtracting timeval structures
- * and decrementing interval timers, optionally reloading the interval
- * timers when they expire.
- */
-/*  NOTE THIS implementation is for  ppc architectures only.
- *  It is infrequently called, since the commpage intercepts
- *  most calls in user mode.
- *
- * XXX Y2038 bug because of assumed return of 32 bit seconds value, and
- * XXX first parameter to clock_gettimeofday()
- */
-int
-ppc_gettimeofday(__unused struct proc *p, 
-				 register struct ppc_gettimeofday_args *uap, 
-				 register_t *retval)
-{
-	int error = 0;
-	extern lck_spin_t * tz_slock;
-
-	if (uap->tp)
-		clock_gettimeofday(&retval[0], &retval[1]);
-	
-	if (uap->tzp) {
-		struct timezone ltz;
-
-		lck_spin_lock(tz_slock);
-		ltz = tz;
-		lck_spin_unlock(tz_slock);
-		error = copyout((caddr_t)&ltz, uap->tzp, sizeof (tz));
-	}
-
-	return (error);
-}
-
 #ifdef JOE_DEBUG
 joe_debug(char *p) {
 
         printf("%s\n", p);
 }
 #endif
-
-
-/* 
- * WARNING - this is a temporary workaround for binary compatibility issues
- * with anti-piracy software that relies on patching ptrace (3928003).
- * This KPI will be removed in the system release after Tiger.
- */
-uintptr_t temp_patch_ptrace(uintptr_t new_ptrace)
-{
-	struct sysent *		callp;
-	sy_call_t *			old_ptrace;
-
-	if (new_ptrace == 0)
-		return(0);
-		
-	enter_funnel_section(kernel_flock);
-	callp = &sysent[26];
-	old_ptrace = callp->sy_call;
-	
-	/* only allow one patcher of ptrace */
-	if (old_ptrace == (sy_call_t *) ptrace) {
-		callp->sy_call = (sy_call_t *) new_ptrace;
-	}
-	else {
-		old_ptrace = NULL;
-	}
-	exit_funnel_section( );
-	
-	return((uintptr_t)old_ptrace);
-}
-
-void temp_unpatch_ptrace(void)
-{
-	struct sysent *		callp;
-		
-	enter_funnel_section(kernel_flock);
-	callp = &sysent[26];
-	callp->sy_call = (sy_call_t *) ptrace;
-	exit_funnel_section( );
-	
-	return;
-}
diff --git a/bsd/dev/ppc/unix_signal.c b/bsd/dev/ppc/unix_signal.c
index 2c2244493..f6bbfa6ac 100644
--- a/bsd/dev/ppc/unix_signal.c
+++ b/bsd/dev/ppc/unix_signal.c
@@ -202,8 +202,11 @@ sendsig(struct proc *p, user_addr_t catcher, int sig, int mask, __unused u_long
 	int stack_size = 0;
 	void * tstate;
 	int flavor;
-        int ctx32 = 1;
-
+	int ctx32 = 1;
+	int uthsigaltstack = 0;
+	int altstack = 0;
+	
+	
 	th_act = current_thread();
 	ut = get_bsdthread_info(th_act);
 
@@ -306,15 +309,31 @@ sendsig(struct proc *p, user_addr_t catcher, int sig, int mask, __unused u_long
 	}  
 
 	trampact = ps->ps_trampact[sig];
-	oonstack = ps->ps_sigstk.ss_flags & SA_ONSTACK;
+	uthsigaltstack = p->p_lflag & P_LTHSIGSTACK;
+	
+	if (uthsigaltstack != 0 )  {
+		oonstack = ut->uu_sigstk.ss_flags & SA_ONSTACK;
+		altstack = ut->uu_flag & UT_ALTSTACK;
+	} else {
+		oonstack = ps->ps_sigstk.ss_flags & SA_ONSTACK;
+		altstack = ps->ps_flags & SAS_ALTSTACK;
+	}
+	
 
 	/* figure out where our new stack lives */
-	if ((ps->ps_flags & SAS_ALTSTACK) && !oonstack &&
+	if (altstack && !oonstack &&
 		(ps->ps_sigonstack & sigmask(sig))) {
-		sp = ps->ps_sigstk.ss_sp;
-		sp += ps->ps_sigstk.ss_size;
-		stack_size = ps->ps_sigstk.ss_size;
-		ps->ps_sigstk.ss_flags |= SA_ONSTACK;
+		if (uthsigaltstack != 0) {
+			sp = ut->uu_sigstk.ss_sp;
+			sp += ut->uu_sigstk.ss_size;
+			stack_size = ut->uu_sigstk.ss_size;
+			ut->uu_sigstk.ss_flags |= SA_ONSTACK;
+		} else {
+			sp = ps->ps_sigstk.ss_sp;
+			sp += ps->ps_sigstk.ss_size;
+			stack_size = ps->ps_sigstk.ss_size;
+			ps->ps_sigstk.ss_flags |= SA_ONSTACK;
+		}
 	}
 	else {
 		if (ctx32 == 0)
@@ -638,8 +657,9 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval)
 	struct uthread * ut;
 	int vec_used = 0;
 	void *tsptr, *fptr, *vptr;
-        int infostyle = uap->infostyle;
-
+	int infostyle = uap->infostyle;
+	int uthsigaltstack = 0;
+	
 	th_act = current_thread();
 
 	ut = (struct uthread *)get_bsdthread_info(th_act);
@@ -681,11 +701,21 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval)
 	error = copyin(uctx.uc_mcontext64, mactx, uctx.uc_mcsize);
 	if (error)
 		return(error);
-	
-	if ((uctx.uc_onstack & 01))
+
+	uthsigaltstack = p->p_lflag & P_LTHSIGSTACK;
+
+
+	if (uctx.uc_onstack & 01) {
+		if (uthsigaltstack != 0)
+			ut->uu_sigstk.ss_flags |= SA_ONSTACK;
+		else
 			p->p_sigacts->ps_sigstk.ss_flags |= SA_ONSTACK;
-	else
-		p->p_sigacts->ps_sigstk.ss_flags &= ~SA_ONSTACK;
+	} else {
+		if (uthsigaltstack != 0)
+			ut->uu_sigstk.ss_flags &= ~SA_ONSTACK;
+		else
+			p->p_sigacts->ps_sigstk.ss_flags &= ~SA_ONSTACK;
+	}
 
 	ut->uu_sigmask = uctx.uc_sigmask & ~sigcantmask;
 	if (ut->uu_siglist & ~ut->uu_sigmask)
diff --git a/bsd/dev/unix_startup.c b/bsd/dev/unix_startup.c
index 34514f27d..daef72f73 100644
--- a/bsd/dev/unix_startup.c
+++ b/bsd/dev/unix_startup.c
@@ -43,6 +43,8 @@
 #include <sys/mbuf.h>
 #include <sys/systm.h>
 #include <sys/tty.h>
+#include <sys/vnode.h>
+#include <sys/sysctl.h>
 #include <dev/ppc/cons.h>
 
 extern vm_map_t mb_map;
@@ -53,20 +55,28 @@ extern u_long   tcp_recvspace;
 void            bsd_bufferinit(void);
 extern void     md_prepare_for_shutdown(int, int, char *);
 
+int		bsd_mbuf_cluster_reserve(void);
+
 /*
  * Declare these as initialized data so we can patch them.
  */
 
 #ifdef	NBUF
-int             nbuf = NBUF;
+int             max_nbuf_headers = NBUF;
 int             niobuf = NBUF / 2;
-
+int 		nbuf_hashelements = NBUF;
+int 		nbuf = NBUF;
 #else
-int             nbuf = 0;
+int             max_nbuf_headers = 0;
 int             niobuf = 0;
-
+int 		nbuf_hashelements = 0;
+int		nbuf = 0;
 #endif
 
+SYSCTL_INT (_kern, OID_AUTO, nbuf, CTLFLAG_RD, &nbuf, 0, "");
+SYSCTL_INT (_kern, OID_AUTO, maxnbuf, CTLFLAG_RW, &max_nbuf_headers, 0, "");
+
+__private_extern__ int customnbuf = 0;
 int             srv = 0;	/* Flag indicates a server boot when set */
 int             ncl = 0;
 
@@ -83,21 +93,30 @@ bsd_startupearly(void)
 	vm_size_t       size;
 	kern_return_t   ret;
 
-	if (nbuf == 0)
-		nbuf = atop(sane_size / 100);	/* Get 1% of ram, but no more than we can map */
-	if (nbuf > 8192)
-		nbuf = 8192;
-	if (nbuf < 256)
-		nbuf = 256;
+	/* clip the number of buf headers upto 16k */
+	if (max_nbuf_headers == 0)
+		max_nbuf_headers = atop(sane_size / 50);	/* Get 2% of ram, but no more than we can map */
+	if ((customnbuf == 0) && (max_nbuf_headers > 16384))
+		max_nbuf_headers = 16384;
+	if (max_nbuf_headers < 256)
+		max_nbuf_headers = 256;
+
+	/* clip the number of hash elements  to 200000 */
+	if ( (customnbuf == 0 ) && nbuf_hashelements == 0) {
+		nbuf_hashelements = atop(sane_size / 50);
+		if (nbuf_hashelements > 200000)
+			nbuf_hashelements = 200000;
+	} else
+		nbuf_hashelements = max_nbuf_headers;
 
 	if (niobuf == 0)
-		niobuf = nbuf;
+		niobuf = max_nbuf_headers;
 	if (niobuf > 4096)
 		niobuf = 4096;
 	if (niobuf < 128)
 		niobuf = 128;
 
-	size = (nbuf + niobuf) * sizeof(struct buf);
+	size = (max_nbuf_headers + niobuf) * sizeof(struct buf);
 	size = round_page(size);
 
 	ret = kmem_suballoc(kernel_map,
@@ -122,13 +141,11 @@ bsd_startupearly(void)
 	buf = (struct buf *) firstaddr;
 	bzero(buf, size);
 
-	if (sane_size > (64 * 1024 * 1024) || ncl) {
+	{
 		int             scale;
 
-		if ((nmbclusters = ncl) == 0) {
-			if ((nmbclusters = ((sane_size / 16)/MCLBYTES)) > 32768)
-				nmbclusters = 32768;
-		}
+		nmbclusters = bsd_mbuf_cluster_reserve() / MCLBYTES;
+
 		if ((scale = nmbclusters / NMBCLUSTERS) > 1) {
 			tcp_sendspace *= scale;
 			tcp_recvspace *= scale;
@@ -139,6 +156,17 @@ bsd_startupearly(void)
 				tcp_recvspace = 32 * 1024;
 		}
 	}
+
+	/*
+	 * Size vnodes based on memory 
+	 * Number vnodes  is (memsize/64k) + 1024 
+	 * This is the calculation that is used by launchd in tiger
+	 * we are clipping the max based on 16G 
+	 * ie ((16*1024*1024*1024)/(64 *1024)) + 1024 = 263168;
+	 */
+	desiredvnodes  = (sane_size/65536) + 1024;
+	if (desiredvnodes > 263168)
+		desiredvnodes = 263168;
 }
 
 void
@@ -165,3 +193,24 @@ bsd_bufferinit(void)
 	 */
 	bufinit();
 }
+
+/*
+ * this has been broken out into a separate routine that
+ * can be called from the x86 early vm initialization to
+ * determine how much lo memory to reserve on systems with
+ * DMA hardware that can't fully address all of the physical
+ * memory that is present.
+ */
+int
+bsd_mbuf_cluster_reserve(void)
+{
+	if (sane_size > (64 * 1024 * 1024) || ncl) {
+
+	        if ((nmbclusters = ncl) == 0) {
+		        if ((nmbclusters = ((sane_size / 16)/MCLBYTES)) > 32768)
+			        nmbclusters = 32768;
+		}
+	}
+
+	return (nmbclusters * MCLBYTES);
+}
diff --git a/bsd/dev/vn/vn.c b/bsd/dev/vn/vn.c
index 74f8b3404..f9a2212b9 100644
--- a/bsd/dev/vn/vn.c
+++ b/bsd/dev/vn/vn.c
@@ -1120,7 +1120,7 @@ vniocattach_file(struct vn_softc *vn,
 	int error, flags;
 
 	context.vc_proc = p;
-	context.vc_ucred = proc_ucred(p);
+	context.vc_ucred = kauth_cred_proc_ref(p);
 	
 	flags = FREAD|FWRITE;
 	if (in_kernel) {
@@ -1134,8 +1134,10 @@ vniocattach_file(struct vn_softc *vn,
 	/* vn_open gives both long- and short-term references */
 	error = vn_open(&nd, flags, 0);
 	if (error) {
-		if (error != EACCES && error != EPERM && error != EROFS)
+		if (error != EACCES && error != EPERM && error != EROFS) {
+			kauth_cred_unref(&context.vc_ucred);
 			return (error);
+		}
 		flags &= ~FWRITE;
 		if (in_kernel) {
 			NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE32, 
@@ -1147,8 +1149,10 @@ vniocattach_file(struct vn_softc *vn,
 			       vniop->vn_file, &context);
 		}
 		error = vn_open(&nd, flags, 0);
-		if (error)
+		if (error) {
+			kauth_cred_unref(&context.vc_ucred);
 			return (error);
+		}
 	}
 	if (nd.ni_vp->v_type != VREG) {
 		error = EINVAL;
@@ -1157,17 +1161,19 @@ vniocattach_file(struct vn_softc *vn,
 		error = vnode_size(nd.ni_vp, &file_size, &context);
 	}
 	if (error != 0) {
-		(void) vn_close(nd.ni_vp, flags, proc_ucred(p), p);
+		(void) vn_close(nd.ni_vp, flags, context.vc_ucred, p);
 		vnode_put(nd.ni_vp);
+		kauth_cred_unref(&context.vc_ucred);
 		return (error);
 	}
 	cred = kauth_cred_proc_ref(p);
 	nd.ni_vp->v_flag |= VNOCACHE_DATA;
 	error = setcred(nd.ni_vp, p, cred);
 	if (error) {
-		(void)vn_close(nd.ni_vp, flags, proc_ucred(p), p);
+		(void)vn_close(nd.ni_vp, flags, context.vc_ucred, p);
 		vnode_put(nd.ni_vp);
-		kauth_cred_rele(cred);
+		kauth_cred_unref(&cred);
+		kauth_cred_unref(&context.vc_ucred);
 		return(error);
 	}
 	vn->sc_secsize = DEV_BSIZE;
@@ -1187,6 +1193,7 @@ vniocattach_file(struct vn_softc *vn,
 		vn->sc_flags |= VNF_READONLY;
 	/* lose the short-term reference */
 	vnode_put(nd.ni_vp);
+	kauth_cred_unref(&context.vc_ucred);
 	return(0);
 }
 
@@ -1201,7 +1208,7 @@ vniocattach_shadow(struct vn_softc *vn, struct user_vn_ioctl *vniop,
 	off_t file_size;
 
 	context.vc_proc = p;
-	context.vc_ucred = proc_ucred(p);
+	context.vc_ucred = kauth_cred_proc_ref(p);
 	
 	flags = FREAD|FWRITE;
 	if (in_kernel) {
@@ -1216,20 +1223,23 @@ vniocattach_shadow(struct vn_softc *vn, struct user_vn_ioctl *vniop,
 	error = vn_open(&nd, flags, 0);
 	if (error) {
 		/* shadow MUST be writable! */
+		kauth_cred_unref(&context.vc_ucred);
 		return (error);
 	}
 	if (nd.ni_vp->v_type != VREG 
 	    || (error = vnode_size(nd.ni_vp, &file_size, &context))) {
-		(void)vn_close(nd.ni_vp, flags, proc_ucred(p), p);
+		(void)vn_close(nd.ni_vp, flags, context.vc_ucred, p);
 		vnode_put(nd.ni_vp);
+		kauth_cred_unref(&context.vc_ucred);
 		return (error ? error : EINVAL);
 	}
 	map = shadow_map_create(vn->sc_fsize, file_size,
 				0, vn->sc_secsize);
 	if (map == NULL) {
-		(void)vn_close(nd.ni_vp, flags, proc_ucred(p), p);
+		(void)vn_close(nd.ni_vp, flags, context.vc_ucred, p);
 		vnode_put(nd.ni_vp);
 		vn->sc_shadow_vp = NULL;
+		kauth_cred_unref(&context.vc_ucred);
 		return (ENOMEM);
 	}
 	vn->sc_shadow_vp = nd.ni_vp;
@@ -1240,6 +1250,7 @@ vniocattach_shadow(struct vn_softc *vn, struct user_vn_ioctl *vniop,
 
 	/* lose the short-term reference */
 	vnode_put(nd.ni_vp);
+	kauth_cred_unref(&context.vc_ucred);
 	return(0);
 }
 
@@ -1305,8 +1316,7 @@ vnclear(struct vn_softc *vn, struct proc * p)
 	}
 	vn->sc_flags &= ~(VNF_INITED | VNF_READONLY);
 	if (vn->sc_cred) {
-		kauth_cred_rele(vn->sc_cred);
-		vn->sc_cred = NULL;
+		kauth_cred_unref(&vn->sc_cred);
 	}
 	vn->sc_size = 0;
 	vn->sc_fsize = 0;
diff --git a/bsd/hfs/hfs.h b/bsd/hfs/hfs.h
index 55645f8fe..76b23b70f 100644
--- a/bsd/hfs/hfs.h
+++ b/bsd/hfs/hfs.h
@@ -63,10 +63,10 @@
 
 #define HFS_MAX_DEFERED_ALLOC	(1024*1024)
 
-// 32 gigs is a "big" file (i.e. one that when deleted
+// 400 mb is a "big" file (i.e. one that when deleted
 // would touch enough data that we should break it into
 // multiple separate transactions
-#define HFS_BIGFILE_SIZE (32LL * 1024LL * 1024LL * 1024LL)
+#define HFS_BIGFILE_SIZE (400LL * 1024LL * 1024LL)
 
 
 enum { kMDBSize = 512 };				/* Size of I/O transfer to read entire MDB */
@@ -254,6 +254,10 @@ typedef struct hfsmount {
 	lck_mtx_t      hfs_mutex;      /* protects access to hfsmount data */
 	void          *hfs_freezing_proc;  /* who froze the fs */
 	lck_rw_t       hfs_insync;     /* protects sync/freeze interaction */
+
+	/* Resize variables: */
+	u_int32_t		hfs_resize_filesmoved;
+	u_int32_t		hfs_resize_totalfiles;
 } hfsmount_t;
 
 typedef hfsmount_t  ExtendedVCB;
@@ -281,6 +285,7 @@ typedef hfsmount_t  ExtendedVCB;
 #define HFS_FRAGMENTED_FREESPACE  0x100
 #define HFS_NEED_JNL_RESET        0x200
 #define HFS_HAS_SPARSE_DEVICE     0x400
+#define HFS_RESIZE_IN_PROGRESS    0x800
 
 
 #define HFS_MOUNT_LOCK(hfsmp, metadata)                      \
@@ -497,6 +502,7 @@ extern void hfs_checkextendedsecurity(struct hfsmount *hfsmp);
 
 extern int  hfs_extendfs(struct hfsmount *, u_int64_t, vfs_context_t);
 extern int  hfs_truncatefs(struct hfsmount *, u_int64_t, vfs_context_t);
+extern int  hfs_resize_progress(struct hfsmount *, u_int32_t *);
 
 extern int  hfs_isallocated(struct hfsmount *, u_long, u_long);
 
diff --git a/bsd/hfs/hfs_btreeio.c b/bsd/hfs/hfs_btreeio.c
index 3c6995c84..1ae5a93b7 100644
--- a/bsd/hfs/hfs_btreeio.c
+++ b/bsd/hfs/hfs_btreeio.c
@@ -295,6 +295,8 @@ exit:
 }
 
 
+#define HFS_CLUMP_ADJ_LIMIT  (200*1024*1024)
+
 __private_extern__
 OSStatus ExtendBTreeFile(FileReference vp, FSSize minEOF, FSSize maxEOF)
 {
@@ -327,7 +329,13 @@ OSStatus ExtendBTreeFile(FileReference vp, FSSize minEOF, FSSize maxEOF)
 	}
 
 	vcb = VTOVCB(vp);
-	
+
+	/* Take past growth into account when extending the catalog file. */
+	if ((VTOC(vp)->c_fileid == kHFSCatalogFileID) &&
+	    (bytesToAdd / vcb->blockSize) < filePtr->fcbExtents[0].blockCount) {
+			bytesToAdd = filePtr->fcbExtents[0].blockCount * (UInt64)vcb->blockSize;
+			bytesToAdd = MIN(bytesToAdd, HFS_CLUMP_ADJ_LIMIT);
+	}
 	/*
 	 * The Extents B-tree can't have overflow extents. ExtendFileC will
 	 * return an error if an attempt is made to extend the Extents B-tree
diff --git a/bsd/hfs/hfs_chash.c b/bsd/hfs/hfs_chash.c
index 58a61cd72..5161e46a8 100644
--- a/bsd/hfs/hfs_chash.c
+++ b/bsd/hfs/hfs_chash.c
@@ -102,11 +102,8 @@ hfs_chashinit()
 	cnodehashtbl = hashinit(desiredvnodes, M_HFSMNT, &cnodehash);
 
 	chash_lck_grp_attr= lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(chash_lck_grp_attr);
 	chash_lck_grp  = lck_grp_alloc_init("cnode_hash", chash_lck_grp_attr);
-
 	chash_lck_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(chash_lck_attr);
 
 	lck_mtx_init(&hfs_chash_mutex, chash_lck_grp, chash_lck_attr);
 }
diff --git a/bsd/hfs/hfs_cnode.c b/bsd/hfs/hfs_cnode.c
index a3f321236..2e1ca961e 100644
--- a/bsd/hfs/hfs_cnode.c
+++ b/bsd/hfs/hfs_cnode.c
@@ -1001,7 +1001,8 @@ void
 hfs_unlock(struct cnode *cp)
 {
         vnode_t rvp = NULLVP;
-        vnode_t dvp = NULLVP;
+        vnode_t vp = NULLVP;
+	u_int32_t c_flag;
 
 	/* System files need to keep track of owner */
 	if ((cp->c_fileid < kHFSFirstUserCatalogNodeID) &&
@@ -1018,21 +1019,31 @@ hfs_unlock(struct cnode *cp)
 			}
 		}
 	}
-	if (cp->c_flag & C_NEED_DVNODE_PUT)
-	        dvp = cp->c_vp;
-
-	if (cp->c_flag & C_NEED_RVNODE_PUT)
-	        rvp = cp->c_rsrc_vp;
-
-	cp->c_flag &= ~(C_NEED_DVNODE_PUT | C_NEED_RVNODE_PUT);
+	c_flag = cp->c_flag;
+	cp->c_flag &= ~(C_NEED_DVNODE_PUT | C_NEED_RVNODE_PUT | C_NEED_DATA_SETSIZE | C_NEED_RSRC_SETSIZE);
+	if (c_flag & (C_NEED_DVNODE_PUT | C_NEED_DATA_SETSIZE)) {
+		vp = cp->c_vp;
+	}
+	if (c_flag & (C_NEED_RVNODE_PUT | C_NEED_RSRC_SETSIZE)) {
+                rvp = cp->c_rsrc_vp;
+	}
 
-	cp-> c_lockowner = NULL;
+	cp->c_lockowner = NULL;
 	lck_rw_done(&cp->c_rwlock);
 
-	if (dvp)
-	        vnode_put(dvp);
-	if (rvp)
-	        vnode_put(rvp);
+	/* Perform any vnode post processing after cnode lock is dropped. */
+	if (vp) {
+		if (c_flag & C_NEED_DATA_SETSIZE)
+			ubc_setsize(vp, 0);
+		if (c_flag & C_NEED_DVNODE_PUT)
+			vnode_put(vp);
+	}
+	if (rvp) {
+		if (c_flag & C_NEED_RSRC_SETSIZE)
+			ubc_setsize(rvp, 0);
+		if (c_flag & C_NEED_RVNODE_PUT)
+			vnode_put(rvp);
+	}
 }
 
 /*
diff --git a/bsd/hfs/hfs_cnode.h b/bsd/hfs/hfs_cnode.h
index 83b92d198..a4d0a3981 100644
--- a/bsd/hfs/hfs_cnode.h
+++ b/bsd/hfs/hfs_cnode.h
@@ -163,6 +163,9 @@ typedef struct cnode cnode_t;
 #define C_FORCEUPDATE      0x00100  /* force the catalog entry update */
 #define C_HASXATTRS        0x00200  /* cnode has extended attributes */
 
+#define C_NEED_DATA_SETSIZE  0x01000  /* Do a ubc_setsize(0) on c_rsrc_vp after the unlock */
+#define C_NEED_RSRC_SETSIZE  0x02000  /* Do a ubc_setsize(0) on c_vp after the unlock */
+
 
 #define ZFTIMELIMIT	(5 * 60)
 
diff --git a/bsd/hfs/hfs_encodings.c b/bsd/hfs/hfs_encodings.c
index 951b18795..a4272c19b 100644
--- a/bsd/hfs/hfs_encodings.c
+++ b/bsd/hfs/hfs_encodings.c
@@ -75,11 +75,8 @@ hfs_converterinit(void)
 	SLIST_INIT(&hfs_encoding_list);
 
 	encodinglst_lck_grp_attr= lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(encodinglst_lck_grp_attr);
 	encodinglst_lck_grp  = lck_grp_alloc_init("cnode_hash", encodinglst_lck_grp_attr);
-
 	encodinglst_lck_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(encodinglst_lck_attr);
 
 	lck_mtx_init(&encodinglst_mutex, encodinglst_lck_grp, encodinglst_lck_attr);
 
diff --git a/bsd/hfs/hfs_endian.c b/bsd/hfs/hfs_endian.c
index b96c3f6a9..dd8b7f13c 100644
--- a/bsd/hfs/hfs_endian.c
+++ b/bsd/hfs/hfs_endian.c
@@ -33,8 +33,6 @@
  * volume format.
  */
 
-#include <architecture/byte_order.h>
-
 #include "hfs_endian.h"
 #include "hfs_dbg.h"
 #include "hfscommon/headers/BTreesPrivate.h"
diff --git a/bsd/hfs/hfs_endian.h b/bsd/hfs/hfs_endian.h
index 004e36c12..519c40104 100644
--- a/bsd/hfs/hfs_endian.h
+++ b/bsd/hfs/hfs_endian.h
@@ -40,17 +40,16 @@
  */
 #include "hfs.h"
 #include "hfscommon/headers/BTreesInternal.h"
-#include <architecture/byte_order.h>
+#include <libkern/OSByteOrder.h>
 
 /*********************/
 /* BIG ENDIAN Macros */
 /*********************/
-#if BYTE_ORDER == BIG_ENDIAN
+#define SWAP_BE16(__a) 							OSSwapBigToHostInt16 (__a)
+#define SWAP_BE32(__a) 							OSSwapBigToHostInt32 (__a)
+#define SWAP_BE64(__a) 							OSSwapBigToHostInt64 (__a)
 
-    /* HFS is always big endian, make swaps into no-ops */
-    #define SWAP_BE16(__a) (__a)
-    #define SWAP_BE32(__a) (__a)
-    #define SWAP_BE64(__a) (__a)
+#if BYTE_ORDER == BIG_ENDIAN
     
     /* HFS is always big endian, no swapping needed */
     #define SWAP_HFS_PLUS_FORK_DATA(__a)
@@ -60,11 +59,6 @@
 /************************/
 #elif BYTE_ORDER == LITTLE_ENDIAN
 
-    /* HFS is always big endian, make swaps actually swap */
-    #define SWAP_BE16(__a) 							NXSwapBigShortToHost (__a)
-    #define SWAP_BE32(__a) 							NXSwapBigLongToHost (__a)
-    #define SWAP_BE64(__a) 							NXSwapBigLongLongToHost (__a)
-    
     #define SWAP_HFS_PLUS_FORK_DATA(__a)			hfs_swap_HFSPlusForkData ((__a))
 
 #else
diff --git a/bsd/hfs/hfs_fsctl.h b/bsd/hfs/hfs_fsctl.h
index 6f06e9d8e..312fce955 100644
--- a/bsd/hfs/hfs_fsctl.h
+++ b/bsd/hfs/hfs_fsctl.h
@@ -47,6 +47,9 @@ struct hfs_backingstoreinfo {
 
 /* HFS FS CONTROL COMMANDS */
 
+#define HFSIOC_RESIZE_PROGRESS  _IOR('h', 1, u_int32_t)
+#define HFS_RESIZE_PROGRESS  IOCBASECMD(HFSIOC_RESIZE_PROGRESS)
+
 #define HFSIOC_RESIZE_VOLUME  _IOW('h', 2, u_int64_t)
 #define HFS_RESIZE_VOLUME  IOCBASECMD(HFSIOC_RESIZE_VOLUME)
 
diff --git a/bsd/hfs/hfs_hotfiles.c b/bsd/hfs/hfs_hotfiles.c
index fa59c27e0..2516461a9 100644
--- a/bsd/hfs/hfs_hotfiles.c
+++ b/bsd/hfs/hfs_hotfiles.c
@@ -294,13 +294,10 @@ hfs_recording_stop(struct hfsmount *hfsmp)
 	if (hfsmp->hfc_stage != HFC_RECORDING)
 		return (EPERM);
 
-	hotfiles_collect(hfsmp);
-
-	if (hfsmp->hfc_stage != HFC_RECORDING)
-		return (0);
-
 	hfsmp->hfc_stage = HFC_BUSY;
 
+	hotfiles_collect(hfsmp);
+
 	/*
 	 * Convert hot file data into a simple file id list....
 	 *
@@ -765,6 +762,7 @@ hfs_addhotfile_internal(struct vnode *vp)
 
 	if ((ffp->ff_bytesread == 0) ||
 	    (ffp->ff_blocks == 0) ||
+	    (ffp->ff_size == 0) ||
 	    (ffp->ff_blocks > hotdata->maxblocks) ||
 	    (cp->c_flag & (C_DELETED | C_NOEXISTS)) ||
 	    (cp->c_flags & UF_NODUMP) ||
@@ -828,7 +826,7 @@ hfs_removehotfile(struct vnode *vp)
 	cp = VTOC(vp);
 
 	if ((ffp->ff_bytesread == 0) || (ffp->ff_blocks == 0) ||
-	    (cp->c_atime < hfsmp->hfc_timebase)) {
+	    (ffp->ff_size == 0) || (cp->c_atime < hfsmp->hfc_timebase)) {
 		return (0);
 	}
 
diff --git a/bsd/hfs/hfs_quota.c b/bsd/hfs/hfs_quota.c
index 1354664f7..b2945e081 100644
--- a/bsd/hfs/hfs_quota.c
+++ b/bsd/hfs/hfs_quota.c
@@ -169,7 +169,11 @@ hfs_chkdq(cp, change, cred, flags)
 		return (0);
 	}
 	p = current_proc();
-	if (cred == NOCRED)
+	/*
+	 * This use of proc_ucred() is safe because kernproc credential never
+	 * changes.
+	 */
+	if (!IS_VALID_CRED(cred))
 		cred = proc_ucred(kernproc);
 	if (suser(cred, NULL) || proc_forcequota(p)) {
 		for (i = 0; i < MAXQUOTAS; i++) {
@@ -310,7 +314,11 @@ hfs_chkiq(cp, change, cred, flags)
 		return (0);
 	}
 	p = current_proc();
-	if (cred == NOCRED)
+	/*
+	 * This use of proc_ucred() is safe because kernproc credential never
+	 * changes.
+	 */
+	if (!IS_VALID_CRED(cred))
 		cred = proc_ucred(kernproc);
 	if (suser(cred, NULL) || proc_forcequota(p)) {
 		for (i = 0; i < MAXQUOTAS; i++) {
@@ -501,8 +509,8 @@ hfs_quotaon(p, mp, type, fnamep)
 	if (error) {
 		(void) vnode_close(vp, FREAD|FWRITE, NULL);
 
-	        kauth_cred_rele(qfp->qf_cred);
-		qfp->qf_cred = NOCRED;
+		if (IS_VALID_CRED(qfp->qf_cred))
+		        kauth_cred_unref(&qfp->qf_cred);
 	        qfp->qf_vp = NULLVP;
 		goto out;
 	}
@@ -569,7 +577,6 @@ hfs_quotaoff(__unused struct proc *p, struct mount *mp, register int type)
 	struct hfsmount *hfsmp = VFSTOHFS(mp);
 	struct quotafile *qfp;
 	int error;
-	kauth_cred_t cred;
 	struct hfs_quotaoff_cargs args;
 
 	qfp = &hfsmp->hfs_qfiles[type];
@@ -604,11 +611,9 @@ hfs_quotaoff(__unused struct proc *p, struct mount *mp, register int type)
 	error = vnode_close(qvp, FREAD|FWRITE, NULL);
 
 	qfp->qf_vp = NULLVP;
-	cred = qfp->qf_cred;
-	if (cred != NOCRED) {
-		qfp->qf_cred = NOCRED;
-		kauth_cred_rele(cred);
-	}
+
+	if (IS_VALID_CRED(qfp->qf_cred))
+		kauth_cred_unref(&qfp->qf_cred);
 	for (type = 0; type < MAXQUOTAS; type++)
 		if (hfsmp->hfs_qfiles[type].qf_vp != NULLVP)
 			break;
diff --git a/bsd/hfs/hfs_readwrite.c b/bsd/hfs/hfs_readwrite.c
index fd934e2f5..b9c8bf912 100644
--- a/bsd/hfs/hfs_readwrite.c
+++ b/bsd/hfs/hfs_readwrite.c
@@ -871,6 +871,18 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 
 	switch (ap->a_command) {
 
+	case HFS_RESIZE_PROGRESS: {
+
+		vfsp = vfs_statfs(HFSTOVFS(hfsmp));
+		if (suser(cred, NULL) &&
+			kauth_cred_getuid(cred) != vfsp->f_owner) {
+			return (EACCES); /* must be owner of file system */
+		}
+		if (!vnode_isvroot(vp)) {
+			return (EINVAL);
+		}
+		return hfs_resize_progress(hfsmp, (u_int32_t *)ap->a_data);
+	}
 	case HFS_RESIZE_VOLUME: {
 		u_int64_t newsize;
 		u_int64_t cursize;
@@ -1082,7 +1094,7 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 		dev_t dev = VTOC(vp)->c_dev;
 		
 		short flags;
-		struct ucred myucred;	/* XXX ILLEGAL */
+		struct ucred myucred;
 		int num_files;
 		int *file_ids = NULL;
 		short *access = NULL;
@@ -1096,6 +1108,9 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 		struct cnode *skip_cp = VTOC(vp);
 		struct vfs_context	my_context;
 
+		/* set up front for common exit code */
+		my_context.vc_ucred = NOCRED;
+
 		/* first, return error if not run as root */
 		if (cred->cr_ruid != 0) {
 			return EPERM;
@@ -1166,6 +1181,12 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 			check_leaf = false;
 		}
 		
+		/*
+		 * Create a templated credential; this credential may *NOT*
+		 * be used unless instantiated with a kauth_cred_create();
+		 * there must be a correcponding kauth_cred_unref() when it
+		 * is no longer in use (i.e. before it goes out of scope).
+		 */
 		memset(&myucred, 0, sizeof(myucred));
 		myucred.cr_ref = 1;
 		myucred.cr_uid = myucred.cr_ruid = myucred.cr_svuid = user_access_structp->uid;
@@ -1180,7 +1201,7 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 		myucred.cr_gmuid = myucred.cr_uid;
 		
 		my_context.vc_proc = p;
-		my_context.vc_ucred = &myucred;
+		my_context.vc_ucred = kauth_cred_create(&myucred);
 
 		/* Check access to each file_id passed in */
 		for (i = 0; i < num_files; i++) {
@@ -1188,7 +1209,7 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 			cnid = (cnid_t) file_ids[i];
 			
 			/* root always has access */
-			if (!suser(&myucred, NULL)) {
+			if (!suser(my_context.vc_ucred, NULL)) {
 				access[i] = 0;
 				continue;
 			}
@@ -1204,7 +1225,7 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 							
 				/* before calling CheckAccess(), check the target file for read access */
 				myPerms = DerivePermissionSummary(cnattr.ca_uid, cnattr.ca_gid,
-								  cnattr.ca_mode, hfsmp->hfs_mp, &myucred, p  );
+								  cnattr.ca_mode, hfsmp->hfs_mp, my_context.vc_ucred, p  );
 				
 				
 				/* fail fast if no access */ 
@@ -1225,7 +1246,7 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 			}
 			
 			myaccess = do_access_check(hfsmp, &error, &cache, catkey.hfsPlus.parentID, 
-						   skip_cp, p, &myucred, dev);
+						   skip_cp, p, my_context.vc_ucred, dev);
 			
 			if ( myaccess ) {
 				access[i] = 0; // have access.. no errors to report
@@ -1251,6 +1272,12 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 
 			    hfs_unlock(VTOC(vp));
 			    if (vnode_vtype(vp) == VDIR) {
+			    	/*
+				 * XXX This code assumes that none of the
+				 * XXX callbacks from vnode_authorize() will
+				 * XXX take a persistent ref on the context
+				 * XXX credential, which is a bad assumption.
+				 */
 				myErr = vnode_authorize(vp, NULL, (KAUTH_VNODE_SEARCH | KAUTH_VNODE_LIST_DIRECTORY), &my_context);
 			    } else {
 				myErr = vnode_authorize(vp, NULL, KAUTH_VNODE_READ_DATA, &my_context);
@@ -1278,6 +1305,9 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 		release_pathbuff((char *) cache.haveaccess);
 		release_pathbuff((char *) file_ids);
 		release_pathbuff((char *) access);
+		/* clean up local context, if needed */
+		if (IS_VALID_CRED(my_context.vc_ucred))
+			kauth_cred_unref(&my_context.vc_ucred);
 		
 		return (error);
 	} /* HFS_BULKACCESS */
@@ -2163,7 +2193,7 @@ hfs_truncate(struct vnode *vp, off_t length, int flags, int skipsetsize,
 
 	if (length < filebytes) {
 		while (filebytes > length) {
-			if ((filebytes - length) > HFS_BIGFILE_SIZE) {
+			if ((filebytes - length) > HFS_BIGFILE_SIZE && overflow_extents(fp)) {
 		    		filebytes -= HFS_BIGFILE_SIZE;
 			} else {
 		    		filebytes = length;
@@ -2175,7 +2205,7 @@ hfs_truncate(struct vnode *vp, off_t length, int flags, int skipsetsize,
 		}
 	} else if (length > filebytes) {
 		while (filebytes < length) {
-			if ((length - filebytes) > HFS_BIGFILE_SIZE) {
+			if ((length - filebytes) > HFS_BIGFILE_SIZE && overflow_extents(fp)) {
 				filebytes += HFS_BIGFILE_SIZE;
 			} else {
 				filebytes = length;
@@ -2572,7 +2602,8 @@ hfs_vnop_bwrite(struct vnop_bwrite_args *ap)
 	/* Trap B-Tree writes */
 	if ((VTOC(vp)->c_fileid == kHFSExtentsFileID) ||
 	    (VTOC(vp)->c_fileid == kHFSCatalogFileID) ||
-	    (VTOC(vp)->c_fileid == kHFSAttributesFileID)) {
+	    (VTOC(vp)->c_fileid == kHFSAttributesFileID) ||
+	    (vp == VTOHFS(vp)->hfc_filevp)) {
 
 		/* 
 		 * Swap and validate the node if it is in native byte order.
@@ -2841,10 +2872,10 @@ out:
 		lockflags = 0;
 	}
 
-	// See comment up above about calls to hfs_fsync()
-	//
-	//if (retval == 0)
-	//	retval = hfs_fsync(vp, MNT_WAIT, 0, p);
+	/* Push cnode's new extent data to disk. */
+	if (retval == 0) {
+		(void) hfs_update(vp, MNT_WAIT);
+	}
 
 	if (hfsmp->jnl) {
 		if (cp->c_cnid < kHFSFirstUserCatalogNodeID)
@@ -2938,7 +2969,7 @@ hfs_clonefile(struct vnode *vp, int blkstart, int blkcnt, int blksize)
 	filesize = VTOF(vp)->ff_blocks * blksize;  /* virtual file size */
 	writebase = blkstart * blksize;
 	copysize = blkcnt * blksize;
-	iosize = bufsize = MIN(copysize, 4096 * 16);
+	iosize = bufsize = MIN(copysize, 128 * 1024);
 	offset = 0;
 
 	if (kmem_alloc(kernel_map, (vm_offset_t *)&bufp, bufsize)) {
diff --git a/bsd/hfs/hfs_vfsops.c b/bsd/hfs/hfs_vfsops.c
index ada09a3d1..7d7094a78 100644
--- a/bsd/hfs/hfs_vfsops.c
+++ b/bsd/hfs/hfs_vfsops.c
@@ -146,7 +146,9 @@ static int hfs_unmount(struct mount *mp, int mntflags, vfs_context_t context);
 static int hfs_vfs_vget(struct mount *mp, ino64_t ino, struct vnode **vpp, vfs_context_t context);
 static int hfs_vptofh(struct vnode *vp, int *fhlenp, unsigned char *fhp, vfs_context_t context);
 
-static int hfs_reclaimspace(struct hfsmount *hfsmp, u_long startblk);
+static int hfs_reclaimspace(struct hfsmount *hfsmp, u_long startblk, u_long reclaimblks);
+static int hfs_overlapped_overflow_extents(struct hfsmount *hfsmp, u_int32_t startblk,
+                                           u_int32_t catblks, u_int32_t fileID, int rsrcfork);
 
 
 /*
@@ -1567,9 +1569,9 @@ hfs_sync_metadata(void *arg)
 	priIDSector = (daddr64_t)((vcb->hfsPlusIOPosOffset / sectorsize) +
 				  HFS_PRI_SECTOR(sectorsize));
 	retval = (int)buf_meta_bread(hfsmp->hfs_devvp, priIDSector, sectorsize, NOCRED, &bp);
-	if (retval != 0) {
-		panic("hfs: sync_metadata: can't read super-block?! (retval 0x%x, priIDSector)\n",
-			  retval, priIDSector);
+	if ((retval != 0) && (retval != ENXIO)) {
+		printf("hfs_sync_metadata: can't read volume header at %d! (retval 0x%x)\n",
+			priIDSector, retval);
 	}
 
 	if (retval == 0 && ((buf_flags(bp) & (B_DELWRI | B_LOCKED)) == B_DELWRI)) {
@@ -1771,7 +1773,7 @@ hfs_fhtovp(struct mount *mp, int fhlen, unsigned char *fhp, struct vnode **vpp,
 	if (fhlen < sizeof(struct hfsfid))
 		return (EINVAL);
 
-	result = hfs_vget(VFSTOHFS(mp), hfsfhp->hfsfid_cnid, &nvp, 0);
+	result = hfs_vget(VFSTOHFS(mp), ntohl(hfsfhp->hfsfid_cnid), &nvp, 0);
 	if (result) {
 		if (result == ENOENT)
 			result = ESTALE;
@@ -1789,7 +1791,7 @@ hfs_fhtovp(struct mount *mp, int fhlen, unsigned char *fhp, struct vnode **vpp,
 	 * error prone. Future, would be change the "wrap bit" to a unique
 	 * wrap number and use that for generation number. For now do this.
 	 */  
-	if ((hfsfhp->hfsfid_gen < VTOC(nvp)->c_itime)) {
+	if ((ntohl(hfsfhp->hfsfid_gen) < VTOC(nvp)->c_itime)) {
 		hfs_unlock(VTOC(nvp));
 		vnode_put(nvp);
 		return (ESTALE);
@@ -1819,8 +1821,8 @@ hfs_vptofh(struct vnode *vp, int *fhlenp, unsigned char *fhp, vfs_context_t cont
 
 	cp = VTOC(vp);
 	hfsfhp = (struct hfsfid *)fhp;
-	hfsfhp->hfsfid_cnid = cp->c_fileid;
-	hfsfhp->hfsfid_gen = cp->c_itime;
+	hfsfhp->hfsfid_cnid = htonl(cp->c_fileid);
+	hfsfhp->hfsfid_gen = htonl(cp->c_itime);
 	*fhlenp = sizeof(struct hfsfid);
 	
 	return (0);
@@ -1851,10 +1853,7 @@ hfs_init(__unused struct vfsconf *vfsp)
 	hfs_group_attr   = lck_grp_attr_alloc_init();
 	hfs_mutex_group  = lck_grp_alloc_init("hfs-mutex", hfs_group_attr);
 	hfs_rwlock_group = lck_grp_alloc_init("hfs-rwlock", hfs_group_attr);
-
-	/* Turn on lock debugging */
-	//lck_attr_setdebug(hfs_lock_attr);
-
+	
 
 	return (0);
 }
@@ -2027,6 +2026,16 @@ hfs_sysctl(int *name, __unused u_int namelen, user_addr_t oldp, size_t *oldlenp,
 
 		hfs_global_exclusive_lock_acquire(hfsmp);
 		
+		/*
+		 * Flush all dirty metadata buffers.
+		 */
+		buf_flushdirtyblks(hfsmp->hfs_devvp, MNT_WAIT, 0, "hfs_sysctl");
+		buf_flushdirtyblks(hfsmp->hfs_extents_vp, MNT_WAIT, 0, "hfs_sysctl");
+		buf_flushdirtyblks(hfsmp->hfs_catalog_vp, MNT_WAIT, 0, "hfs_sysctl");
+		buf_flushdirtyblks(hfsmp->hfs_allocation_vp, MNT_WAIT, 0, "hfs_sysctl");
+		if (hfsmp->hfs_attribute_vp)
+			buf_flushdirtyblks(hfsmp->hfs_attribute_vp, MNT_WAIT, 0, "hfs_sysctl");
+
 		HFSTOVCB(hfsmp)->vcbJinfoBlock = name[1];
 		HFSTOVCB(hfsmp)->vcbAtrb |= kHFSVolumeJournaledMask;
 		hfsmp->jvp = jvp;
@@ -2227,11 +2236,6 @@ hfs_vget(struct hfsmount *hfsmp, cnid_t cnid, struct vnode **vpp, int skiplock)
 		    (bcmp(cndesc.cd_nameptr, HFS_INODE_PREFIX, HFS_INODE_PREFIX_LEN) == 0)) {
 			linkref = strtoul((const char*)&cndesc.cd_nameptr[HFS_INODE_PREFIX_LEN], NULL, 10);
 			cnattr.ca_rdev = linkref;
-
-			// patch up the parentcnid
-			if (cnattr.ca_attrblks != 0) {
-			    cndesc.cd_parentcnid = cnattr.ca_attrblks;
-			}
 		}
 	}
 
@@ -3018,7 +3022,6 @@ __private_extern__
 int
 hfs_truncatefs(struct hfsmount *hfsmp, u_int64_t newsize, __unused vfs_context_t context)
 {
-	struct vnode* rvp = NULL;
 	struct  buf *bp = NULL;
 	u_int64_t oldsize;
 	u_int32_t newblkcnt;
@@ -3027,20 +3030,22 @@ hfs_truncatefs(struct hfsmount *hfsmp, u_int64_t newsize, __unused vfs_context_t
 	int transaction_begun = 0;
 	int error;
 
-	/*
-	 * Grab the root vnode to serialize with another hfs_truncatefs call.
-	 */
-	error = hfs_vget(hfsmp, kHFSRootFolderID, &rvp, 0);
-	if (error) {
-		return (error);
+	
+	lck_mtx_lock(&hfsmp->hfs_mutex);
+	if (hfsmp->hfs_flags & HFS_RESIZE_IN_PROGRESS) {
+		lck_mtx_unlock(&hfsmp->hfs_mutex);
+		return (EALREADY);
 	}
+	hfsmp->hfs_flags |= HFS_RESIZE_IN_PROGRESS;
+	hfsmp->hfs_resize_filesmoved = 0;
+	hfsmp->hfs_resize_totalfiles = 0;
+	lck_mtx_unlock(&hfsmp->hfs_mutex);
+
 	/*
-	 * - HFS Plus file systems only. 
-	 * - Journaling must be enabled.
+	 * - Journaled HFS Plus volumes only.
 	 * - No embedded volumes.
 	 */
-	if ((hfsmp->hfs_flags & HFS_STANDARD) ||
-	    (hfsmp->jnl == NULL) ||
+	if ((hfsmp->jnl == NULL) ||
 	    (hfsmp->hfsPlusIOPosOffset != 0)) {
 		error = EPERM;
 		goto out;
@@ -3057,13 +3062,12 @@ hfs_truncatefs(struct hfsmount *hfsmp, u_int64_t newsize, __unused vfs_context_t
 		goto out;
 	}
 	/* Make sure there's enough space to work with. */
-	if (reclaimblks > (hfsmp->freeBlocks / 4)) {
+	if (reclaimblks >= hfs_freeblks(hfsmp, 1)) {
 		error = ENOSPC;
 		goto out;
 	}
-
-	printf("hfs_truncatefs: shrinking %s by %d blocks out of %d\n",
-	       hfsmp->vcbVN, reclaimblks, hfsmp->totalBlocks);
+	/* Start with a clean journal. */
+	journal_flush(hfsmp->jnl);
 
 	if (hfs_start_transaction(hfsmp) != 0) {
 		error = EINVAL;
@@ -3083,7 +3087,7 @@ hfs_truncatefs(struct hfsmount *hfsmp, u_int64_t newsize, __unused vfs_context_t
 		transaction_begun = 0;
 
 		/* Attempt to reclaim some space. */ 
-		if (hfs_reclaimspace(hfsmp, newblkcnt) != 0) {
+		if (hfs_reclaimspace(hfsmp, newblkcnt, reclaimblks) != 0) {
 			printf("hfs_truncatefs: couldn't reclaim space on %s\n", hfsmp->vcbVN);
 			error = ENOSPC;
 			goto out;
@@ -3097,7 +3101,7 @@ hfs_truncatefs(struct hfsmount *hfsmp, u_int64_t newsize, __unused vfs_context_t
 		/* Check if we're clear now. */
 		if (hfs_isallocated(hfsmp, newblkcnt, reclaimblks - 1)) {
 			printf("hfs_truncatefs: didn't reclaim enough space on %s\n", hfsmp->vcbVN);
-			error = ENOSPC;
+			error = EAGAIN;  /* tell client to try again */
 			goto out;
 		}
 	}
@@ -3126,21 +3130,26 @@ hfs_truncatefs(struct hfsmount *hfsmp, u_int64_t newsize, __unused vfs_context_t
 
 	/*
 	 * Invalidate the existing alternate volume header.
+	 *
+	 * Don't do this as a transaction (don't call journal_modify_block)
+	 * since this block will be outside of the truncated file system!
 	 */
 	if (hfsmp->hfs_alt_id_sector) {
 		if (buf_meta_bread(hfsmp->hfs_devvp, hfsmp->hfs_alt_id_sector,
 		                   hfsmp->hfs_phys_block_size, NOCRED, &bp) == 0) {
-			journal_modify_block_start(hfsmp->jnl, bp);
 	
 			bzero((void*)((char *)buf_dataptr(bp) + HFS_ALT_OFFSET(hfsmp->hfs_phys_block_size)), kMDBSize);
-	
-			journal_modify_block_end(hfsmp->jnl, bp);
+			(void) VNOP_BWRITE(bp);
 		} else if (bp) {
 			buf_brelse(bp);
 		}
 		bp = NULL;
 	}
 
+	/* Log successful shrinking. */
+	printf("hfs_truncatefs: shrank \"%s\" to %d blocks (was %d blocks)\n",
+	       hfsmp->vcbVN, newblkcnt, hfsmp->totalBlocks);
+
 	/*
 	 * Adjust file system variables and flush them to disk.
 	 */
@@ -3158,54 +3167,90 @@ out:
 	}
 	if (transaction_begun) {
 		hfs_end_transaction(hfsmp);
+		journal_flush(hfsmp->jnl);
 	}
-	if (rvp) {
-		hfs_unlock(VTOC(rvp));
-		vnode_put(rvp);
-	}
+
+	lck_mtx_lock(&hfsmp->hfs_mutex);
+	hfsmp->hfs_flags &= ~HFS_RESIZE_IN_PROGRESS;
+	lck_mtx_unlock(&hfsmp->hfs_mutex);
+
 	return (error);
 }
 
+
 /*
  * Reclaim space at the end of a file system.
  */
 static int
-hfs_reclaimspace(struct hfsmount *hfsmp, u_long startblk)
+hfs_reclaimspace(struct hfsmount *hfsmp, u_long startblk, u_long reclaimblks)
 {
 	struct vnode *vp = NULL;
 	FCB *fcb;
 	struct BTreeIterator * iterator = NULL;
 	struct FSBufferDescriptor btdata;
 	struct HFSPlusCatalogFile filerec;
+	struct filefork *fp;
 	u_int32_t  saved_next_allocation;
 	cnid_t * cnidbufp;
 	size_t cnidbufsize;
-	int filecnt;
+	int filecnt = 0;
 	int maxfilecnt;
 	u_long block;
+	u_long datablks;
+	u_long rsrcblks;
+	u_long blkstomove = 0;
 	int lockflags;
 	int i;
 	int error;
+	int lastprogress = 0;
 
-	/* 
-	 * Check if Attributes file overlaps.
-	 */
+
+	/* Check if Attributes file overlaps reclaim area. */
 	if (hfsmp->hfs_attribute_vp) {
-		struct filefork *fp;
-	
 		fp = VTOF(hfsmp->hfs_attribute_vp);
+		datablks = 0;
 		for (i = 0; i < kHFSPlusExtentDensity; ++i) {
-			block = fp->ff_extents[i].startBlock +
-				fp->ff_extents[i].blockCount;
+			if (fp->ff_extents[i].blockCount == 0) {
+				break;
+			}
+			datablks += fp->ff_extents[i].blockCount;
+			block = fp->ff_extents[i].startBlock + fp->ff_extents[i].blockCount;
 			if (block >= startblk) {
 				printf("hfs_reclaimspace: Attributes file can't move\n");
 				return (EPERM);
 			}
 		}
+		if ((i == kHFSPlusExtentDensity) && (fp->ff_blocks > datablks)) {
+			if (hfs_overlapped_overflow_extents(hfsmp, startblk, datablks, kHFSAttributesFileID, 0)) {
+				printf("hfs_reclaimspace: Attributes file can't move\n");
+				return (EPERM);
+			}
+		}
+	}
+	/* Check if Catalog file overlaps reclaim area. */
+	fp = VTOF(hfsmp->hfs_catalog_vp);
+	datablks = 0;
+	for (i = 0; i < kHFSPlusExtentDensity; ++i) {
+		if (fp->ff_extents[i].blockCount == 0) {
+			break;
+		}
+		datablks += fp->ff_extents[i].blockCount;
+		block = fp->ff_extents[i].startBlock + fp->ff_extents[i].blockCount;
+		if (block >= startblk) {
+			printf("hfs_reclaimspace: Catalog file can't move\n");
+			return (EPERM);
+		}
+	}
+	if ((i == kHFSPlusExtentDensity) && (fp->ff_blocks > datablks)) {
+		if (hfs_overlapped_overflow_extents(hfsmp, startblk, datablks, kHFSCatalogFileID, 0)) {
+			printf("hfs_reclaimspace: Catalog file can't move\n");
+			return (EPERM);
+		}
 	}
 
-	/* For now we'll move a maximum of 16,384 files. */
-	maxfilecnt = MIN(hfsmp->hfs_filecount, 16384);
+	/* For now move a maximum of 250,000 files. */
+	maxfilecnt = MIN(hfsmp->hfs_filecount, 250000);
+	maxfilecnt = MIN((u_long)maxfilecnt, reclaimblks);
 	cnidbufsize = maxfilecnt * sizeof(cnid_t);
 	if (kmem_alloc(kernel_map, (vm_offset_t *)&cnidbufp, cnidbufsize)) {
 		return (ENOMEM);
@@ -3225,14 +3270,13 @@ hfs_reclaimspace(struct hfsmount *hfsmp, u_long startblk)
 	btdata.itemSize = sizeof(filerec);
 	btdata.itemCount = 1;
 
-	/* Keep the Catalog file locked during iteration. */
-	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
+	/* Keep the Catalog and extents files locked during iteration. */
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG | SFL_EXTENTS, HFS_SHARED_LOCK);
+
 	error = BTIterateRecord(fcb, kBTreeFirstRecord, iterator, NULL, NULL);
 	if (error) {
-		hfs_systemfile_unlock(hfsmp, lockflags);
-		goto out;
+		goto end_iteration;
 	}
-
 	/*
 	 * Iterate over all the catalog records looking for files
 	 * that overlap into the space we're trying to free up.
@@ -3240,41 +3284,99 @@ hfs_reclaimspace(struct hfsmount *hfsmp, u_long startblk)
 	for (filecnt = 0; filecnt < maxfilecnt; ) {
 		error = BTIterateRecord(fcb, kBTreeNextRecord, iterator, &btdata, NULL);
 		if (error) {
-			if (error == btNotFound)
-				error = 0;
+			if (error == fsBTRecordNotFoundErr || error == fsBTEndOfIterationErr) {
+				error = 0;				
+			}
 			break;
 		}
-		if (filerec.recordType != kHFSPlusFileRecord ||
-		    filerec.fileID == hfsmp->hfs_jnlfileid)
+		if (filerec.recordType != kHFSPlusFileRecord) {
 			continue;
+		}
+		datablks = rsrcblks = 0;
 		/* 
 		 * Check if either fork overlaps target space.
 		 */
 		for (i = 0; i < kHFSPlusExtentDensity; ++i) {
-			block = filerec.dataFork.extents[i].startBlock +
-			        filerec.dataFork.extents[i].blockCount;
-			if (block >= startblk) {
-				if (filerec.fileID == hfsmp->hfs_jnlfileid) {
-					printf("hfs_reclaimspace: cannot move active journal\n");
-					error = EPERM;
+			if (filerec.dataFork.extents[i].blockCount != 0) {
+				datablks += filerec.dataFork.extents[i].blockCount;
+				block = filerec.dataFork.extents[i].startBlock +
+						filerec.dataFork.extents[i].blockCount;
+				if (block >= startblk) {
+					if ((filerec.fileID == hfsmp->hfs_jnlfileid) ||
+						(filerec.fileID == hfsmp->hfs_jnlinfoblkid)) {
+						printf("hfs_reclaimspace: cannot move active journal\n");
+						error = EPERM;
+						goto end_iteration;
+					}
+					cnidbufp[filecnt++] = filerec.fileID;
+					blkstomove += filerec.dataFork.totalBlocks;
 					break;
 				}
-				cnidbufp[filecnt++] = filerec.fileID;
-				break;
 			}
-			block = filerec.resourceFork.extents[i].startBlock +
-			        filerec.resourceFork.extents[i].blockCount;
-			if (block >= startblk) {
-				cnidbufp[filecnt++] = filerec.fileID;
-				break;
+			if (filerec.resourceFork.extents[i].blockCount != 0) {
+				rsrcblks += filerec.resourceFork.extents[i].blockCount;
+				block = filerec.resourceFork.extents[i].startBlock +
+						filerec.resourceFork.extents[i].blockCount;
+				if (block >= startblk) {
+					cnidbufp[filecnt++] = filerec.fileID;
+					blkstomove += filerec.resourceFork.totalBlocks;
+					break;
+				}
+			}
+		}
+		/*
+		 * Check for any overflow extents that overlap.
+		 */
+		if (i == kHFSPlusExtentDensity) {
+			if (filerec.dataFork.totalBlocks > datablks) {
+				if (hfs_overlapped_overflow_extents(hfsmp, startblk, datablks, filerec.fileID, 0)) {
+					cnidbufp[filecnt++] = filerec.fileID;
+					blkstomove += filerec.dataFork.totalBlocks;
+				}
+			} else if (filerec.resourceFork.totalBlocks > rsrcblks) {
+				if (hfs_overlapped_overflow_extents(hfsmp, startblk, rsrcblks, filerec.fileID, 1)) {
+					cnidbufp[filecnt++] = filerec.fileID;
+					blkstomove += filerec.resourceFork.totalBlocks;
+				}
 			}
 		}
 	}
+
+end_iteration:
+	if (filecnt == 0) {
+		error = ENOSPC;
+	}
 	/* All done with catalog. */
 	hfs_systemfile_unlock(hfsmp, lockflags);
 	if (error)
 		goto out;
 
+	/*
+	 * Double check space requirements to make sure
+	 * there is enough space to relocate any files
+	 * that reside in the reclaim area.
+	 *
+	 *                                          Blocks To Move --------------
+	 *                                                            |    |    |
+	 *                                                            V    V    V
+	 * ------------------------------------------------------------------------
+	 * |                                                        | /   ///  // |
+	 * |                                                        | /   ///  // |
+	 * |                                                        | /   ///  // |
+	 * ------------------------------------------------------------------------
+	 *
+	 * <------------------- New Total Blocks ------------------><-- Reclaim -->
+	 *
+	 * <------------------------ Original Total Blocks ----------------------->
+	 *
+	 */
+	if ((reclaimblks + blkstomove) >= hfs_freeblks(hfsmp, 1)) {
+		error = ENOSPC;
+		goto out;
+	}
+	hfsmp->hfs_resize_filesmoved = 0;
+	hfsmp->hfs_resize_totalfiles = filecnt;
+	
 	/* Now move any files that are in the way. */
 	for (i = 0; i < filecnt; ++i) {
 		struct vnode * rvp;
@@ -3302,28 +3404,127 @@ hfs_reclaimspace(struct hfsmount *hfsmp, u_long startblk)
 		hfs_unlock(VTOC(vp));
 		vnode_put(vp);
 		vp = NULL;
+
+		++hfsmp->hfs_resize_filesmoved;
+
+		/* Report intermediate progress. */
+		if (filecnt > 100) {
+			int progress;
+
+			progress = (i * 100) / filecnt;
+			if (progress > (lastprogress + 9)) {
+				printf("hfs_reclaimspace: %d%% done...\n", progress);
+				lastprogress = progress;
+			}
+		}
 	}
 	if (vp) {
 		hfs_unlock(VTOC(vp));
 		vnode_put(vp);
 		vp = NULL;
 	}
-
-	/*
-	 * Note: this implementation doesn't handle overflow extents.
-	 */
+	if (hfsmp->hfs_resize_filesmoved != 0) {
+		printf("hfs_reclaimspace: relocated %d files on \"%s\"\n",
+		       (int)hfsmp->hfs_resize_filesmoved, hfsmp->vcbVN);
+	}
 out:
 	kmem_free(kernel_map, (vm_offset_t)iterator, sizeof(*iterator));
 	kmem_free(kernel_map, (vm_offset_t)cnidbufp, cnidbufsize);
 
-	/* On errors restore the roving allocation pointer. */
-	if (error) {
+	/*
+	 * Restore the roving allocation pointer on errors.
+	 * (but only if we didn't move any files)
+	 */
+	if (error && hfsmp->hfs_resize_filesmoved == 0) {
 		hfsmp->nextAllocation = saved_next_allocation;
 	}
 	return (error);
 }
 
 
+/*
+ * Check if there are any overflow extents that overlap.
+ */
+static int
+hfs_overlapped_overflow_extents(struct hfsmount *hfsmp, u_int32_t startblk, u_int32_t catblks, u_int32_t fileID, int rsrcfork)
+{
+	struct BTreeIterator * iterator = NULL;
+	struct FSBufferDescriptor btdata;
+	HFSPlusExtentRecord extrec;
+	HFSPlusExtentKey *extkeyptr;
+	FCB *fcb;
+	u_int32_t block;
+	u_int8_t forktype;
+	int overlapped = 0;
+	int i;
+	int error;
+
+	forktype = rsrcfork ? 0xFF : 0;
+	if (kmem_alloc(kernel_map, (vm_offset_t *)&iterator, sizeof(*iterator))) {
+		return (0);
+	}	
+	bzero(iterator, sizeof(*iterator));
+	extkeyptr = (HFSPlusExtentKey *)&iterator->key;
+	extkeyptr->keyLength = kHFSPlusExtentKeyMaximumLength;
+	extkeyptr->forkType = forktype;
+	extkeyptr->fileID = fileID;
+	extkeyptr->startBlock = catblks;
+
+	btdata.bufferAddress = &extrec;
+	btdata.itemSize = sizeof(extrec);
+	btdata.itemCount = 1;
+	
+	fcb = VTOF(hfsmp->hfs_extents_vp);
+
+	error = BTSearchRecord(fcb, iterator, &btdata, NULL, iterator);
+	while (error == 0) {
+		/* Stop when we encounter a different file. */
+		if ((extkeyptr->fileID != fileID) ||
+		    (extkeyptr->forkType != forktype)) {
+			break;
+		}
+		/* 
+		 * Check if the file overlaps target space.
+		 */
+		for (i = 0; i < kHFSPlusExtentDensity; ++i) {
+			if (extrec[i].blockCount == 0) {
+				break;
+			}
+			block = extrec[i].startBlock + extrec[i].blockCount;
+			if (block >= startblk) {
+				overlapped = 1;
+				break;
+			}
+		}
+		/* Look for more records. */
+		error = BTIterateRecord(fcb, kBTreeNextRecord, iterator, &btdata, NULL);
+	}
+
+	kmem_free(kernel_map, (vm_offset_t)iterator, sizeof(*iterator));
+	return (overlapped);
+}
+
+
+/*
+ * Calculate the progress of a file system resize operation.
+ */
+__private_extern__
+int
+hfs_resize_progress(struct hfsmount *hfsmp, u_int32_t *progress)
+{
+	if ((hfsmp->hfs_flags & HFS_RESIZE_IN_PROGRESS) == 0) {
+		return (ENXIO);
+	}
+
+	if (hfsmp->hfs_resize_totalfiles > 0)
+		*progress = (hfsmp->hfs_resize_filesmoved * 100) / hfsmp->hfs_resize_totalfiles;
+	else
+		*progress = 0;
+
+	return (0);
+}
+
+
 /*
  * Get file system attributes.
  */
diff --git a/bsd/hfs/hfs_vfsutils.c b/bsd/hfs/hfs_vfsutils.c
index 6024b382d..a77c6b179 100644
--- a/bsd/hfs/hfs_vfsutils.c
+++ b/bsd/hfs/hfs_vfsutils.c
@@ -747,7 +747,14 @@ overflow_extents(struct filefork *fp)
 {
 	u_long blocks;
 
-	if (VTOVCB(FTOV(fp))->vcbSigWord == kHFSPlusSigWord) {
+       //
+       // If the vnode pointer is NULL then we're being called
+       // from hfs_remove_orphans() with a faked-up filefork
+       // and therefore it has to be an HFS+ volume.  Otherwise
+       // we check through the volume header to see what type
+       // of volume we're on.
+       //
+       if (FTOV(fp) == NULL || VTOVCB(FTOV(fp))->vcbSigWord == kHFSPlusSigWord) {
 		if (fp->ff_extents[7].blockCount == 0)
 			return (0);
 
@@ -1311,7 +1318,7 @@ hfs_remove_orphans(struct hfsmount * hfsmp)
 				cnode.c_rsrcfork = NULL;
 				fsize = (u_int64_t)dfork.ff_blocks * (u_int64_t)HFSTOVCB(hfsmp)->blockSize;
 				while (fsize > 0) {
-					if (fsize > HFS_BIGFILE_SIZE) {
+					if (fsize > HFS_BIGFILE_SIZE && overflow_extents(&dfork)) {
 						fsize -= HFS_BIGFILE_SIZE;
 					} else {
 						fsize = 0;
diff --git a/bsd/hfs/hfs_vnops.c b/bsd/hfs/hfs_vnops.c
index 614fde16a..e510c619c 100644
--- a/bsd/hfs/hfs_vnops.c
+++ b/bsd/hfs/hfs_vnops.c
@@ -1571,18 +1571,17 @@ hfs_removefile(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 	if ((cp->c_flag & C_HARDLINK) == 0 &&
 	    (!dataforkbusy || !rsrcforkbusy)) {
 		/*
-		 * A ubc_setsize can cause a pagein here 
-		 * so we need to the drop cnode lock. Note
-		 * that we still hold the truncate lock.
+		 * A ubc_setsize can cause a pagein so defer it
+		 * until after the cnode lock is dropped.  The
+		 * cnode lock cannot be dropped/reacquired here
+		 * since we might already hold the journal lock.
 		 */
-		hfs_unlock(cp);
 		if (!dataforkbusy && cp->c_datafork->ff_blocks && !isbigfile) {
-			ubc_setsize(vp, 0);
+			cp->c_flag |= C_NEED_DATA_SETSIZE;
 		}
 		if (!rsrcforkbusy && rvp) {
-			ubc_setsize(rvp, 0);
+			cp->c_flag |= C_NEED_RSRC_SETSIZE;
 		}
-		hfs_lock(cp, HFS_FORCE_LOCK);
 	} else {
 	    struct cat_desc cndesc;
 
@@ -1896,10 +1895,10 @@ out:
 __private_extern__ void
 replace_desc(struct cnode *cp, struct cat_desc *cdp)
 {
-    if (&cp->c_desc == cdp) {
-        return;
-    }
-         
+	if (&cp->c_desc == cdp) {
+		return;
+	}
+
 	/* First release allocated name buffer */
 	if (cp->c_desc.cd_flags & CD_HASBUF && cp->c_desc.cd_nameptr != 0) {
 		char *name = cp->c_desc.cd_nameptr;
@@ -2835,7 +2834,7 @@ hfs_update(struct vnode *vp, __unused int waitfor)
 	 * we have to do the update.
 	 */
 	if (ISSET(cp->c_flag, C_FORCEUPDATE) == 0 &&
-	    (ISSET(cp->c_flag, C_DELETED) || 
+	    (ISSET(cp->c_flag, C_DELETED) ||
 	    (dataforkp && cp->c_datafork->ff_unallocblocks) ||
 	    (rsrcforkp && cp->c_rsrcfork->ff_unallocblocks))) {
 	//	cp->c_flag &= ~(C_ACCESS | C_CHANGE | C_UPDATE);
@@ -3719,7 +3718,6 @@ struct vnodeopv_entry_desc hfs_specop_entries[] = {
 	{ &vnop_pathconf_desc, (VOPFUNC)spec_pathconf },		/* pathconf */
 	{ &vnop_advlock_desc, (VOPFUNC)err_advlock },		/* advlock */
 	{ &vnop_bwrite_desc, (VOPFUNC)hfs_vnop_bwrite },
-	{ &vnop_devblocksize_desc, (VOPFUNC)spec_devblocksize }, /* devblocksize */
 	{ &vnop_pagein_desc, (VOPFUNC)hfs_vnop_pagein },		/* Pagein */
 	{ &vnop_pageout_desc, (VOPFUNC)hfs_vnop_pageout },	/* Pageout */
         { &vnop_copyfile_desc, (VOPFUNC)err_copyfile },		/* copyfile */
diff --git a/bsd/hfs/hfscommon/BTree/BTreeNodeReserve.c b/bsd/hfs/hfscommon/BTree/BTreeNodeReserve.c
index f5d9245d2..6310ef09a 100644
--- a/bsd/hfs/hfscommon/BTree/BTreeNodeReserve.c
+++ b/bsd/hfs/hfscommon/BTree/BTreeNodeReserve.c
@@ -97,11 +97,9 @@ BTReserveSetup()
 	nr_hashtbl = hashinit(NR_CACHE, M_HFSMNT, &nr_hashmask);
 
 	nr_lck_grp_attr= lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(nr_lck_grp_attr);
 	nr_lck_grp  = lck_grp_alloc_init("btree_node_reserve", nr_lck_grp_attr);
 
 	nr_lck_attr = lck_attr_alloc_init();
-	lck_attr_setdebug(nr_lck_attr);
 
 	lck_mtx_init(&nr_mutex, nr_lck_grp, nr_lck_attr);
 }
diff --git a/bsd/i386/_types.h b/bsd/i386/_types.h
index c3df6ca3f..227f21176 100644
--- a/bsd/i386/_types.h
+++ b/bsd/i386/_types.h
@@ -39,7 +39,7 @@ typedef __signed char		__int8_t;
 typedef char			__int8_t;
 #endif	/* !__GNUC__ */
 typedef unsigned char		__uint8_t;
-typedef	unsigned short		__int16_t;
+typedef	short			__int16_t;
 typedef	unsigned short		__uint16_t;
 typedef int			__int32_t;
 typedef unsigned int		__uint32_t;
diff --git a/bsd/i386/param.h b/bsd/i386/param.h
index d6a65ea80..69ebef79c 100644
--- a/bsd/i386/param.h
+++ b/bsd/i386/param.h
@@ -87,7 +87,7 @@
 #define	DEV_BSIZE	512
 #define	DEV_BSHIFT	9		/* log2(DEV_BSIZE) */
 #define BLKDEV_IOSIZE	2048
-#define	MAXPHYS		(64 * 1024)	/* max raw I/O transfer size */
+#define	MAXPHYS		(128 * 1024)	/* max raw I/O transfer size */
 
 #define	CLSIZE		1
 #define	CLSIZELOG2	0
diff --git a/bsd/i386/reg.h b/bsd/i386/reg.h
index cf1824662..415533afd 100644
--- a/bsd/i386/reg.h
+++ b/bsd/i386/reg.h
@@ -37,21 +37,6 @@
 #ifndef _BSD_I386_REG_H_
 #define _BSD_I386_REG_H_
 
-/* FIXME - should include mach/i386/thread_status.h and 
-  construct the values from i386_saved_state
- */
-#define	EDX	9
-#define	ECX	10
-#define	EAX	11
-#define	EIP	14
-#define	EFL	16
-#define	ESP	7
-#define	UESP	17
-#define	PS	EFL
-#define	PC	EIP
-#define	SP	UESP
-
-
 
 #endif	/* _BSD_I386_REG_H_ */
 
diff --git a/bsd/i386/setjmp.h b/bsd/i386/setjmp.h
index 715a2945a..0696ed304 100644
--- a/bsd/i386/setjmp.h
+++ b/bsd/i386/setjmp.h
@@ -37,6 +37,20 @@
 #include <sys/cdefs.h>
 #include <machine/signal.h>
 
+
+#if defined(__x86_64__)
+/*
+ * _JBLEN is number of ints required to save the following:
+ * rflags, rip, rbp, rsp, rbx, r12, r13, r14, r15... these are 8 bytes each
+ * mxcsr, fp control word, sigmask... these are 4 bytes each
+ * add 16 ints for future expansion needs...
+ */
+#define _JBLEN ((9 * 2) + 3 + 16)
+typedef int jmp_buf[_JBLEN];
+typedef int sigjmp_buf[_JBLEN + 1];
+
+#else
+
 /*
  * _JBLEN is number of ints required to save the following:
  * eax, ebx, ecx, edx, edi, esi, ebp, esp, ss, eflags, eip,
@@ -54,6 +68,7 @@ typedef int jmp_buf[_JBLEN];
 typedef int sigjmp_buf[_JBLEN + 1];
 #endif
 
+#endif
 
 __BEGIN_DECLS
 extern int setjmp(jmp_buf env);
diff --git a/bsd/i386/types.h b/bsd/i386/types.h
index 3325a7c73..fd53af566 100644
--- a/bsd/i386/types.h
+++ b/bsd/i386/types.h
@@ -94,7 +94,11 @@ typedef	long long		int64_t;
 #endif
 typedef	unsigned long long	u_int64_t;
 
+#if __LP64__
+typedef int64_t			register_t;
+#else
 typedef int32_t			register_t;
+#endif
 
 #ifndef _INTPTR_T
 #define _INTPTR_T
@@ -108,14 +112,17 @@ typedef unsigned long int	uintptr_t;
 /* These types are used for  reserving the largest possible size. */
 // LP64todo - typedef mach_vm_address_t	user_addr_t;	/* varying length pointers from user space */ 
 // LP64todo - typedef mach_vm_size_t		user_size_t;	/* varying length values from user space (unsigned) */
-typedef u_int32_t		user_addr_t;	
-typedef u_int32_t		user_size_t;	
-typedef int32_t			user_ssize_t;
-typedef int32_t			user_long_t;
-typedef u_int32_t		user_ulong_t;
-typedef int32_t			user_time_t;
+typedef u_int64_t		user_addr_t;	
+typedef u_int64_t		user_size_t;	
+typedef int64_t			user_ssize_t;
+typedef int64_t			user_long_t;
+typedef u_int64_t		user_ulong_t;
+typedef int64_t			user_time_t;
 #define USER_ADDR_NULL	((user_addr_t) 0)
-#define CAST_USER_ADDR_T(a_ptr)   ((user_addr_t)(a_ptr))
+#define CAST_USER_ADDR_T(a_ptr)   ((user_addr_t)((uintptr_t)(a_ptr)))
+
+/* This defines the size of syscall arguments after copying into the kernel: */
+typedef u_int64_t		syscall_arg_t;
 
 #ifndef __offsetof
 #define __offsetof(type, field) ((size_t)(&((type *)0)->field))
diff --git a/bsd/i386/ucontext.h b/bsd/i386/ucontext.h
index 0ce0243b3..d99d1add9 100644
--- a/bsd/i386/ucontext.h
+++ b/bsd/i386/ucontext.h
@@ -33,20 +33,29 @@
 #include <sys/appleapiopts.h>
 #include <mach/thread_status.h>
 
-#ifdef __APPLE_API_UNSTABLE
-/* WARNING: THIS WILL CHANGE;  DO NOT COUNT ON THIS */
-/* Needs to be finalized as to what it should contain */
 #ifndef _POSIX_C_SOURCE
 struct mcontext
 #else /* _POSIX_C_SOURCE */
 struct __darwin_mcontext
 #endif /* _POSIX_C_SOURCE */
 {
-	struct sigcontext sc;
+#if __LP64__
+	x86_exception_state64_t	es;
+	x86_thread_state64_t 	ss;	
+	x86_float_state64_t	fs;
+#else
+	x86_exception_state32_t	es;
+	x86_thread_state32_t 	ss;	
+	x86_float_state32_t	fs;
+#endif
 };
 
 #ifndef _POSIX_C_SOURCE
-#define I386_MCONTEXT_SIZE	sizeof(struct mcontext)	
+#if __LP64__
+#define I386_MCONTEXT_SIZE	(x86_THREAD_STATE64_COUNT + x86_FLOAT_STATE64_COUNT + x86_EXCEPTION_STATE64_COUNT) * sizeof(int)
+#else
+#define I386_MCONTEXT_SIZE	(x86_THREAD_STATE32_COUNT + x86_FLOAT_STATE32_COUNT + x86_EXCEPTION_STATE32_COUNT) * sizeof(int)
+#endif
 #endif /* _POSIX_C_SOURCE */
 
 #ifndef _MCONTEXT_T
@@ -54,21 +63,20 @@ struct __darwin_mcontext
 typedef __darwin_mcontext_t	mcontext_t;
 #endif
 
-#ifndef _POSIX_C_SOURCE
 
-struct mcontext64
-{
-	struct sigcontext sc;
+#ifdef XNU_KERNEL_PRIVATE
+struct mcontext64 {
+	x86_exception_state64_t	es;
+	x86_thread_state64_t 	ss;	
+	x86_float_state64_t	fs;
 };
-#define I386_MCONTEXT64_SIZE	sizeof(struct mcontext64)	
 
-#ifndef _MCONTEXT64_T
-#define _MCONTEXT64_T
-typedef struct mcontext64 * mcontext64_t;
+struct mcontext32 {
+	x86_exception_state32_t	es;
+	x86_thread_state32_t 	ss;	
+	x86_float_state32_t	fs;
+};
 #endif
 
-#endif /* _POSIX_C_SOURCE */
-
-#endif /* __APPLE_API_UNSTABLE */
 
 #endif /* _I386_UCONTEXT_H_ */
diff --git a/bsd/i386/vmparam.h b/bsd/i386/vmparam.h
index 3cdbfebda..5998ed40e 100644
--- a/bsd/i386/vmparam.h
+++ b/bsd/i386/vmparam.h
@@ -31,7 +31,10 @@
 
 #include <sys/resource.h>
 
-#define	USRSTACK	(0xC0000000)
+/* Rosetta dependency on this address */
+#define	USRSTACK	VM_USRSTACK32
+
+#define	USRSTACK64	VM_USRSTACK64
 
 /*
  * Virtual memory related constants, all in bytes
@@ -43,10 +46,10 @@
 #define	MAXDSIZ		(RLIM_INFINITY)		/* max data size */
 #endif
 #ifndef	DFLSSIZ
-#define	DFLSSIZ		(8*1024*1024 - 7*4*1024)	/* initial stack size limit */
+#define	DFLSSIZ		(8*1024*1024)		/* initial stack size limit */
 #endif
 #ifndef	MAXSSIZ
-#define	MAXSSIZ		(64*1024*1024 - 7*4*1024)	/* max stack size */
+#define	MAXSSIZ		(64*1024*1024)		/* max stack size */
 #endif
 #ifndef	DFLCSIZ
 #define DFLCSIZ		(0)			/* initial core size limit */
diff --git a/bsd/isofs/cd9660/cd9660_lookup.c b/bsd/isofs/cd9660/cd9660_lookup.c
index 970f2531f..d4e33d4f6 100644
--- a/bsd/isofs/cd9660/cd9660_lookup.c
+++ b/bsd/isofs/cd9660/cd9660_lookup.c
@@ -197,9 +197,6 @@ cd9660_lookup(struct vnop_lookup_args *ap)
 	    !((len == 1 && *name == '.') || (flags & ISDOTDOT))) {
 		int flags1 = UTF_PRECOMPOSED;
 
-		if (BYTE_ORDER != BIG_ENDIAN)
-			flags1 |= UTF_REVERSE_ENDIAN;
-
 		(void) utf8_decodestr(name, len, (u_int16_t*) altname, &altlen,
 					sizeof(altname), 0, flags1);
 		name = altname;
diff --git a/bsd/isofs/cd9660/cd9660_mount.h b/bsd/isofs/cd9660/cd9660_mount.h
index e525b36a4..6361a7198 100644
--- a/bsd/isofs/cd9660/cd9660_mount.h
+++ b/bsd/isofs/cd9660/cd9660_mount.h
@@ -98,21 +98,13 @@ struct iso_args {
  */
 /* LP64todo - should this move? */
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_iso_args {
 	int	flags;			/* mounting flags, see below */
 	int	ssector;		/* starting sector, 0 for 1st session */
 	int	toc_length;		/* Size of *toc, including the toc.length field */
-	user_addr_t toc;
+	user_addr_t toc __attribute((aligned(8)));
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif /* KERNEL */
 
 #endif /* __APPLE_API_UNSTABLE */
diff --git a/bsd/isofs/cd9660/cd9660_util.c b/bsd/isofs/cd9660/cd9660_util.c
index 0a3b9315b..6abbc84bf 100644
--- a/bsd/isofs/cd9660/cd9660_util.c
+++ b/bsd/isofs/cd9660/cd9660_util.c
@@ -91,7 +91,6 @@
 #include <sys/dir.h>
 #include <sys/attr.h>
 #include <kern/assert.h>
-#include <architecture/byte_order.h>
 
 #include <isofs/cd9660/iso.h>
 #include <isofs/cd9660/cd9660_node.h>
@@ -162,6 +161,10 @@ isofncmp(u_char *fn, int fnlen, u_char *isofn, int isolen)
 /*
  * translate and compare a UCS-2 filename
  * Note: Version number plus ';' may be omitted.
+ *
+ * The name pointed to by "fn" is the search name, whose characters are
+ * in native endian order.  The name "ucsfn" is the on-disk name, whose
+ * characters are in big endian order.
  */
 
 int
@@ -177,7 +180,7 @@ ucsfncmp(u_int16_t *fn, int fnlen, u_int16_t *ucsfn, int ucslen)
 	while (--fnlen >= 0) {
 		if (--ucslen < 0)
 			return *fn;
-		if ((c = *ucsfn++) == UCS_SEPARATOR2) {
+		if ((c = OSSwapBigToHostInt16(*ucsfn++)) == UCS_SEPARATOR2) {
 			switch (*fn++) {
 			default:
 				return *--fn;
@@ -191,7 +194,7 @@ ucsfncmp(u_int16_t *fn, int fnlen, u_int16_t *ucsfn, int ucslen)
 					return -1;
 				}
 			}
-			for (j = 0; --ucslen >= 0; j = j * 10 + *ucsfn++ - '0');
+			for (j = 0; --ucslen >= 0; j = j * 10 + OSSwapBigToHostInt16(*ucsfn++) - '0');
 			return i - j;
 		}
 		if (c != *fn)
@@ -202,10 +205,10 @@ ucsfncmp(u_int16_t *fn, int fnlen, u_int16_t *ucsfn, int ucslen)
 		switch (*ucsfn) {
 		default:
 			return -1;
-		case UCS_SEPARATOR1:
-			if (ucsfn[1] != UCS_SEPARATOR2)
+		case OSSwapHostToBigConstInt16(UCS_SEPARATOR1):
+			if (ucsfn[1] != OSSwapHostToBigConstInt16(UCS_SEPARATOR2))
 				return -1;
-		case UCS_SEPARATOR2:
+		case OSSwapHostToBigConstInt16(UCS_SEPARATOR2):
 			return 0;
 		}
 	}
@@ -292,9 +295,9 @@ ucsfntrans(u_int16_t *infn, int infnlen, u_char *outfn, u_short *outfnlen,
 			/* strip file version number */
 			for (fnidx--; fnidx > 0; fnidx--) {
 				/* stop when ';' is found */
-				if (infn[fnidx] == UCS_SEPARATOR2) {
+				if (infn[fnidx] == OSSwapHostToBigConstInt16(UCS_SEPARATOR2)) {
 					/* drop dangling dot */
-					if (fnidx > 0 && infn[fnidx-1] == UCS_SEPARATOR1)
+					if (fnidx > 0 && infn[fnidx-1] == OSSwapHostToBigConstInt16(UCS_SEPARATOR1))
 						fnidx--;
 					break;
 				}
diff --git a/bsd/isofs/cd9660/cd9660_vfsops.c b/bsd/isofs/cd9660/cd9660_vfsops.c
index c2960e23b..7542ca845 100644
--- a/bsd/isofs/cd9660/cd9660_vfsops.c
+++ b/bsd/isofs/cd9660/cd9660_vfsops.c
@@ -85,7 +85,6 @@
 #include <sys/stat.h>
 #include <sys/ubc.h>
 #include <sys/utfconv.h>
-#include <architecture/byte_order.h>
 
 #include <isofs/cd9660/iso.h>
 #include <isofs/cd9660/iso_rrip.h>
@@ -938,7 +937,7 @@ cd9660_fhtovp(mount_t mp, int fhlen, unsigned char *fhp, vnode_t *vpp, vfs_conte
 	       ifhp->ifid_ino, ifhp->ifid_start);
 #endif
 	
-	if ( (error = VFS_VGET(mp, (ino64_t)ifhp->ifid_ino, &nvp, context)) ) {
+	if ( (error = VFS_VGET(mp, (ino64_t)ntohl(ifhp->ifid_ino), &nvp, context)) ) {
 		*vpp = NULLVP;
 		return (error);
 	}
@@ -1598,8 +1597,8 @@ cd9660_vptofh(struct vnode *vp, int *fhlenp, unsigned char *fhp, __unused vfs_co
 	
 	ifhp = (struct ifid *)fhp;
 	
-	ifhp->ifid_ino = ip->i_number;
-	ifhp->ifid_start = ip->iso_start;
+	ifhp->ifid_ino = htonl(ip->i_number);
+	ifhp->ifid_start = htonl(ip->iso_start);
 	*fhlenp = sizeof(struct ifid);
 	
 #ifdef	ISOFS_DBG
diff --git a/bsd/isofs/cd9660/cd9660_vnops.c b/bsd/isofs/cd9660/cd9660_vnops.c
index a63afbf12..599e0844f 100644
--- a/bsd/isofs/cd9660/cd9660_vnops.c
+++ b/bsd/isofs/cd9660/cd9660_vnops.c
@@ -99,7 +99,7 @@
 #include <sys/lock.h>
 #include <sys/ubc_internal.h>
 #include <sys/uio_internal.h>
-#include <architecture/byte_order.h>
+#include <libkern/OSByteOrder.h>
 
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>		/* kmem_alloc, kmem_free */
@@ -981,12 +981,12 @@ cd9660_xa_init(struct iso_node *ip, struct iso_directory_record *isodir)
 	sectors = ip->i_size / 2048;
 
 	strncpy(header->riff, "RIFF", 4);
-	header->fileSize = NXSwapHostLongToLittle(sectors * CDXA_SECTOR_SIZE + sizeof(struct riff_header) - 8);
+	header->fileSize = OSSwapHostToLittleInt32(sectors * CDXA_SECTOR_SIZE + sizeof(struct riff_header) - 8);
 	strncpy(header->cdxa, "CDXA", 4);
 	strncpy(header->fmt, "fmt ", 4);
-	header->fmtSize = NXSwapHostLongToLittle(16);
+	header->fmtSize = OSSwapHostToLittleConstInt32(16);
 	strncpy(header->data, "data", 4);
-	header->dataSize = NXSwapHostLongToLittle(sectors * CDXA_SECTOR_SIZE);
+	header->dataSize = OSSwapHostToLittleInt32(sectors * CDXA_SECTOR_SIZE);
 
 	/*
 	 * Copy the CD-ROM XA extended directory information into the header.  As far as
@@ -1367,7 +1367,6 @@ struct vnodeopv_entry_desc cd9660_specop_entries[] = {
 	{ &vnop_pathconf_desc, (VOPFUNC)spec_pathconf },	/* pathconf */
 	{ &vnop_advlock_desc, (VOPFUNC)spec_advlock },	/* advlock */
 	{ &vnop_bwrite_desc, (VOPFUNC)vn_bwrite },
-	{ &vnop_devblocksize_desc, (VOPFUNC)spec_devblocksize }, /* devblocksize */
 	{ &vnop_pagein_desc, (VOPFUNC)cd9660_pagein },		/* Pagein */
 	{ &vnop_pageout_desc, (VOPFUNC)cd9660_pageout },		/* Pageout */
 	{ &vnop_blktooff_desc, (VOPFUNC)cd9660_blktooff },	/* blktooff */
diff --git a/bsd/kern/bsd_init.c b/bsd/kern/bsd_init.c
index 278b7ba3b..862a0557a 100644
--- a/bsd/kern/bsd_init.c
+++ b/bsd/kern/bsd_init.c
@@ -123,6 +123,8 @@
 
 #include <net/init.h>
 
+#include <machine/exec.h>
+
 extern int app_profile;		/* on/off switch for pre-heat cache */
 
 char    copyright[] =
@@ -162,9 +164,13 @@ char	hostname[MAXHOSTNAMELEN];
 int		hostnamelen;
 char	domainname[MAXDOMNAMELEN];
 int		domainnamelen;
-char	classichandler[32] = {0};  
-uint32_t	classichandler_fsid = -1L;
-long	classichandler_fileid = -1L;
+#if __i386__
+struct exec_archhandler exec_archhandler_ppc = {
+	.path = "/usr/libexec/oah/translate",
+};
+#else /* __i386__ */
+struct exec_archhandler exec_archhandler_ppc;
+#endif /* __i386__ */
 
 char rootdevice[16]; 	/* hfs device names have at least 9 chars */
 
@@ -182,6 +188,7 @@ vm_map_t	mb_map;
 semaphore_t execve_semaphore;
 
 int	cmask = CMASK;
+extern int customnbuf;
 
 int parse_bsd_args(void);
 extern int bsd_hardclockinit;
@@ -199,6 +206,12 @@ extern void sysv_sem_lock_init(void);
 extern void sysv_msg_lock_init(void);
 extern void pshm_lock_init();
 extern void psem_lock_init();
+extern int maxprocperuid;
+
+/* kmem access not enabled by default; can be changed with boot-args */
+int setup_kmem = 0;
+
+extern void stackshot_lock_init();
 
 /*
  * Initialization code.
@@ -267,6 +280,7 @@ bsd_init()
 	extern kauth_cred_t rootcred;
 	register int i;
 	int s;
+	int error;
 	thread_t	th;
 	struct vfs_context context;
 	void		lightning_bolt(void );
@@ -308,17 +322,13 @@ bsd_init()
 	/* give kernproc a name */
 	process_name("kernel_task", p);
 
-
 	/* allocate proc lock group attribute and group */
 	proc_lck_grp_attr= lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(proc_lck_grp_attr);
-
+	
 	proc_lck_grp = lck_grp_alloc_init("proc",  proc_lck_grp_attr);
 
-
 	/* Allocate proc lock attribute */
 	proc_lck_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(proc_lck_attr);
 
 	lck_mtx_init(&p->p_mlock, proc_lck_grp, proc_lck_attr);
 	lck_mtx_init(&p->p_fdmlock, proc_lck_grp, proc_lck_attr);
@@ -390,7 +400,7 @@ bsd_init()
 		limit0.pl_rlimit[i].rlim_cur = 
 			limit0.pl_rlimit[i].rlim_max = RLIM_INFINITY;
 	limit0.pl_rlimit[RLIMIT_NOFILE].rlim_cur = NOFILE;
-	limit0.pl_rlimit[RLIMIT_NPROC].rlim_cur = MAXUPRC;
+	limit0.pl_rlimit[RLIMIT_NPROC].rlim_cur = maxprocperuid;
 	limit0.pl_rlimit[RLIMIT_NPROC].rlim_max = maxproc;
 	limit0.pl_rlimit[RLIMIT_STACK] = vm_initial_limit_stack;
 	limit0.pl_rlimit[RLIMIT_DATA] = vm_initial_limit_data;
@@ -479,6 +489,8 @@ bsd_init()
 	psem_cache_init();
 	time_zone_slock_init();
 
+	/* Stack snapshot facility lock */
+	stackshot_lock_init();
 	/*
 	 * Initialize protocols.  Block reception of incoming packets
 	 * until everything is ready.
@@ -590,6 +602,13 @@ bsd_init()
 
 	bsd_utaskbootstrap();
 
+#if __i386__
+	// this should be done after the root filesystem is mounted
+	error = set_archhandler(p, CPU_TYPE_POWERPC);
+	if (error)
+		exec_archhandler_ppc.path[0] = 0;
+#endif	
+	
 	/* invoke post-root-mount hook */
 	if (mountroot_post_hook != NULL)
 		mountroot_post_hook();
@@ -630,7 +649,6 @@ bsdinit_task(void)
 
 
 	ut = (uthread_t)get_bsdthread_info(th_act);
-	ut->uu_ar0 = (void *)get_user_regs(th_act);
 
 	bsd_hardclockinit = 1;	/* Start bsd hardclock */
 	bsd_init_task = get_threadtask(th_act);
@@ -800,7 +818,9 @@ parse_bsd_args()
 
 	PE_parse_boot_arg("srv", &srv);
 	PE_parse_boot_arg("ncl", &ncl);
-	PE_parse_boot_arg("nbuf", &nbuf);
+	if (PE_parse_boot_arg("nbuf", &max_nbuf_headers)) 
+		customnbuf = 1;
+	PE_parse_boot_arg("kmem", &setup_kmem);
 
 	return 0;
 }
diff --git a/bsd/kern/bsd_stubs.c b/bsd/kern/bsd_stubs.c
index 70f1088aa..cc8e42431 100644
--- a/bsd/kern/bsd_stubs.c
+++ b/bsd/kern/bsd_stubs.c
@@ -37,6 +37,8 @@
 #include <sys/proc_internal.h>
 #include <sys/buf.h>	/* for SET */
 #include <sys/user.h>
+#include <sys/sysent.h>
+#include <sys/sysproto.h>
 
 /* Just to satisfy pstat command */
 int     dmmin, dmmax, dmtext;
@@ -47,7 +49,7 @@ kmem_mb_alloc(vm_map_t  mbmap, int size)
         vm_offset_t addr;
 	if (kernel_memory_allocate(mbmap, &addr, size,
    		0,
-		KMA_NOPAGEWAIT|KMA_KOBJECT) == KERN_SUCCESS)
+		KMA_NOPAGEWAIT|KMA_KOBJECT|KMA_LOMEM) == KERN_SUCCESS)
    			return(addr);
 	else
 		return(0);
@@ -298,3 +300,66 @@ tbeproc(void *procp)
 	return;
 }
 
+
+/* 
+ * WARNING - this is a temporary workaround for binary compatibility issues
+ * with anti-piracy software that relies on patching ptrace (3928003).
+ * This KPI will be removed in the system release after Tiger.
+ */
+uintptr_t temp_patch_ptrace(uintptr_t new_ptrace)
+{
+	struct sysent *		callp;
+	sy_call_t *			old_ptrace;
+#ifndef __ppc__
+	boolean_t	funnel_state;
+#endif
+
+	if (new_ptrace == 0)
+		return(0);
+		
+#ifdef __ppc__
+	enter_funnel_section(kernel_flock);
+#else
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+#endif
+	callp = &sysent[26];
+	old_ptrace = callp->sy_call;
+	
+	/* only allow one patcher of ptrace */
+	if (old_ptrace == (sy_call_t *) ptrace) {
+		callp->sy_call = (sy_call_t *) new_ptrace;
+	}
+	else {
+		old_ptrace = NULL;
+	}
+#ifdef __ppc__
+	exit_funnel_section( );
+#else
+	(void)thread_funnel_set(kernel_flock, funnel_state);
+#endif
+	
+	return((uintptr_t)old_ptrace);
+}
+
+void temp_unpatch_ptrace(void)
+{
+	struct sysent *		callp;
+#ifndef __ppc__
+	boolean_t	funnel_state;
+#endif
+		
+#ifdef __ppc__
+	enter_funnel_section(kernel_flock);
+#else
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+#endif
+	callp = &sysent[26];
+	callp->sy_call = (sy_call_t *) ptrace;
+#ifdef __ppc__
+	exit_funnel_section( );
+#else
+	(void)thread_funnel_set(kernel_flock, funnel_state);
+#endif
+	
+	return;
+}
diff --git a/bsd/kern/init_sysent.c b/bsd/kern/init_sysent.c
index e1a59c1aa..8aaf06fbc 100644
--- a/bsd/kern/init_sysent.c
+++ b/bsd/kern/init_sysent.c
@@ -38,11 +38,7 @@
 #include <sys/types.h>
 #include <sys/sysent.h>
 #include <sys/sysproto.h>
-#ifdef __ppc__
 #define AC(name) (sizeof(struct name) / sizeof(uint64_t))
-#else
-#define AC(name) (sizeof(struct name) / sizeof(register_t))
-#endif
 
 
 /* The casts are bogus but will do for now. */
@@ -154,11 +150,7 @@ __private_extern__ struct sysent sysent[] = {
 	{AC(getpriority_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getpriority, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 100 = getpriority */
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 101 = nosys old send */
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 102 = nosys old recv */
-#ifdef __ppc__
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 103 = nosys old sigreturn */
-#else
-	{AC(sigreturn_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL|UNSAFE_64BIT, (sy_call_t *)sigreturn, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 103 = sigreturn */
-#endif
 	{AC(bind_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)bind, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 104 = bind */
 	{AC(setsockopt_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)setsockopt, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T}, /* 105 = setsockopt */
 	{AC(listen_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)listen, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 106 = listen */
@@ -171,11 +163,7 @@ __private_extern__ struct sysent sysent[] = {
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 113 = nosys old recvmsg */
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 114 = nosys old sendmsg */
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 115 = nosys old vtrace */
-#ifdef __ppc__
-	{AC(ppc_gettimeofday_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)ppc_gettimeofday, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 116 = ppc_gettimeofday */
-#else
 	{AC(gettimeofday_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)gettimeofday, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 116 = gettimeofday */
-#endif
 	{AC(getrusage_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getrusage, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 117 = getrusage */
 	{AC(getsockopt_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)getsockopt, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T}, /* 118 = getsockopt */
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 119 = nosys old resuba */
@@ -251,12 +239,8 @@ __private_extern__ struct sysent sysent[] = {
 	{AC(setgid_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setgid, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 181 = setgid */
 	{AC(setegid_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setegid, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 182 = setegid */
 	{AC(seteuid_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)seteuid, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 183 = seteuid */
-#ifdef __ppc__
 	{AC(sigreturn_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)sigreturn, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 184 = sigreturn */
-#else
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 184 = nosys */
-#endif
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 185 = nosys */
+	{AC(chud_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)chud, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 185 = chud */
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 186 = nosys */
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 187 = nosys */
 	{AC(stat_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)stat, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 188 = stat */
@@ -281,7 +265,6 @@ __private_extern__ struct sysent sysent[] = {
 	{AC(mlock_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)mlock, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 203 = mlock */
 	{AC(munlock_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)munlock, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 204 = munlock */
 	{AC(undelete_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)undelete, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 205 = undelete */
-#ifdef __ppc__
 	{AC(ATsocket_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)ATsocket, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 206 = ATsocket */
 	{AC(ATgetmsg_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)ATgetmsg, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 207 = ATgetmsg */
 	{AC(ATputmsg_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)ATputmsg, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 208 = ATputmsg */
@@ -290,16 +273,6 @@ __private_extern__ struct sysent sysent[] = {
 	{AC(ATPgetreq_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)ATPgetreq, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 211 = ATPgetreq */
 	{AC(ATPgetrsp_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)ATPgetrsp, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 212 = ATPgetrsp */
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 213 = nosys Reserved for AppleTalk */
-#else
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE}, /* 206 = ATsocket */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE}, /* 207 = ATgetmsg */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE}, /* 208 = ATputmsg */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE}, /* 209 = ATPsndreq */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE}, /* 210 = ATPsndrsp */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE}, /* 211 = ATPgetreq */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE}, /* 212 = ATPgetrsp */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 213 = nosys Reserved for AppleTalk */
-#endif /* __ppc__ */
 	{AC(kqueue_from_portset_np_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)kqueue_from_portset_np, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 214 = kqueue_from_portset_np */
 	{AC(kqueue_portset_np_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)kqueue_portset_np, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 215 = kqueue_portset_np */
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE}, /* 216 = mkcomplex soon to be obsolete */
@@ -431,7 +404,7 @@ __private_extern__ struct sysent sysent[] = {
 	{AC(__pthread_canceled_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)__pthread_canceled, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 333 = __pthread_canceled */
 	{AC(__semwait_signal_args), _SYSCALL_CANCEL_POST, NO_FUNNEL, (sy_call_t *)__semwait_signal, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 334 = __semwait_signal */
 	{AC(utrace_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)utrace, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 335 = utrace */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 336 = nosys */
+	{AC(proc_info_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)proc_info, munge_wwwlww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 336 = proc_info */
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 337 = nosys */
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 338 = nosys */
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 339 = nosys */
@@ -460,7 +433,7 @@ __private_extern__ struct sysent sysent[] = {
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)kqueue, NULL, NULL, _SYSCALL_RET_INT_T}, /* 362 = kqueue */
 	{AC(kevent_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)kevent, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 363 = kevent */
 	{AC(lchown_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)lchown, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 364 = lchown */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 365 = nosys */
+	{AC(stack_snapshot_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)stack_snapshot, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 365 = stack_snapshot */
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 366 = nosys */
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 367 = nosys */
 	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 368 = nosys */
diff --git a/bsd/kern/kdebug.c b/bsd/kern/kdebug.c
index c72e598fb..286a9d8ce 100644
--- a/bsd/kern/kdebug.c
+++ b/bsd/kern/kdebug.c
@@ -24,6 +24,7 @@
 
 #include <sys/errno.h>
 #include <sys/param.h>
+#include <sys/systm.h>
 #include <sys/proc_internal.h>
 #include <sys/vm.h>
 #include <sys/sysctl.h>
@@ -41,6 +42,38 @@
 #include <vm/vm_kern.h>
 #include <sys/lock.h>
 
+#include <sys/malloc.h>
+#include <sys/kauth.h>
+
+#include <mach/mach_host.h>		/* for host_info() */
+#include <libkern/OSAtomic.h>
+
+/* XXX should have prototypes, but Mach does not provide one */
+void task_act_iterate_wth_args(task_t, void(*)(thread_t, void *), void *);
+int cpu_number(void);	/* XXX <machine/...> include path broken */
+
+/* XXX should probably be static, but it's debugging code... */
+int kdbg_read(user_addr_t, size_t *);
+void kdbg_control_chud(int, void *);
+int kdbg_control(int *, u_int, user_addr_t, size_t *);
+int kdbg_getentropy (user_addr_t, size_t *, int);
+int kdbg_readmap(user_addr_t, size_t *);
+int kdbg_getreg(kd_regtype *);
+int kdbg_setreg(kd_regtype *);
+int kdbg_setrtcdec(kd_regtype *);
+int kdbg_setpidex(kd_regtype *);
+int kdbg_setpid(kd_regtype *);
+void kdbg_mapinit(void);
+int kdbg_reinit(void);
+int kdbg_bootstrap(void);
+
+static int  create_buffers(void);
+static void delete_buffers(void);
+
+#ifdef ppc
+extern uint32_t maxDec;
+#endif
+
 /* trace enable status */
 unsigned int kdebug_enable = 0;
 
@@ -58,12 +91,27 @@ unsigned int      kd_entropy_buftomem = 0;
 
 unsigned int kdebug_slowcheck=SLOW_NOLOG;
 
-/* kd_buf kd_buffer[kd_bufsize/sizeof(kd_buf)]; */
-kd_buf * kd_bufptr;
-unsigned int kd_buftomem=0;
-kd_buf * kd_buffer=0;
-kd_buf * kd_buflast;
-kd_buf * kd_readlast;
+unsigned int kd_cpus;
+
+struct kd_bufinfo {
+        kd_buf * kd_stop;
+        kd_buf * kd_bufptr;
+        kd_buf * kd_buffer;
+        kd_buf * kd_buflast;
+        kd_buf * kd_readlast;
+        int      kd_wrapped;	/* plus, the global flag KDBG_WRAPPED is set if one of the buffers has wrapped */
+        uint64_t kd_prev_timebase;
+        int	 kd_pad[24];	/* pad out to 128 bytes so that no cache line is shared between CPUs */
+
+};
+
+struct kd_bufinfo *kdbip = NULL;
+
+#define KDCOPYBUF_COUNT	1024
+#define KDCOPYBUF_SIZE	(KDCOPYBUF_COUNT * sizeof(kd_buf))
+kd_buf *kdcopybuf = NULL;
+
+
 unsigned int nkdbufs = 8192;
 unsigned int kd_bufsize = 0;
 unsigned int kdebug_flags = 0;
@@ -74,17 +122,28 @@ unsigned int kdlog_value2=0;
 unsigned int kdlog_value3=0;
 unsigned int kdlog_value4=0;
 
-unsigned long long kd_prev_timebase = 0LL;
+static lck_mtx_t  * kd_trace_mtx_sysctl;
+static lck_grp_t  * kd_trace_mtx_sysctl_grp;
+static lck_attr_t * kd_trace_mtx_sysctl_attr;
+static lck_grp_attr_t   *kd_trace_mtx_sysctl_grp_attr;
+
+static lck_grp_t       *stackshot_subsys_lck_grp;
+static lck_grp_attr_t  *stackshot_subsys_lck_grp_attr;
+static lck_attr_t      *stackshot_subsys_lck_attr;
+static lck_mtx_t        stackshot_subsys_mutex;
+
+void *stackshot_snapbuf = NULL;
+
+int
+stack_snapshot2(pid_t pid, user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t options, register_t *retval);
 
-static lck_mtx_t  * kd_trace_mtx;
-static lck_grp_t  * kd_trace_mtx_grp;
-static lck_attr_t * kd_trace_mtx_attr;
-static lck_grp_attr_t   *kd_trace_mtx_grp_attr;
+extern void
+kdp_snapshot_preflight(int pid, void  *tracebuf, uint32_t tracebuf_size, uint32_t options);
 
-static lck_spin_t * kd_trace_lock;
-static lck_grp_t  * kd_trace_lock_grp;
-static lck_attr_t * kd_trace_lock_attr;
-static lck_grp_attr_t   *kd_trace_lock_grp_attr;
+extern int
+kdp_stack_snapshot_geterror(void);
+extern unsigned int
+kdp_stack_snapshot_bytes_traced(void);
 
 kd_threadmap *kd_mapptr = 0;
 unsigned int kd_mapsize = 0;
@@ -98,7 +157,7 @@ pid_t global_state_pid = -1;       /* Used to control exclusive use of kd_buffer
 /* task to string structure */
 struct tts
 {
-  task_t   *task;            /* from procs task */
+  task_t    task;            /* from procs task */
   pid_t     pid;             /* from procs p_pid  */
   char      task_comm[20];   /* from procs p_comm */
 };
@@ -122,12 +181,11 @@ typedef void (*kd_chudhook_fn) (unsigned int debugid, unsigned int arg1,
 
 kd_chudhook_fn kdebug_chudhook = 0;   /* pointer to CHUD toolkit function */
 
+__private_extern__ void stackshot_lock_init( void );
 
 /* Support syscall SYS_kdebug_trace */
-kdebug_trace(p, uap, retval)
-     struct proc *p;
-     struct kdebug_trace_args *uap;
-     register_t *retval;
+int
+kdebug_trace(__unused struct proc *p, struct kdebug_trace_args *uap, __unused register_t *retval)
 {
     if ( (kdebug_enable == 0) )
         return(EINVAL);
@@ -137,30 +195,92 @@ kdebug_trace(p, uap, retval)
 }
 
 
-void
-kernel_debug(debugid, arg1, arg2, arg3, arg4, arg5)
-unsigned int debugid, arg1, arg2, arg3, arg4, arg5;
+
+static int
+create_buffers(void)
+{
+        unsigned int	cpu, i;
+	int     nentries;
+
+	nentries = nkdbufs / kd_cpus;
+	kd_bufsize = nentries * sizeof(kd_buf);
+
+	bzero((char *)kdbip, sizeof(struct kd_bufinfo) * kd_cpus);
+
+	if (kdcopybuf == 0) {
+	        if (kmem_alloc(kernel_map, (unsigned int *)&kdcopybuf, (vm_size_t)KDCOPYBUF_SIZE) != KERN_SUCCESS)
+		        return ENOMEM;
+	}
+	for (cpu = 0; cpu < kd_cpus; cpu++) {
+	        if (kmem_alloc(kernel_map, (unsigned int *)&kdbip[cpu].kd_buffer, kd_bufsize) != KERN_SUCCESS)
+		        break;
+	}
+	if (cpu < kd_cpus) {
+	        for (i = 0; i < cpu; i++)
+		        kmem_free(kernel_map, (vm_offset_t)kdbip[i].kd_buffer, kd_bufsize);
+		kd_bufsize = 0;
+
+		kmem_free(kernel_map, (vm_offset_t)kdcopybuf, KDCOPYBUF_SIZE);
+		kdcopybuf = NULL;
+		
+		return(ENOMEM);
+	}
+	for (cpu = 0; cpu < kd_cpus; cpu++) {
+		kdbip[cpu].kd_bufptr = kdbip[cpu].kd_buffer;
+		kdbip[cpu].kd_buflast = &kdbip[cpu].kd_bufptr[nentries];
+		kdbip[cpu].kd_readlast = kdbip[cpu].kd_bufptr;
+	}
+	kdebug_flags |= KDBG_BUFINIT;
+
+	return(0);
+}
+
+
+static void
+delete_buffers(void)
 {
+        unsigned int	cpu;
+
+	if (kd_bufsize && (kdebug_flags & KDBG_BUFINIT)) {
+	        for (cpu = 0; cpu < kd_cpus; cpu++)
+		        kmem_free(kernel_map, (vm_offset_t)kdbip[cpu].kd_buffer, kd_bufsize);
+		kd_bufsize = 0;
+	}
+	if (kdcopybuf) {
+		kmem_free(kernel_map, (vm_offset_t)kdcopybuf, KDCOPYBUF_SIZE);
+		kdcopybuf = NULL;
+	}
+	kdebug_flags &= ~KDBG_BUFINIT;
+}
+
+
+static void
+kernel_debug_internal(unsigned int debugid, unsigned int arg1, unsigned int arg2, unsigned int arg3,
+		      unsigned int arg4, unsigned int arg5, int entropy_flag)
+{
+	int s;
 	kd_buf * kd;
 	struct proc *curproc;
-	int      s;
 	unsigned long long now;
+	int cpu;
+	
+	s = ml_set_interrupts_enabled(FALSE);
 
+	now = mach_absolute_time() & KDBG_TIMESTAMP_MASK;
+	cpu = cpu_number();
 
 	if (kdebug_enable & KDEBUG_ENABLE_CHUD) {
 	    if (kdebug_chudhook)
 	        kdebug_chudhook(debugid, arg1, arg2, arg3, arg4, arg5);
 
 	    if ( !(kdebug_enable & (KDEBUG_ENABLE_ENTROPY | KDEBUG_ENABLE_TRACE)))
-	        return;
+	        goto out;
 	}
-	s = ml_set_interrupts_enabled(FALSE);
-	lck_spin_lock(kd_trace_lock);
 
 	if (kdebug_slowcheck == 0)
 	    goto record_trace;
 
-	if (kdebug_enable & KDEBUG_ENABLE_ENTROPY)
+	if (entropy_flag && (kdebug_enable & KDEBUG_ENABLE_ENTROPY))
 	  {
 	    if (kd_entropy_indx < kd_entropy_count)
 	      {
@@ -177,11 +297,7 @@ unsigned int debugid, arg1, arg2, arg3, arg4, arg5;
 	  }
 
 	if ( (kdebug_slowcheck & SLOW_NOLOG) )
-	  {
-	    lck_spin_unlock(kd_trace_lock);
-	    ml_set_interrupts_enabled(s);
-	    return;
-	  }
+	    goto out;
 	
 	if (kdebug_flags & KDBG_PIDCHECK)
 	  {
@@ -189,11 +305,7 @@ unsigned int debugid, arg1, arg2, arg3, arg4, arg5;
 	    curproc = current_proc();
 	    if ((curproc && !(curproc->p_flag & P_KDEBUG)) &&
 		((debugid&0xffff0000) != (MACHDBG_CODE(DBG_MACH_SCHED, 0) | DBG_FUNC_NONE)))
-	      {
-		lck_spin_unlock(kd_trace_lock);
-		ml_set_interrupts_enabled(s);
-		return;
-	      }
+		goto out;
 	  }
 	else if (kdebug_flags & KDBG_PIDEXCLUDE)
 	  {
@@ -201,22 +313,14 @@ unsigned int debugid, arg1, arg2, arg3, arg4, arg5;
 	    curproc = current_proc();
 	    if ((curproc && (curproc->p_flag & P_KDEBUG)) &&
 		((debugid&0xffff0000) != (MACHDBG_CODE(DBG_MACH_SCHED, 0) | DBG_FUNC_NONE)))
-	      {
-		lck_spin_unlock(kd_trace_lock);
-		ml_set_interrupts_enabled(s);
-		return;
-	      }
+		goto out;
 	  }
 
 	if (kdebug_flags & KDBG_RANGECHECK)
 	  {
-	    if ((debugid < kdlog_beg) || (debugid >= kdlog_end) 
-		&& (debugid >> 24 != DBG_TRACE))
-	      {
-		lck_spin_unlock(kd_trace_lock);
-		ml_set_interrupts_enabled(s);
-		return;
-	      }
+	    if ((debugid < kdlog_beg)
+	        || ((debugid >= kdlog_end) && (debugid >> 24 != DBG_TRACE)))
+		goto out;
 	  }
 	else if (kdebug_flags & KDBG_VALCHECK)
 	  {
@@ -225,234 +329,114 @@ unsigned int debugid, arg1, arg2, arg3, arg4, arg5;
 		(debugid & DBG_FUNC_MASK) != kdlog_value3 &&
 		(debugid & DBG_FUNC_MASK) != kdlog_value4 &&
 		(debugid >> 24 != DBG_TRACE))
-	      {
-		lck_spin_unlock(kd_trace_lock);
-		ml_set_interrupts_enabled(s);
-		return;
-	      }
+		goto out;
 	  }
 
 record_trace:
-	kd = kd_bufptr;
+	kd = kdbip[cpu].kd_bufptr;
 	kd->debugid = debugid;
 	kd->arg1 = arg1;
 	kd->arg2 = arg2;
 	kd->arg3 = arg3;
 	kd->arg4 = arg4;
-	kd->arg5 = (int)current_thread();
+	kd->arg5 = arg5;
 	          
-	now = mach_absolute_time() & KDBG_TIMESTAMP_MASK;
-
-	/* Watch for out of order timestamps */	
-
-	if (now < kd_prev_timebase)
+	/*
+	 * Watch for out of order timestamps
+	 */	
+	if (now < kdbip[cpu].kd_prev_timebase)
 	  {
-	    now = ++kd_prev_timebase & KDBG_TIMESTAMP_MASK;
+	    /*
+	     * if so, just store the previous timestamp + a cycle
+	     */
+	    now = ++kdbip[cpu].kd_prev_timebase & KDBG_TIMESTAMP_MASK;
 	  }
 	else
 	  {
-	    /* Then just store the previous timestamp */
-	    kd_prev_timebase = now;
+	    kdbip[cpu].kd_prev_timebase = now;
 	  }
-	kd->timestamp = now | (((uint64_t)cpu_number()) << KDBG_CPU_SHIFT);
+	kd->timestamp = now | (((uint64_t)cpu) << KDBG_CPU_SHIFT);
+
+	kdbip[cpu].kd_bufptr++;
 
-	kd_bufptr++;
+	if (kdbip[cpu].kd_bufptr >= kdbip[cpu].kd_buflast)
+	  	kdbip[cpu].kd_bufptr = kdbip[cpu].kd_buffer;
 
-	if (kd_bufptr >= kd_buflast)
-	  	kd_bufptr = kd_buffer;
-	if (kd_bufptr == kd_readlast) {
+	if (kdbip[cpu].kd_bufptr == kdbip[cpu].kd_readlast) {
 	        if (kdebug_flags & KDBG_NOWRAP)
 			kdebug_slowcheck |= SLOW_NOLOG;
+		kdbip[cpu].kd_wrapped = 1;
 		kdebug_flags |= KDBG_WRAPPED;
 	}
-	lck_spin_unlock(kd_trace_lock);
+
+out:
 	ml_set_interrupts_enabled(s);
 }
 
 void
-kernel_debug1(debugid, arg1, arg2, arg3, arg4, arg5)
-unsigned int debugid, arg1, arg2, arg3, arg4, arg5;
+kernel_debug(unsigned int debugid, unsigned int arg1, unsigned int arg2, unsigned int arg3,
+		      unsigned int arg4, __unused unsigned int arg5)
 {
-	kd_buf * kd;
-	struct proc *curproc;
-	int      s;
-	unsigned long long now;
-
-	if (kdebug_enable & KDEBUG_ENABLE_CHUD) {
-	    if (kdebug_chudhook)
-	        (void)kdebug_chudhook(debugid, arg1, arg2, arg3, arg4, arg5);
+	kernel_debug_internal(debugid, arg1, arg2, arg3, arg4, (int)current_thread(), 1);
+}
 
-	    if ( !(kdebug_enable & (KDEBUG_ENABLE_ENTROPY | KDEBUG_ENABLE_TRACE)))
-	        return;
-	}
-	s = ml_set_interrupts_enabled(FALSE);
-	lck_spin_lock(kd_trace_lock);
+void
+kernel_debug1(unsigned int debugid, unsigned int arg1, unsigned int arg2, unsigned int arg3,
+		      unsigned int arg4, unsigned int arg5)
+{
+	kernel_debug_internal(debugid, arg1, arg2, arg3, arg4, arg5, 0);
+}
 
-	if (kdebug_slowcheck == 0)
-	    goto record_trace1;
+static void
+kdbg_lock_init(void)
+{
+	host_basic_info_data_t hinfo;
+	mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
 
-	if ( (kdebug_slowcheck & SLOW_NOLOG) )
-	  {
-	    lck_spin_unlock(kd_trace_lock);
-	    ml_set_interrupts_enabled(s);
-	    return;
-	  }
 
-	if (kdebug_flags & KDBG_PIDCHECK)
-	  {
-	    /* If kdebug flag is not set for current proc, return  */
-	    curproc = current_proc();
-	    if ((curproc && !(curproc->p_flag & P_KDEBUG)) &&
-		((debugid&0xffff0000) != (MACHDBG_CODE(DBG_MACH_SCHED, 0) | DBG_FUNC_NONE)))
-	      {
-		lck_spin_unlock(kd_trace_lock);
-	        ml_set_interrupts_enabled(s);
-		return;
-	      }
-	  }
-	else if (kdebug_flags & KDBG_PIDEXCLUDE)
-	  {
-	    /* If kdebug flag is set for current proc, return  */
-	    curproc = current_proc();
-	    if ((curproc && (curproc->p_flag & P_KDEBUG)) &&
-		((debugid&0xffff0000) != (MACHDBG_CODE(DBG_MACH_SCHED, 0) | DBG_FUNC_NONE)))
-	      {
-		lck_spin_unlock(kd_trace_lock);
-	        ml_set_interrupts_enabled(s);
+	if (kdebug_flags & KDBG_LOCKINIT)
 		return;
-	      }
-	  }
 
-	if (kdebug_flags & KDBG_RANGECHECK)
-	  {
-	    if ((debugid < kdlog_beg) || (debugid >= kdlog_end)
-		&& (debugid >> 24 != DBG_TRACE))
-	      {
-		lck_spin_unlock(kd_trace_lock);
-		ml_set_interrupts_enabled(s);
-		return;
-	      }
-	  }
-	else if (kdebug_flags & KDBG_VALCHECK)
-	  {
-	    if ((debugid & DBG_FUNC_MASK) != kdlog_value1 &&
-		(debugid & DBG_FUNC_MASK) != kdlog_value2 &&
-		(debugid & DBG_FUNC_MASK) != kdlog_value3 &&
-		(debugid & DBG_FUNC_MASK) != kdlog_value4 &&
-		(debugid >> 24 != DBG_TRACE))
-	      {
-		lck_spin_unlock(kd_trace_lock);
-		ml_set_interrupts_enabled(s);
-		return;
-	      }
-	  }
+	/* get the number of cpus and cache it */
+#define BSD_HOST 1
+	host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
+	kd_cpus = hinfo.physical_cpu_max;
 
-record_trace1:
-	kd = kd_bufptr;
-	kd->debugid = debugid;
-	kd->arg1 = arg1;
-	kd->arg2 = arg2;
-	kd->arg3 = arg3;
-	kd->arg4 = arg4;
-	kd->arg5 = arg5;
-
-	now = mach_absolute_time() & KDBG_TIMESTAMP_MASK;
-
-	/* Watch for out of order timestamps */	
-
-	if (now < kd_prev_timebase)
-	  {
-	    now = ++kd_prev_timebase & KDBG_TIMESTAMP_MASK;
-	  }
-	else
-	  {
-	    /* Then just store the previous timestamp */
-	    kd_prev_timebase = now;
-	  }
-	kd->timestamp = now | (((uint64_t)cpu_number()) << KDBG_CPU_SHIFT);
-
-	kd_bufptr++;
-
-	if (kd_bufptr >= kd_buflast)
-	  	kd_bufptr = kd_buffer;
-	if (kd_bufptr == kd_readlast) {
-	        if (kdebug_flags & KDBG_NOWRAP)
-			kdebug_slowcheck |= SLOW_NOLOG;
-		kdebug_flags |= KDBG_WRAPPED;
-	}
-	lck_spin_unlock(kd_trace_lock);
-	ml_set_interrupts_enabled(s);
-}
-
-
-static void
-kdbg_lock_init()
-{
+	if (kmem_alloc(kernel_map, (unsigned int *)&kdbip, sizeof(struct kd_bufinfo) * kd_cpus) != KERN_SUCCESS)
+	        return;
 
-        if (kdebug_flags & KDBG_LOCKINIT)
-                return;
-        /*
+	/*
 	 * allocate lock group attribute and group
 	 */
-        kd_trace_lock_grp_attr = lck_grp_attr_alloc_init();
-	//lck_grp_attr_setstat(kd_trace_lock_grp_attr);
-	kd_trace_lock_grp = lck_grp_alloc_init("kdebug", kd_trace_lock_grp_attr);
-		
-        kd_trace_mtx_grp_attr = lck_grp_attr_alloc_init();
-	//lck_grp_attr_setstat(kd_trace_mtx_grp_attr);
-	kd_trace_mtx_grp = lck_grp_alloc_init("kdebug", kd_trace_mtx_grp_attr);
+	kd_trace_mtx_sysctl_grp_attr = lck_grp_attr_alloc_init();
+	kd_trace_mtx_sysctl_grp = lck_grp_alloc_init("kdebug", kd_trace_mtx_sysctl_grp_attr);
 		
 	/*
 	 * allocate the lock attribute
 	 */
-	kd_trace_lock_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(kd_trace_lock_attr);
-
-	kd_trace_mtx_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(kd_trace_mtx_attr);
+	kd_trace_mtx_sysctl_attr = lck_attr_alloc_init();
 
 
 	/*
 	 * allocate and initialize spin lock and mutex
 	 */
-	kd_trace_lock  = lck_spin_alloc_init(kd_trace_lock_grp, kd_trace_lock_attr);
-	kd_trace_mtx   = lck_mtx_alloc_init(kd_trace_mtx_grp, kd_trace_mtx_attr);
+	kd_trace_mtx_sysctl = lck_mtx_alloc_init(kd_trace_mtx_sysctl_grp, kd_trace_mtx_sysctl_attr);
 
 	kdebug_flags |= KDBG_LOCKINIT;
 }
 
 
 int
-kdbg_bootstrap()
+kdbg_bootstrap(void)
 {
+        kdebug_flags &= ~KDBG_WRAPPED;
 
-	kd_bufsize = nkdbufs * sizeof(kd_buf);
-
-	if (kmem_alloc(kernel_map, &kd_buftomem,
-			      (vm_size_t)kd_bufsize) == KERN_SUCCESS) 
-	    kd_buffer = (kd_buf *) kd_buftomem;
-	else
-	    kd_buffer= (kd_buf *) 0;
-	kdebug_flags &= ~KDBG_WRAPPED;
-
-	if (kd_buffer) {
-		kdebug_flags |= (KDBG_INIT | KDBG_BUFINIT);
-		kd_bufptr = kd_buffer;
-		kd_buflast = &kd_bufptr[nkdbufs];
-		kd_readlast = kd_bufptr;
-		kd_prev_timebase = 0LL;
-		return(0);
-	} else {
-		kd_bufsize=0;
-		kdebug_flags &= ~(KDBG_INIT | KDBG_BUFINIT);
-		return(EINVAL);
-	}
-	
+	return (create_buffers());
 }
 
-kdbg_reinit()
+int
+kdbg_reinit(void)
 {
-    int s;
     int ret=0;
 
     /*
@@ -460,17 +444,18 @@ kdbg_reinit()
      * First make sure we're not in
      * the middle of cutting a trace
      */
-    s = ml_set_interrupts_enabled(FALSE);
-    lck_spin_lock(kd_trace_lock);
 
     kdebug_enable &= ~KDEBUG_ENABLE_TRACE;
     kdebug_slowcheck |= SLOW_NOLOG;
 
-    lck_spin_unlock(kd_trace_lock);
-    ml_set_interrupts_enabled(s);
+    /*
+     * make sure the SLOW_NOLOG is seen
+     * by everyone that might be trying
+     * to cut a trace..
+     */
+    IOSleep(100);
 
-    if ((kdebug_flags & KDBG_INIT) && (kdebug_flags & KDBG_BUFINIT) && kd_bufsize && kd_buffer)
-        kmem_free(kernel_map, (vm_offset_t)kd_buffer, kd_bufsize);
+    delete_buffers();
 
     if ((kdebug_flags & KDBG_MAPINIT) && kd_mapsize && kd_mapptr)
       {
@@ -481,12 +466,13 @@ kdbg_reinit()
 	kd_mapcount = 0;
       }  
 
-    ret= kdbg_bootstrap();
+    ret = kdbg_bootstrap();
 
     return(ret);
 }
 
-void kdbg_trace_data(struct proc *proc, long *arg_pid)
+void
+kdbg_trace_data(struct proc *proc, long *arg_pid)
 {
     if (!proc)
         *arg_pid = 0;
@@ -497,7 +483,8 @@ void kdbg_trace_data(struct proc *proc, long *arg_pid)
 }
 
 
-void kdbg_trace_string(struct proc *proc, long *arg1, long *arg2, long *arg3, long *arg4)
+void
+kdbg_trace_string(struct proc *proc, long *arg1, long *arg2, long *arg3, long *arg4)
 {
     int i;
     char *dbg_nameptr; 
@@ -521,7 +508,7 @@ void kdbg_trace_string(struct proc *proc, long *arg1, long *arg2, long *arg3, lo
     dbg_parms[2]=0L;
     dbg_parms[3]=0L;
   
-    if(dbg_namelen > sizeof(dbg_parms))
+    if(dbg_namelen > (int)sizeof(dbg_parms))
       dbg_namelen = sizeof(dbg_parms);
     
     for(i=0;dbg_namelen > 0; i++)
@@ -538,9 +525,10 @@ void kdbg_trace_string(struct proc *proc, long *arg1, long *arg2, long *arg3, lo
 }
 
 static void
-kdbg_resolve_map(thread_t th_act, krt_t *t)
+kdbg_resolve_map(thread_t th_act, void *opaque)
 {
   kd_threadmap *mapptr;
+  krt_t *t = (krt_t *)opaque;
 
   if(t->count < t->maxcount)
     {
@@ -563,7 +551,8 @@ kdbg_resolve_map(thread_t th_act, krt_t *t)
     }
 }
 
-void kdbg_mapinit()
+void
+kdbg_mapinit(void)
 {
 	struct proc *p;
 	struct krt akrt;
@@ -631,7 +620,7 @@ void kdbg_mapinit()
 				task_reference(p->task);
 				tts_mapptr[i].task = p->task;
 				tts_mapptr[i].pid  = p->p_pid;
-				(void)strncpy(&tts_mapptr[i].task_comm, p->p_comm, sizeof(tts_mapptr[i].task_comm) - 1);
+				(void)strncpy(tts_mapptr[i].task_comm, p->p_comm, sizeof(tts_mapptr[i].task_comm) - 1);
 				i++;
 			}
 		}
@@ -660,34 +649,31 @@ void kdbg_mapinit()
 static void
 kdbg_clear(void)
 {
-        int s;
-
         /*
 	 * Clean up the trace buffer
 	 * First make sure we're not in
 	 * the middle of cutting a trace
 	 */
-	s = ml_set_interrupts_enabled(FALSE);
-	lck_spin_lock(kd_trace_lock);
 
 	kdebug_enable &= ~KDEBUG_ENABLE_TRACE;
 	kdebug_slowcheck = SLOW_NOLOG;
 
+	/*
+	 * make sure the SLOW_NOLOG is seen
+	 * by everyone that might be trying
+	 * to cut a trace..
+	 */
+	IOSleep(100);
+
 	if (kdebug_enable & KDEBUG_ENABLE_ENTROPY)
 		kdebug_slowcheck |= SLOW_ENTROPY;
 
-	lck_spin_unlock(kd_trace_lock);
-	ml_set_interrupts_enabled(s);
-
         global_state_pid = -1;
-	kdebug_flags &= ~KDBG_BUFINIT;
 	kdebug_flags &= (unsigned int)~KDBG_CKTYPES;
 	kdebug_flags &= ~(KDBG_NOWRAP | KDBG_RANGECHECK | KDBG_VALCHECK);
 	kdebug_flags &= ~(KDBG_PIDCHECK | KDBG_PIDEXCLUDE);
-	kmem_free(kernel_map, (vm_offset_t)kd_buffer, kd_bufsize);
-	kd_buffer = (kd_buf *)0;
-	kd_bufsize = 0;
-	kd_prev_timebase = 0LL;
+
+	delete_buffers();
 
 	/* Clean up the thread map buffer */
 	kdebug_flags &= ~KDBG_MAPINIT;
@@ -697,6 +683,7 @@ kdbg_clear(void)
 	kd_mapcount = 0;
 }
 
+int
 kdbg_setpid(kd_regtype *kdr)
 {
   pid_t pid;
@@ -734,6 +721,7 @@ kdbg_setpid(kd_regtype *kdr)
 }
 
 /* This is for pid exclusion in the trace buffer */
+int
 kdbg_setpidex(kd_regtype *kdr)
 {
   pid_t pid;
@@ -771,6 +759,7 @@ kdbg_setpidex(kd_regtype *kdr)
 }
 
 /* This is for setting a maximum decrementer value */
+int
 kdbg_setrtcdec(kd_regtype *kdr)
 {
   int ret=0;
@@ -782,9 +771,6 @@ kdbg_setrtcdec(kd_regtype *kdr)
 	ret = EINVAL;
 #ifdef ppc
 	else {
-
-		extern uint32_t maxDec;
-
 		maxDec = decval ? decval : 0x7FFFFFFF;	/* Set or reset the max decrementer */
 	}
 #else
@@ -795,9 +781,10 @@ kdbg_setrtcdec(kd_regtype *kdr)
   return(ret);
 }
 
+int
 kdbg_setreg(kd_regtype * kdr)
 {
-	int i,j, ret=0;
+	int ret=0;
 	unsigned int val_1, val_2, val;
 	switch (kdr->type) {
 	
@@ -858,11 +845,13 @@ kdbg_setreg(kd_regtype * kdr)
 	return(ret);
 }
 
-kdbg_getreg(kd_regtype * kdr)
+int
+kdbg_getreg(__unused kd_regtype * kdr)
 {
+#if 0	
 	int i,j, ret=0;
 	unsigned int val_1, val_2, val;
-#if 0	
+
 	switch (kdr->type) {
 	case KDBG_CLASSTYPE :
 		val_1 = (kdr->value1 & 0xff);
@@ -906,7 +895,7 @@ kdbg_readmap(user_addr_t buffer, size_t *number)
 {
   int avail = *number;
   int ret = 0;
-  int count = 0;
+  unsigned int count = 0;
 
   count = avail/sizeof (kd_threadmap);
 
@@ -945,7 +934,6 @@ kdbg_getentropy (user_addr_t buffer, size_t *number, int ms_timeout)
 {
   int avail = *number;
   int ret = 0;
-  int count = 0;     /* The number of timestamp entries that will fill buffer */
 
   if (kd_entropy_buffer)
     return(EBUSY);
@@ -1014,7 +1002,8 @@ kdbg_getentropy (user_addr_t buffer, size_t *number, int ms_timeout)
  *        address of the enabled kdebug_chudhook function
 */
 
-void kdbg_control_chud(int val, void *fn)
+void
+kdbg_control_chud(int val, void *fn)
 {
         if (val) {
                 /* enable chudhook */
@@ -1029,11 +1018,12 @@ void kdbg_control_chud(int val, void *fn)
 }
 
 	
-kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep)
+int
+kdbg_control(int *name, __unused u_int namelen, user_addr_t where, size_t *sizep)
 {
     int ret=0;
-	int size=*sizep;
-	int max_entries;
+	size_t size=*sizep;
+	unsigned int max_entries;
 	unsigned int value = name[1];
 	kd_regtype kd_Reg;
 	kbufinfo_t kd_bufinfo;
@@ -1042,7 +1032,11 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep)
 
 
 	kdbg_lock_init();
-	lck_mtx_lock(kd_trace_mtx);
+
+	if ( !(kdebug_flags & KDBG_LOCKINIT))
+	        return(ENOMEM);
+
+	lck_mtx_lock(kd_trace_mtx_sysctl);
 
 	if (name[0] == KERN_KDGETBUF) {
 	    /* 
@@ -1054,7 +1048,7 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep)
 		 * There is not enough room to return even
 		 * the first element of the info structure.
 		 */
-	        lck_mtx_unlock(kd_trace_mtx);
+	        lck_mtx_unlock(kd_trace_mtx_sysctl);
 
 		return(EINVAL);
 	    }
@@ -1073,7 +1067,7 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep)
 		     * Provide all the info we have
 		    */
 	        if (copyout (&kd_bufinfo, where, sizeof(kd_bufinfo))) {
-		        lck_mtx_unlock(kd_trace_mtx);
+		        lck_mtx_unlock(kd_trace_mtx_sysctl);
 
                 return(EINVAL);
 		    }
@@ -1084,12 +1078,12 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep)
 		 	 * as much info as there is room for.
 		 	 */
 	        if (copyout (&kd_bufinfo, where, size)) {
-		        lck_mtx_unlock(kd_trace_mtx);
+		        lck_mtx_unlock(kd_trace_mtx_sysctl);
 
 		        return(EINVAL);
 		    }
 	    }
-	    lck_mtx_unlock(kd_trace_mtx);
+	    lck_mtx_unlock(kd_trace_mtx_sysctl);
 
 	    return(0);
 	} else if (name[0] == KERN_KDGETENTROPY) {
@@ -1097,15 +1091,15 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep)
 	        ret = EBUSY;
 	    else
 	        ret = kdbg_getentropy(where, sizep, value);
-	    lck_mtx_unlock(kd_trace_mtx);
+	    lck_mtx_unlock(kd_trace_mtx_sysctl);
 
 	    return (ret);
 	}
 	
-	if (curproc = current_proc())
+	if ((curproc = current_proc()) != NULL)
 	    curpid = curproc->p_pid;
 	else {
-	    lck_mtx_unlock(kd_trace_mtx);
+	    lck_mtx_unlock(kd_trace_mtx_sysctl);
 
 	    return (ESRCH);
 	}
@@ -1121,7 +1115,7 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep)
 	        /*
 		 * The global pid exists, deny this request
 		 */
-	        lck_mtx_unlock(kd_trace_mtx);
+	        lck_mtx_unlock(kd_trace_mtx_sysctl);
 
 		return(EBUSY);
 	    }
@@ -1145,6 +1139,8 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep)
 			  ret=EINVAL;
 			  break;
 			}
+		      kdbg_mapinit();
+
 		      kdebug_enable |= KDEBUG_ENABLE_TRACE;
 		      kdebug_slowcheck &= ~SLOW_NOLOG;
 		    }
@@ -1153,7 +1149,6 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep)
 		      kdebug_enable &= ~KDEBUG_ENABLE_TRACE;
 		      kdebug_slowcheck |= SLOW_NOLOG;
 		    }
-		  kdbg_mapinit();
 		  break;
 		case KERN_KDSETBUF:
 		  /* We allow a maximum buffer size of 25% of either ram or max mapped address, whichever is smaller */
@@ -1234,109 +1229,287 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep)
 		default:
 			ret= EINVAL;
 	}
-	lck_mtx_unlock(kd_trace_mtx);
+	lck_mtx_unlock(kd_trace_mtx_sysctl);
 
 	return(ret);
 }
 
+
+/*
+ * This code can run concurrently with kernel_debug_internal()
+ * without the need of any locks, because all reads of kd_bufptr[i],
+ * which get modified by kernel_debug_internal(), are safe.
+ */
+int
 kdbg_read(user_addr_t buffer, size_t *number)
 {
-int avail=*number;
-int count=0;
-int copycount=0;
-int totalcount=0;
-int s;
-unsigned int my_kdebug_flags;
-kd_buf * my_kd_bufptr;
+	unsigned int count;
+	unsigned int cpu;
+	int mincpu;
+	uint64_t  mintime, t, last_wrap_time;
+	int last_wrap_cpu;
+	int error = 0;
+	kd_buf *tempbuf;
+	uint32_t tempbuf_count;
+	uint32_t tempbuf_number;
+	unsigned int old_kdebug_flags, new_kdebug_flags;
+	unsigned int old_kdebug_slowcheck, new_kdebug_slowcheck;
+	count = *number/sizeof(kd_buf);
+	*number = 0;
+
+	if (count == 0 || !(kdebug_flags & KDBG_BUFINIT) || kdcopybuf == 0)
+		return EINVAL;
 
-	s = ml_set_interrupts_enabled(FALSE);
-	lck_spin_lock(kd_trace_lock);
+	/*
+	 * because we hold kd_trace_mtx_sysctl, no other control threads can 
+	 * be playing with kdebug_flags... the code that cuts new events could
+	 * be running, but it only reads kdebug_flags, it doesn't write it..
+	 * use an OSCompareAndSwap to make sure the other processors see the 
+	 * change of state immediately, not to protect against 2 threads racing to update it
+	 */
+	old_kdebug_slowcheck = kdebug_slowcheck;
+	do {
+	        old_kdebug_flags = kdebug_flags;
+		new_kdebug_flags = old_kdebug_flags & ~KDBG_WRAPPED;
+		new_kdebug_flags |= KDBG_NOWRAP;
+	} while ( !OSCompareAndSwap((UInt32)old_kdebug_flags, (UInt32)new_kdebug_flags, (UInt32 *)&kdebug_flags));
+
+	last_wrap_time = 0;
+	last_wrap_cpu  = -1;
+
+	for (cpu = 0; cpu < kd_cpus; cpu++) {
+	        kd_buf *cur_bufptr;
+		
+		if ((cur_bufptr = kdbip[cpu].kd_bufptr) >= kdbip[cpu].kd_buflast)
+		        cur_bufptr = kdbip[cpu].kd_buffer;
 
-	my_kdebug_flags = kdebug_flags;
-	my_kd_bufptr = kd_bufptr;
+		if (kdbip[cpu].kd_wrapped) {
+		        kdbip[cpu].kd_wrapped = 0;
+			kdbip[cpu].kd_readlast = cur_bufptr;
+			kdbip[cpu].kd_stop = cur_bufptr;
 
-	lck_spin_unlock(kd_trace_lock);
-	ml_set_interrupts_enabled(s);
+			if (kd_cpus > 1 && ((cur_bufptr->timestamp & KDBG_TIMESTAMP_MASK) > last_wrap_time)) {
+			        last_wrap_time = cur_bufptr->timestamp & KDBG_TIMESTAMP_MASK;
+				last_wrap_cpu = cpu;
+			}
+		} else {
+		        if (kdbip[cpu].kd_readlast == cur_bufptr)
+			        kdbip[cpu].kd_stop = 0;
+			else
+			        kdbip[cpu].kd_stop = cur_bufptr;
+		}
+	}
+	if (count > nkdbufs)
+		count = nkdbufs;
 
-	count = avail/sizeof(kd_buf);
+	if ((tempbuf_count = count) > KDCOPYBUF_COUNT)
+	        tempbuf_count = KDCOPYBUF_COUNT;
 
-	if (count) {
-		if ((my_kdebug_flags & KDBG_BUFINIT) && kd_bufsize && kd_buffer) {
-			if (count > nkdbufs)
-			        count = nkdbufs;
-			
-			if (!(my_kdebug_flags & KDBG_WRAPPED)) {
-			        if (my_kd_bufptr == kd_readlast) {
-				        *number = 0;
-					return(0);
-				}	
-				if (my_kd_bufptr > kd_readlast) {
-				        copycount = my_kd_bufptr - kd_readlast;
-					if (copycount > count)
-					        copycount = count;
-
-					if (copyout(kd_readlast, buffer, copycount * sizeof(kd_buf))) {
-					        *number = 0;
-						return(EINVAL);
-					}
-					kd_readlast += copycount;
-					*number = copycount;
-					return(0);
+	while (count) {
+	        tempbuf = kdcopybuf;
+		tempbuf_number = 0;
+
+	        while (tempbuf_count) {
+			mintime = 0xffffffffffffffffULL; /* all actual timestamps are below */
+			mincpu = -1;
+
+			for (cpu = 0; cpu < kd_cpus; cpu++) {
+			        if (kdbip[cpu].kd_stop == 0)     /* empty buffer */
+				        continue;
+				t = kdbip[cpu].kd_readlast[0].timestamp & KDBG_TIMESTAMP_MASK;
+
+				if (t < mintime) {
+				        mintime = t;
+					mincpu = cpu;
 				}
 			}
-			if ( (my_kdebug_flags & KDBG_WRAPPED) ) {
-			        /* Note that by setting kd_readlast equal to my_kd_bufptr,
-				 * we now treat the kd_buffer read the same as if we weren't
-				 * wrapped and my_kd_bufptr was less than kd_readlast.
+			if (mincpu < 0)
+			        /*
+				 * all buffers ran empty early
 				 */
-			        kd_readlast = my_kd_bufptr;
-				kdebug_flags &= ~KDBG_WRAPPED;
+			        break;
+
+			if (last_wrap_cpu == mincpu) {
+			        tempbuf->debugid = MISCDBG_CODE(DBG_BUFFER, 0) | DBG_FUNC_NONE;
+				tempbuf->arg1 = 0;
+				tempbuf->arg2 = 0;
+				tempbuf->arg3 = 0;
+				tempbuf->arg4 = 0;
+				tempbuf->arg5 = (int)current_thread();
+	          
+				tempbuf->timestamp = last_wrap_time | (((uint64_t)last_wrap_cpu) << KDBG_CPU_SHIFT);
+
+			        tempbuf++;
+
+				last_wrap_cpu = -1;
+
+			} else {
+			        *(tempbuf++) = kdbip[mincpu].kd_readlast[0];
+
+				kdbip[mincpu].kd_readlast++;
+
+				if (kdbip[mincpu].kd_readlast == kdbip[mincpu].kd_buflast)
+				        kdbip[mincpu].kd_readlast = kdbip[mincpu].kd_buffer;
+				if (kdbip[mincpu].kd_readlast == kdbip[mincpu].kd_stop)
+				        kdbip[mincpu].kd_stop = 0;
 			}
-			/*
-			 * first copyout from readlast to end of kd_buffer
-			 */
-			copycount = kd_buflast - kd_readlast;
-			if (copycount > count)
-			        copycount = count;
-			if (copyout(kd_readlast, buffer, copycount * sizeof(kd_buf))) {
+			tempbuf_count--;
+			tempbuf_number++;
+		}
+		if (tempbuf_number) {
+		        if ((error = copyout(kdcopybuf, buffer, tempbuf_number * sizeof(kd_buf)))) {
 			        *number = 0;
-				return(EINVAL);
-			}
-			buffer += (copycount * sizeof(kd_buf));
-			count -= copycount;
-			totalcount = copycount;
-			kd_readlast += copycount;
-
-			if (kd_readlast == kd_buflast)
-			        kd_readlast = kd_buffer;
-			if (count == 0) {
-				*number = totalcount;
-				return(0);
-			}
-			/* second copyout from top of kd_buffer to bufptr */
-			copycount = my_kd_bufptr - kd_readlast;
-			if (copycount > count)
-			        copycount = count;
-			if (copycount == 0) {
-				*number = totalcount;
-				return(0);
+				error = EINVAL;
+				break;
 			}
-			if (copyout(kd_readlast, buffer, copycount * sizeof(kd_buf)))
-				return(EINVAL);
-
-			kd_readlast += copycount;
-			totalcount += copycount;
-			*number = totalcount;
-			return(0);
+			count   -= tempbuf_number;
+			*number += tempbuf_number;
+			buffer  += (tempbuf_number * sizeof(kd_buf));
+		}
+		if (tempbuf_count)
+		       /*
+			* all trace buffers are empty
+			*/
+		        break;
 
-		} /* end if KDBG_BUFINIT */		
-	} /* end if count */
-	return (EINVAL);
+		if ((tempbuf_count = count) > KDCOPYBUF_COUNT)
+		        tempbuf_count = KDCOPYBUF_COUNT;
+	}
+	if ( !(old_kdebug_flags & KDBG_NOWRAP)) {
+	        do {
+		        old_kdebug_flags = kdebug_flags;
+			new_kdebug_flags = old_kdebug_flags & ~KDBG_NOWRAP;
+		} while ( !OSCompareAndSwap((UInt32)old_kdebug_flags, (UInt32)new_kdebug_flags, (UInt32 *)&kdebug_flags));
+
+		if ( !(old_kdebug_slowcheck & SLOW_NOLOG)) {
+		        do {
+			        old_kdebug_slowcheck = kdebug_slowcheck;
+				new_kdebug_slowcheck = old_kdebug_slowcheck & ~SLOW_NOLOG;
+			} while ( !OSCompareAndSwap((UInt32)old_kdebug_slowcheck, (UInt32)new_kdebug_slowcheck, (UInt32 *)&kdebug_slowcheck));
+		}
+	}
+	return (error);
 }
 
+
 unsigned char *getProcName(struct proc *proc);
 unsigned char *getProcName(struct proc *proc) {
 
 	return (unsigned char *) &proc->p_comm;	/* Return pointer to the proc name */
 
 }
+
+#define STACKSHOT_SUBSYS_LOCK() lck_mtx_lock(&stackshot_subsys_mutex)
+#define STACKSHOT_SUBSYS_UNLOCK() lck_mtx_unlock(&stackshot_subsys_mutex)
+#ifdef __i386__
+#define TRAP_DEBUGGER __asm__ volatile("int3");
+#endif
+#ifdef __ppc__
+#define TRAP_DEBUGGER   __asm__ volatile("tw 4,r3,r3");
+#endif
+
+#define SANE_TRACEBUF_SIZE 2*1024*1024
+
+/* Initialize the mutex governing access to the stack snapshot subsystem */
+__private_extern__ void
+stackshot_lock_init( void )
+{
+	stackshot_subsys_lck_grp_attr = lck_grp_attr_alloc_init();
+
+	stackshot_subsys_lck_grp = lck_grp_alloc_init("stackshot_subsys_lock", stackshot_subsys_lck_grp_attr);
+
+	stackshot_subsys_lck_attr = lck_attr_alloc_init();
+
+	lck_mtx_init(&stackshot_subsys_mutex, stackshot_subsys_lck_grp, stackshot_subsys_lck_attr);
+}
+
+/*
+ * stack_snapshot:   Obtains a coherent set of stack traces for all threads
+ *		     on the system, tracing both kernel and user stacks
+ *		     where available. Uses machine specific trace routines
+ *		     for ppc, ppc64 and x86.
+ * Inputs:	     uap->pid - process id of process to be traced, or -1
+ *		     for the entire system
+ *		     uap->tracebuf - address of the user space destination
+ *		     buffer 
+ *		     uap->tracebuf_size - size of the user space trace buffer
+ *		     uap->options - various options, including the maximum
+ *		     number of frames to trace.
+ * Outputs:	     EPERM if the caller is not privileged
+ *		     EINVAL if the supplied trace buffer isn't sanely sized
+ *		     ENOMEM if we don't have enough memory to satisfy the
+ *		     request
+ *		     ENOENT if the target pid isn't found
+ *		     ENOSPC if the supplied buffer is insufficient
+ *		     *retval contains the number of bytes traced, if successful
+ *		     and -1 otherwise. If the request failed due to
+ *		     tracebuffer exhaustion, we copyout as much as possible.
+ */
+int
+stack_snapshot(struct proc *p, register struct stack_snapshot_args *uap, register_t *retval) {
+	int error = 0;
+
+	if ((error = suser(kauth_cred_get(), &p->p_acflag)))
+                return(error);
+
+	return stack_snapshot2(uap->pid, uap->tracebuf, uap->tracebuf_size,
+	    uap->options, retval);
+}
+
+int
+stack_snapshot2(pid_t pid, user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t options, register_t *retval)
+{
+	int error = 0;
+	unsigned bytesTraced = 0;
+
+	*retval = -1;
+/* Serialize tracing */	
+	STACKSHOT_SUBSYS_LOCK();
+	
+	if ((tracebuf_size <= 0) || (tracebuf_size > SANE_TRACEBUF_SIZE)) {
+		error = EINVAL;
+		goto error_exit;
+	}
+
+	MALLOC(stackshot_snapbuf, void *, tracebuf_size, M_TEMP, M_WAITOK);
+
+	if (stackshot_snapbuf == NULL) {
+		error = ENOMEM;
+		goto error_exit;
+	}
+/* Preload trace parameters*/	
+	kdp_snapshot_preflight(pid, stackshot_snapbuf, tracebuf_size, options);
+
+/* Trap to the debugger to obtain a coherent stack snapshot; this populates
+ * the trace buffer
+ */
+	TRAP_DEBUGGER;
+
+	bytesTraced = kdp_stack_snapshot_bytes_traced();
+			
+	if (bytesTraced > 0) {
+		if ((error = copyout(stackshot_snapbuf, tracebuf,
+			((bytesTraced < tracebuf_size) ?
+			    bytesTraced : tracebuf_size))))
+			goto error_exit;
+		*retval = bytesTraced;
+	}
+	else {
+		error = ENOENT;
+		goto error_exit;
+	}
+
+	error = kdp_stack_snapshot_geterror();
+	if (error == -1) {
+		error = ENOSPC;
+		*retval = -1;
+		goto error_exit;
+	}
+
+error_exit:
+	if (stackshot_snapbuf != NULL)
+		FREE(stackshot_snapbuf, M_TEMP);
+	stackshot_snapbuf = NULL;
+	STACKSHOT_SUBSYS_UNLOCK();
+	return error;
+}
diff --git a/bsd/kern/kern_acct.c b/bsd/kern/kern_acct.c
index 275071025..7488f70db 100644
--- a/bsd/kern/kern_acct.c
+++ b/bsd/kern/kern_acct.c
@@ -203,6 +203,7 @@ acct_process(p)
 	int t;
 	int error;
 	struct vnode *vp;
+	kauth_cred_t safecred;
 
 	/* If accounting isn't enabled, don't bother */
 	vp = acctp;
@@ -253,15 +254,17 @@ acct_process(p)
 	/* (8) The boolean flags that tell how the process terminated, etc. */
 	an_acct.ac_flag = p->p_acflag;
 
+	safecred = kauth_cred_proc_ref(p);
 	/*
 	 * Now, just write the accounting information to the file.
 	 */
 	if ((error = vnode_getwithref(vp)) == 0) {
 	        error = vn_rdwr(UIO_WRITE, vp, (caddr_t)&an_acct, sizeof (an_acct),
-				(off_t)0, UIO_SYSSPACE32, IO_APPEND|IO_UNIT, p->p_ucred,
+				(off_t)0, UIO_SYSSPACE32, IO_APPEND|IO_UNIT, safecred,
 				(int *)0, p);
 		vnode_put(vp);
 	}
+	kauth_cred_unref(&safecred);
 	return (error);
 }
 
diff --git a/bsd/kern/kern_aio.c b/bsd/kern/kern_aio.c
index af4c6fe96..aadb1654d 100644
--- a/bsd/kern/kern_aio.c
+++ b/bsd/kern/kern_aio.c
@@ -2208,10 +2208,8 @@ aio_init( void )
 	int			i;
 	
 	aio_lock_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(aio_lock_grp_attr);
 	aio_lock_grp = lck_grp_alloc_init("aio", aio_lock_grp_attr);
 	aio_lock_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(aio_lock_attr);
 
 	aio_lock = lck_mtx_alloc_init(aio_lock_grp, aio_lock_attr);
 
diff --git a/bsd/kern/kern_audit.c b/bsd/kern/kern_audit.c
index 35a420bfd..280dd281d 100644
--- a/bsd/kern/kern_audit.c
+++ b/bsd/kern/kern_audit.c
@@ -437,7 +437,7 @@ audit_worker(void)
 	AUDIT_PRINTF(("audit_worker starting\n"));
 
 	TAILQ_INIT(&ar_worklist);
-	audit_cred = NULL;
+	audit_cred = NOCRED;
 	audit_p = current_proc();
 	audit_vp = NULL;
 
@@ -466,7 +466,7 @@ audit_worker(void)
 			old_vp = audit_vp;
 			audit_cred = audit_replacement_cred;
 			audit_vp = audit_replacement_vp;
-			audit_replacement_cred = NULL;
+			audit_replacement_cred = NOCRED;
 			audit_replacement_vp = NULL;
 			audit_replacement_flag = 0;
 
@@ -485,8 +485,7 @@ audit_worker(void)
 				AUDIT_PRINTF(("Closing old audit file\n"));
 				vn_close(old_vp, audit_close_flags, old_cred,
 				    audit_p);
-				kauth_cred_rele(old_cred);
-				old_cred = NOCRED;
+				kauth_cred_unref(&old_cred);
 				old_vp = NULL;
 				AUDIT_PRINTF(("Audit file closed\n"));
 			}
@@ -994,8 +993,12 @@ auditon(struct proc *p, __unused struct auditon_args *uap, __unused register_t *
 			
 			my_cred = kauth_cred_proc_ref(tp);
 			/* 
-			 * set the credential with new info.  If there is no change we get back 
-			 * the same credential we passed in.
+			 * Set the credential with new info.  If there is no
+			 * change, we get back the same credential we passed
+			 * in; if there is a change, we drop the reference on
+			 * the credential we passed in.  The subsequent
+			 * compare is safe, because it is a pointer compare
+			 * rather than a contents compare.
 			 */
 			temp_auditinfo = my_cred->cr_au;
 			temp_auditinfo.ai_mask.am_success = 
@@ -1012,16 +1015,15 @@ auditon(struct proc *p, __unused struct auditon_args *uap, __unused register_t *
 				 */
 				if (tp->p_ucred != my_cred) {
 					proc_unlock(tp);
-					kauth_cred_rele(my_cred);
-					kauth_cred_rele(my_new_cred);
+					kauth_cred_unref(&my_new_cred);
 					/* try again */
 					continue;
 				}
 				tp->p_ucred = my_new_cred;
 				proc_unlock(tp);
 			}
-			/* drop our extra reference */
-			kauth_cred_rele(my_cred);
+			/* drop old proc reference or our extra reference */
+			kauth_cred_unref(&my_cred);
 			break;
 		}
 		break;
@@ -1116,8 +1118,11 @@ setauid(struct proc *p, struct setauid_args *uap, __unused register_t *retval)
 		
 		my_cred = kauth_cred_proc_ref(p);
 		/* 
-		 * set the credential with new info.  If there is no change we get back 
-		 * the same credential we passed in.
+		 * Set the credential with new info.  If there is no change,
+		 * we get back the same credential we passed in; if there is
+		 * a change, we drop the reference on the credential we
+		 * passed in.  The subsequent compare is safe, because it is
+		 * a pointer compare rather than a contents compare.
 		 */
 		temp_auditinfo = my_cred->cr_au;
 		temp_auditinfo.ai_auid = temp_au_id;
@@ -1131,16 +1136,15 @@ setauid(struct proc *p, struct setauid_args *uap, __unused register_t *retval)
 			 */
 			if (p->p_ucred != my_cred) {
 				proc_unlock(p);
-				kauth_cred_rele(my_cred);
-				kauth_cred_rele(my_new_cred);
+				kauth_cred_unref(&my_new_cred);
 				/* try again */
 				continue;
 			}
 			p->p_ucred = my_new_cred;
 			proc_unlock(p);
 		}
-		/* drop our extra reference */
-		kauth_cred_rele(my_cred);
+		/* drop old proc reference or our extra reference */
+		kauth_cred_unref(&my_cred);
 		break;
 	}
 
@@ -1187,6 +1191,7 @@ setaudit(struct proc *p, struct setaudit_args *uap, __unused register_t *retval)
 {
 	int error;
 	struct auditinfo temp_auditinfo;
+	kauth_cred_t safecred;
 
 	error = suser(kauth_cred_get(), &p->p_acflag);
 	if (error)
@@ -1210,8 +1215,11 @@ setaudit(struct proc *p, struct setaudit_args *uap, __unused register_t *retval)
 		
 		my_cred = kauth_cred_proc_ref(p);
 		/* 
-		 * set the credential with new info.  If there is no change we get back 
-		 * the same credential we passed in.
+		 * Set the credential with new info.  If there is no change,
+		 * we get back the same credential we passed in; if there is
+		 * a change, we drop the reference on the credential we
+		 * passed in.  The subsequent compare is safe, because it is
+		 * a pointer compare rather than a contents compare.
 		 */
 		my_new_cred = kauth_cred_setauditinfo(my_cred, &temp_auditinfo);
 	
@@ -1223,23 +1231,24 @@ setaudit(struct proc *p, struct setaudit_args *uap, __unused register_t *retval)
 			 */
 			if (p->p_ucred != my_cred) {
 				proc_unlock(p);
-				kauth_cred_rele(my_cred);
-				kauth_cred_rele(my_new_cred);
+				kauth_cred_unref(&my_new_cred);
 				/* try again */
 				continue;
 			}
 			p->p_ucred = my_new_cred;
 			proc_unlock(p);
 		}
-		/* drop our extra reference */
-		kauth_cred_rele(my_cred);
+		/* drop old proc reference or our extra reference */
+		kauth_cred_unref(&my_cred);
 		break;
 	}
 
 	/* propagate the change from the process to Mach task */
 	set_security_token(p);
 
-	audit_arg_auditinfo(&p->p_ucred->cr_au);
+	safecred = kauth_cred_proc_ref(p);
+	audit_arg_auditinfo(&safecred->cr_au);
+	kauth_cred_unref(&safecred);
 
 	return (0);
 }
@@ -1333,6 +1342,7 @@ audit_new(int event, struct proc *p, __unused struct uthread *uthread)
 {
 	struct kaudit_record *ar;
 	int no_record;
+	kauth_cred_t safecred;
 
 	/*
 	 * Eventually, there may be certain classes of events that
@@ -1375,16 +1385,20 @@ audit_new(int event, struct proc *p, __unused struct uthread *uthread)
 	ar->k_ar.ar_event = event;
 	nanotime(&ar->k_ar.ar_starttime);
 
+	safecred = kauth_cred_proc_ref(p);
 	/* Export the subject credential. */
-	cru2x(p->p_ucred, &ar->k_ar.ar_subj_cred);
-	ar->k_ar.ar_subj_ruid = p->p_ucred->cr_ruid;
-	ar->k_ar.ar_subj_rgid = p->p_ucred->cr_rgid;
-	ar->k_ar.ar_subj_egid = p->p_ucred->cr_groups[0];
-	ar->k_ar.ar_subj_auid = p->p_ucred->cr_au.ai_auid;
-	ar->k_ar.ar_subj_asid = p->p_ucred->cr_au.ai_asid;
+	cru2x(safecred, &ar->k_ar.ar_subj_cred);
+
+	ar->k_ar.ar_subj_ruid = safecred->cr_ruid;
+	ar->k_ar.ar_subj_rgid = safecred->cr_rgid;
+	ar->k_ar.ar_subj_egid = safecred->cr_groups[0];
+	ar->k_ar.ar_subj_auid = safecred->cr_au.ai_auid;
+	ar->k_ar.ar_subj_asid = safecred->cr_au.ai_asid;
+	ar->k_ar.ar_subj_amask = safecred->cr_au.ai_mask;
+	ar->k_ar.ar_subj_term = safecred->cr_au.ai_termid;
+	kauth_cred_unref(&safecred);
+
 	ar->k_ar.ar_subj_pid = p->p_pid;
-	ar->k_ar.ar_subj_amask = p->p_ucred->cr_au.ai_mask;
-	ar->k_ar.ar_subj_term = p->p_ucred->cr_au.ai_termid;
 	bcopy(p->p_comm, ar->k_ar.ar_subj_comm, MAXCOMLEN);
 
 	return (ar);
diff --git a/bsd/kern/kern_authorization.c b/bsd/kern/kern_authorization.c
index 8e04ee8bd..93eebc05d 100644
--- a/bsd/kern/kern_authorization.c
+++ b/bsd/kern/kern_authorization.c
@@ -864,14 +864,32 @@ out:
 
 /*
  * Optimistically copy in a kauth_filesec structure
+ *
  * Parameters:	xsecurity		user space kauth_filesec_t
- *		xsecdstpp		pointer to kauth_filesec_t
+ *		xsecdstpp		pointer to kauth_filesec_t to be
+ *					modified to contain the contain a
+ *					pointer to an allocated copy of the
+ *					user space argument
+ *
+ * Returns:	0			Success
+ *		ENOMEM			Insufficient memory for the copy.
+ *		EINVAL			The user space data was invalid, or
+ *					there were too many ACE entries.
+ *		EFAULT			The user space address was invalid;
+ *					this may mean 'fsec_entrycount' in
+ *					the user copy is corrupt/incorrect.
+ *
+ * Implicit returns: xsecdestpp, modified (only if successful!)
+ *
+ * Notes:	The returned kauth_filesec_t is in host byte order
+ *
+ *		The caller is responsible for freeing the returned
+ *		kauth_filesec_t in the success case using the function
+ *		kauth_filesec_free()
  *
- * Returns: 0 on success, EINVAL or EFAULT depending on failure mode.
- * Modifies: xsecdestpp, which contains a pointer to an allocated
- *	     and copied-in kauth_filesec_t
+ *		Our largest initial guess is 32; this needs to move to
+ *		a manifest constant in <sys/kauth.h>.
  */
-
 int
 kauth_copyinfilesec(user_addr_t xsecurity, kauth_filesec_t *xsecdestpp)
 {
@@ -919,6 +937,7 @@ restart:
 	if ((fsec->fsec_entrycount != KAUTH_FILESEC_NOACL) &&
 	    (fsec->fsec_entrycount > count)) {
 		if (fsec->fsec_entrycount > KAUTH_ACL_MAX_ENTRIES) {
+			/* XXX This should be E2BIG */
 			error = EINVAL;
 			goto out;
 		}
@@ -938,7 +957,23 @@ out:
 }
 
 /*
- * Allocate a filesec structure.
+ * Allocate a block of memory containing a filesec structure, immediately
+ * followed by 'count' kauth_ace structures.
+ *
+ * Parameters:	count			Number of kauth_ace structures needed
+ *
+ * Returns:	!NULL			A pointer to the allocated block
+ *		NULL			Invalid 'count' or insufficient memory
+ *
+ * Notes:	Returned memory area assumes that the structures are packed
+ *		densely, so this function may only be used by code that also
+ *		assumes no padding following structures.
+ *
+ *		The returned structure must be freed by the caller using the
+ *		function kauth_filesec_free(), in case we decide to use an
+ *		allocation mechanism that is aware of the object size at some
+ *		point, since the object size is only available by introspecting
+ *		the object itself.
  */
 kauth_filesec_t
 kauth_filesec_alloc(int count)
@@ -960,6 +995,18 @@ kauth_filesec_alloc(int count)
 	return(fsp);
 }	
 
+/*
+ * Free a kauth_filesec_t that was previous allocated, either by a direct
+ * call to kauth_filesec_alloc() or by calling a function that calls it.
+ *
+ * Parameters:	fsp			kauth_filesec_t to free
+ *
+ * Returns:	(void)
+ *
+ * Notes:	The kauth_filesec_t to be freed is assumed to be in host
+ *		byte order so that this function can introspect it in the
+ *		future to determine its size, if necesssary.
+ */
 void
 kauth_filesec_free(kauth_filesec_t fsp)
 {
@@ -972,6 +1019,73 @@ kauth_filesec_free(kauth_filesec_t fsp)
 	FREE(fsp, M_KAUTH);
 }
 
+/*
+ * Set the endianness of a filesec and an ACL; if 'acl' is NULL, use the 
+ * ACL interior to 'fsec' instead.  If the endianness doesn't change, then
+ * this function will have no effect.
+ *
+ * Parameters:	kendian			The endianness to set; this is either
+ *					KAUTH_ENDIAN_HOST or KAUTH_ENDIAN_DISK.
+ *		fsec			The filesec to convert.
+ *		acl			The ACL to convert (optional)
+ *
+ * Returns:	(void)
+ *
+ * Notes:	We use ntohl() because it has a transitive property on Intel
+ *		machines and no effect on PPC mancines.  This guarantees us
+ *		that the swapping only occurs if the endiannes is wrong.
+ */
+void
+kauth_filesec_acl_setendian(int kendian, kauth_filesec_t fsec, kauth_acl_t acl)
+{
+ 	uint32_t	compare_magic = KAUTH_FILESEC_MAGIC;
+	uint32_t	invert_magic = ntohl(KAUTH_FILESEC_MAGIC);
+	uint32_t	compare_acl_entrycount;
+	uint32_t	i;
+
+	if (compare_magic == invert_magic)
+		return;
+
+	/* If no ACL, use ACL interior to 'fsec' instead */
+	if (acl == NULL)
+		acl = &fsec->fsec_acl;
+
+	compare_acl_entrycount = acl->acl_entrycount;
+
+	/*
+	 * Only convert what needs to be converted, and only if the arguments
+	 * are valid.  The following switch and tests effectively reject
+	 * conversions on invalid magic numbers as a desirable side effect.
+	 */
+ 	switch(kendian) {
+	case KAUTH_ENDIAN_HOST:		/* not in host, convert to host */
+		if (fsec->fsec_magic != invert_magic)
+			return;
+		/* acl_entrycount is byteswapped */
+		compare_acl_entrycount = ntohl(acl->acl_entrycount);
+		break;
+	case KAUTH_ENDIAN_DISK:		/* not in disk, convert to disk */
+		if (fsec->fsec_magic != compare_magic)
+			return;
+		break;
+	default:			/* bad argument */
+		return;
+	}
+	
+	/* We are go for conversion */
+	fsec->fsec_magic = ntohl(fsec->fsec_magic);
+	acl->acl_entrycount = ntohl(acl->acl_entrycount);
+	if (compare_acl_entrycount != KAUTH_FILESEC_NOACL) {
+		acl->acl_flags = ntohl(acl->acl_flags);
+
+		/* swap ACE rights and flags */
+		for (i = 0; i < compare_acl_entrycount; i++) {
+			acl->acl_ace[i].ace_flags = ntohl(acl->acl_ace[i].ace_flags);
+			acl->acl_ace[i].ace_rights = ntohl(acl->acl_ace[i].ace_rights);
+		}
+	}
+ }
+
 
 /*
  * Allocate an ACL buffer.
diff --git a/bsd/kern/kern_control.c b/bsd/kern/kern_control.c
index 723cda78e..e742a0deb 100644
--- a/bsd/kern/kern_control.c
+++ b/bsd/kern/kern_control.c
@@ -74,45 +74,6 @@ static lck_attr_t		*ctl_lck_attr = 0;
 static lck_grp_t		*ctl_lck_grp = 0;
 static lck_mtx_t 		*ctl_mtx;
 
-/*
- * internal structure maintained for each register controller 
- */
-
-struct ctl_cb;
-
-struct kctl
-{
-	TAILQ_ENTRY(kctl) 	next;		/* controller chain */
-	
-	/* controller information provided when registering */
-	char 				name[MAX_KCTL_NAME]; /* unique nke identifier, provided by DTS */
-	u_int32_t			id;
-	u_int32_t			reg_unit;
-	
-	/* misc communication information */
-	u_int32_t	flags;			/* support flags */
-	u_int32_t	recvbufsize;		/* request more than the default buffer size */
-	u_int32_t	sendbufsize;		/* request more than the default buffer size */
-	
-	/* Dispatch functions */
-	ctl_connect_func	connect;	/* Make contact */
-	ctl_disconnect_func	disconnect;	/* Break contact */
-	ctl_send_func		send;		/* Send data to nke */
-	ctl_setopt_func		setopt;		/* set kctl configuration */
-	ctl_getopt_func		getopt;		/* get kctl configuration */
-	
-	TAILQ_HEAD(, ctl_cb) 	kcb_head;
-	u_int32_t 				lastunit;
-};
-
-struct ctl_cb {
-	TAILQ_ENTRY(ctl_cb) 	next;		/* controller chain */
-	lck_mtx_t				*mtx;
-	struct socket			*so; 		/* controlling socket */
-	struct kctl				*kctl; 		/* back pointer to controller */
-	u_int32_t 				unit;
-	void					*userdata;
-};
 
 /* all the controllers are chained */
 TAILQ_HEAD(, kctl) 	ctl_head;
@@ -185,7 +146,6 @@ kern_control_init(void)
 			error = ENOMEM;
 			goto done;
 	}
-	lck_grp_attr_setdefault(ctl_lck_grp_attr);
 			
 	ctl_lck_grp = lck_grp_alloc_init("Kernel Control Protocol", ctl_lck_grp_attr);
 	if (ctl_lck_grp == 0) {
@@ -200,7 +160,6 @@ kern_control_init(void)
 			error = ENOMEM;
 			goto done;
 	}
-	lck_attr_setdefault(ctl_lck_attr);
 	
 	ctl_mtx = lck_mtx_alloc_init(ctl_lck_grp, ctl_lck_attr);
 	if (ctl_mtx == 0) {
@@ -926,12 +885,9 @@ static int
 ctl_lock(struct socket *so, int refcount, int lr)
  {
 	int lr_saved;
-#ifdef __ppc__
-	if (lr == 0) {
-			__asm__ volatile("mflr %0" : "=r" (lr_saved));
-	}
+	if (lr == 0) 
+		lr_saved = (unsigned int) __builtin_return_address(0);
 	else lr_saved = lr;
-#endif
 	
 	if (so->so_pcb) {
 		lck_mtx_lock(((struct ctl_cb *)so->so_pcb)->mtx);
@@ -946,7 +902,9 @@ ctl_lock(struct socket *so, int refcount, int lr)
 	
 	if (refcount)
 		so->so_usecount++;
-	so->reserved3 = (void *)lr_saved;
+
+	so->lock_lr[so->next_lock_lr] = (void *)lr_saved;
+	so->next_lock_lr = (so->next_lock_lr+1) % SO_LCKDBG_MAX;
 	return (0);
 }
 
@@ -956,12 +914,9 @@ ctl_unlock(struct socket *so, int refcount, int lr)
 	int lr_saved;
 	lck_mtx_t * mutex_held;
 	
-#ifdef __ppc__
-	if (lr == 0) {
-		__asm__ volatile("mflr %0" : "=r" (lr_saved));
-	}
+	if (lr == 0) 
+		lr_saved = (unsigned int) __builtin_return_address(0);
 	else lr_saved = lr;
-#endif
 	
 #ifdef MORE_KCTLLOCK_DEBUG
 	printf("ctl_unlock: so=%x sopcb=%x lock=%x ref=%x lr=%x\n",
@@ -979,8 +934,9 @@ ctl_unlock(struct socket *so, int refcount, int lr)
 		mutex_held = ((struct ctl_cb *)so->so_pcb)->mtx;
 	}
 	lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
+	so->unlock_lr[so->next_unlock_lr] = (void *)lr_saved;
+	so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX;
 	lck_mtx_unlock(mutex_held);
-	so->reserved4 = (void *)lr_saved;
 	
 	if (so->so_usecount == 0)
 		ctl_sofreelastref(so);
diff --git a/bsd/kern/kern_core.c b/bsd/kern/kern_core.c
index 90d9ad7bf..5a7f15dde 100644
--- a/bsd/kern/kern_core.c
+++ b/bsd/kern/kern_core.c
@@ -79,17 +79,11 @@ mythread_state_flavor_t thread_flavor_array[]={
 int mynum_flavors=4;
 #elif defined (__i386__)
 mythread_state_flavor_t thread_flavor_array [] = { 
-		{i386_THREAD_STATE, i386_THREAD_STATE_COUNT},
-		{i386_THREAD_FPSTATE, i386_THREAD_FPSTATE_COUNT},
-		{i386_THREAD_EXCEPTSTATE, i386_THREAD_EXCEPTSTATE_COUNT},
-		{i386_THREAD_CTHREADSTATE, i386_THREAD_CTHREADSTATE_COUNT},
-		{i386_NEW_THREAD_STATE, i386_NEW_THREAD_STATE_COUNT},
-		{i386_FLOAT_STATE, i386_FLOAT_STATE_COUNT},
-		{i386_ISA_PORT_MAP_STATE, i386_ISA_PORT_MAP_STATE_COUNT},
-		{i386_V86_ASSIST_STATE, i386_V86_ASSIST_STATE_COUNT},
-		{THREAD_SYSCALL_STATE, i386_THREAD_SYSCALL_STATE_COUNT}
+		{x86_THREAD_STATE, x86_THREAD_STATE_COUNT},
+		{x86_FLOAT_STATE, x86_FLOAT_STATE_COUNT},
+		{x86_EXCEPTION_STATE, x86_EXCEPTION_STATE_COUNT},
 		};
-int mynum_flavors=9;
+int mynum_flavors=3;
 
 #else
 #error architecture not supported
diff --git a/bsd/kern/kern_credential.c b/bsd/kern/kern_credential.c
index bdcb8b45a..c553ace8e 100644
--- a/bsd/kern/kern_credential.c
+++ b/bsd/kern/kern_credential.c
@@ -62,7 +62,7 @@
 
 #define CRED_DIAGNOSTIC 1
 
-# define NULLCRED_CHECK(_c)	do {if (((_c) == NOCRED) || ((_c) == FSCRED)) panic("bad credential %p", _c);} while(0)
+# define NULLCRED_CHECK(_c)	do {if (!IS_VALID_CRED(_c)) panic("bad credential %p", _c);} while(0)
 
 /*
  * Interface to external identity resolver.
@@ -108,13 +108,14 @@ static int	kauth_cred_table_size = 0;
 TAILQ_HEAD(kauth_cred_entry_head, ucred);
 static struct kauth_cred_entry_head * kauth_cred_table_anchor = NULL;
 
-#define KAUTH_CRED_HASH_DEBUG 0
+#define KAUTH_CRED_HASH_DEBUG	0
 
 static int kauth_cred_add(kauth_cred_t new_cred);
 static void kauth_cred_remove(kauth_cred_t cred);
 static inline u_long kauth_cred_hash(const uint8_t *datap, int data_len, u_long start_key);
 static u_long kauth_cred_get_hashkey(kauth_cred_t cred);
 static kauth_cred_t kauth_cred_update(kauth_cred_t old_cred, kauth_cred_t new_cred, boolean_t retain_auditinfo);
+static void kauth_cred_unref_hashlocked(kauth_cred_t *credp);
 
 #if KAUTH_CRED_HASH_DEBUG
 static int	kauth_cred_count = 0;
@@ -1442,6 +1443,11 @@ kauth_cred_issuser(kauth_cred_t cred)
 static lck_mtx_t *kauth_cred_hash_mtx;
 #define KAUTH_CRED_HASH_LOCK()		lck_mtx_lock(kauth_cred_hash_mtx);
 #define KAUTH_CRED_HASH_UNLOCK()	lck_mtx_unlock(kauth_cred_hash_mtx);
+#if KAUTH_CRED_HASH_DEBUG
+#define KAUTH_CRED_HASH_LOCK_ASSERT()	_mutex_assert(kauth_cred_hash_mtx, MA_OWNED)
+#else	/* !KAUTH_CRED_HASH_DEBUG */
+#define KAUTH_CRED_HASH_LOCK_ASSERT()
+#endif	/* !KAUTH_CRED_HASH_DEBUG */
 
 void
 kauth_cred_init(void)
@@ -1519,9 +1525,7 @@ kauth_cred_get(void)
 	if (uthread->uu_ucred == NOCRED) {
 		if ((p = (proc_t) get_bsdtask_info(get_threadtask(current_thread()))) == NULL)
 			panic("thread wants credential but has no BSD process");
-		proc_lock(p);
-		kauth_cred_ref(uthread->uu_ucred = p->p_ucred);
-		proc_unlock(p);
+		uthread->uu_ucred = kauth_cred_proc_ref(p);
 	}
 	return(uthread->uu_ucred);
 }
@@ -1547,14 +1551,12 @@ kauth_cred_get_with_ref(void)
 	 * If we later inline this function, the code in this block should probably be
 	 * called out in a function.
 	 */
-	proc_lock(procp);
 	if (uthread->uu_ucred == NOCRED) {
 		/* take reference for new cred in thread */
-		kauth_cred_ref(uthread->uu_ucred = proc_ucred(procp));
+		uthread->uu_ucred = kauth_cred_proc_ref(procp);
 	}
 	/* take a reference for our caller */
 	kauth_cred_ref(uthread->uu_ucred);
-	proc_unlock(procp);
 	return(uthread->uu_ucred);
 }
 
@@ -1581,7 +1583,7 @@ kauth_cred_alloc(void)
 {
 	kauth_cred_t newcred;
 	
-	MALLOC(newcred, kauth_cred_t, sizeof(*newcred), M_KAUTH, M_WAITOK | M_ZERO);
+	MALLOC(newcred, kauth_cred_t, sizeof(*newcred), M_CRED, M_WAITOK | M_ZERO);
 	if (newcred != 0) {
 		newcred->cr_ref = 1;
 		/* must do this, or cred has same group membership as uid 0 */
@@ -1645,7 +1647,7 @@ kauth_cred_create(kauth_cred_t cred)
 			/* retry if kauth_cred_add returns non zero value */
 			if (err == 0)
 				break;
-			FREE(new_cred, M_KAUTH);
+			FREE(new_cred, M_CRED);
 			new_cred = NULL;
 		}
 	}
@@ -1747,6 +1749,7 @@ kauth_cred_setgid(kauth_cred_t cred, gid_t gid)
 	return(kauth_cred_update(cred, &temp_cred, TRUE));
 }
 
+
 /*
  * Update the given credential using the egid argument.  The given gid is used
  *	set the effective user ID.  We only allocate a new credential when the given 
@@ -1936,27 +1939,49 @@ kauth_cred_ref(kauth_cred_t cred)
 
 /*
  * Drop a reference from the passed credential, potentially destroying it.
+ *
+ * Note:	Assumes credential hash is NOT locked
  */
 void
-kauth_cred_rele(kauth_cred_t cred)
+kauth_cred_unref(kauth_cred_t *credp)
 {
-	int		old_value;
+	KAUTH_CRED_HASH_LOCK();
+	kauth_cred_unref_hashlocked(credp);
+	KAUTH_CRED_HASH_UNLOCK();
+}
 
-	NULLCRED_CHECK(cred);
+/*
+ * Drop a reference from the passed credential, potentially destroying it.
+ *
+ * Note:	Assumes credential hash IS locked
+ */
+static void
+kauth_cred_unref_hashlocked(kauth_cred_t *credp)
+{
+	int		old_value;
 
-	KAUTH_CRED_HASH_LOCK();
-	old_value = OSAddAtomic(-1, &cred->cr_ref);
+	KAUTH_CRED_HASH_LOCK_ASSERT();
+	NULLCRED_CHECK(*credp);
+	old_value = OSAddAtomic(-1, &(*credp)->cr_ref);
 
 #if DIAGNOSTIC
 	if (old_value == 0)
-		panic("kauth_cred_rele: dropping a reference on a cred with no references");
+		panic("%s:0x%08x kauth_cred_rele: dropping a reference on a cred with no references", current_proc()->p_comm, *credp);
+	if (old_value == 1)
+		panic("%s:0x%08x kauth_cred_rele: dropping a reference on a cred with no hash entry", current_proc()->p_comm, *credp);
 #endif
 
 	if (old_value < 3) {
 		/* the last reference is our credential hash table */
-		kauth_cred_remove(cred);
+		kauth_cred_remove(*credp);
 	}
-	KAUTH_CRED_HASH_UNLOCK();
+	*credp = NOCRED;
+}
+
+void
+kauth_cred_rele(kauth_cred_t cred)
+{
+	kauth_cred_unref(&cred);
 }
 
 /*
@@ -2004,7 +2029,10 @@ kauth_cred_copy_real(kauth_cred_t cred)
 	bcopy(cred, &temp_cred, sizeof(temp_cred));
 	temp_cred.cr_uid = cred->cr_ruid;
 	temp_cred.cr_groups[0] = cred->cr_rgid;
-	/* if the cred is not opted out, make sure we are using the r/euid for group checks */
+	/*
+	 * if the cred is not opted out, make sure we are using the r/euid
+	 * for group checks
+	 */
 	if (temp_cred.cr_gmuid != KAUTH_UID_NONE)
 		temp_cred.cr_gmuid = cred->cr_ruid;
 
@@ -2019,8 +2047,9 @@ kauth_cred_copy_real(kauth_cred_t cred)
 			return(cred); 
 		}
 		if (found_cred != NULL) {
-			/* found a match so we bump reference count on new one and decrement 
-			 * reference count on the old one.
+			/*
+			 * found a match so we bump reference count on new one.
+			 * we leave the old one alone.
 			 */
 			kauth_cred_ref(found_cred);
 			KAUTH_CRED_HASH_UNLOCK();
@@ -2037,7 +2066,7 @@ kauth_cred_copy_real(kauth_cred_t cred)
 		/* retry if kauth_cred_add returns non zero value */
 		if (err == 0)
 			break;
-		FREE(newcred, M_KAUTH);
+		FREE(newcred, M_CRED);
 		newcred = NULL;
 	}
 	
@@ -2045,12 +2074,16 @@ kauth_cred_copy_real(kauth_cred_t cred)
 }
 	
 /*
- * common code to update a credential.  model_cred is a temporary, non reference
- * counted credential used only for comparison and modeling purposes.  old_cred
- * is a live reference counted credential that we intend to update using model_cred
- * as our model.
+ * Common code to update a credential.  model_cred is a temporary, non
+ * reference counted credential used only for comparison and modeling
+ * purposes.  old_cred is a live reference counted credential that we
+ * intend to update using model_cred as our model.
+ *
+ * IMPORTANT:	If the old_cred ends up updated by this process, we will, as
+ *		a side effect, drop the reference we held on it going in.
  */
-static kauth_cred_t kauth_cred_update(kauth_cred_t old_cred, kauth_cred_t model_cred, boolean_t retain_auditinfo)
+static kauth_cred_t
+kauth_cred_update(kauth_cred_t old_cred, kauth_cred_t model_cred, boolean_t retain_auditinfo)
 {	
 	kauth_cred_t found_cred, new_cred = NULL;
 	
@@ -2071,12 +2104,13 @@ static kauth_cred_t kauth_cred_update(kauth_cred_t old_cred, kauth_cred_t model_
 			return(old_cred); 
 		}
 		if (found_cred != NULL) {
-			/* found a match so we bump reference count on new one and decrement 
-			 * reference count on the old one.
+			/*
+			 * found a match so we bump reference count on new
+			 * one and decrement reference count on the old one.
 			 */
 			kauth_cred_ref(found_cred);
+			kauth_cred_unref_hashlocked(&old_cred);
 			KAUTH_CRED_HASH_UNLOCK();
-			kauth_cred_rele(old_cred);
 			return(found_cred);
 		}
 	
@@ -2090,11 +2124,11 @@ static kauth_cred_t kauth_cred_update(kauth_cred_t old_cred, kauth_cred_t model_
 		/* retry if kauth_cred_add returns non zero value */
 		if (err == 0)
 			break;
-		FREE(new_cred, M_KAUTH);
+		FREE(new_cred, M_CRED);
 		new_cred = NULL;
 	}
 
-	kauth_cred_rele(old_cred);
+	kauth_cred_unref(&old_cred);
 	return(new_cred);
 }
 
@@ -2103,10 +2137,13 @@ static kauth_cred_t kauth_cred_update(kauth_cred_t old_cred, kauth_cred_t model_
  *	reference to account for our use of the credential in the hash table.
  *	NOTE - expects caller to hold KAUTH_CRED_HASH_LOCK!
  */
-static int kauth_cred_add(kauth_cred_t new_cred)
+static int
+kauth_cred_add(kauth_cred_t new_cred)
 {
 	u_long			hash_key;
-	
+
+	KAUTH_CRED_HASH_LOCK_ASSERT();
+
 	hash_key = kauth_cred_get_hashkey(new_cred);
 	hash_key %= kauth_cred_table_size;
 
@@ -2128,30 +2165,35 @@ static int kauth_cred_add(kauth_cred_t new_cred)
 	return(0);
 }
 
+
 /* 
  *	Remove the given credential from our credential hash table.
  *	NOTE - expects caller to hold KAUTH_CRED_HASH_LOCK!
  */
-static void kauth_cred_remove(kauth_cred_t cred)
+static void
+kauth_cred_remove(kauth_cred_t cred)
 {
 	u_long			hash_key;
 	kauth_cred_t	found_cred;
 
+	KAUTH_CRED_HASH_LOCK_ASSERT();
+
 	hash_key = kauth_cred_get_hashkey(cred);
 	hash_key %= kauth_cred_table_size;
 
 	/* avoid race */
 	if (cred->cr_ref < 1)
 		panic("cred reference underflow");
-	if (cred->cr_ref > 1)
+	if (cred->cr_ref > 1) {
 		return;		/* someone else got a ref */
-		
+	}
+
 	/* find cred in the credential hash table */
 	TAILQ_FOREACH(found_cred, &kauth_cred_table_anchor[hash_key], cr_link) {
 		if (found_cred == cred) {
 			/* found a match, remove it from the hash table */
 			TAILQ_REMOVE(&kauth_cred_table_anchor[hash_key], found_cred, cr_link);
-			FREE(cred, M_KAUTH);
+			FREE(cred, M_CRED);
 #if KAUTH_CRED_HASH_DEBUG
 			kauth_cred_count--;
 #endif
@@ -2160,7 +2202,7 @@ static void kauth_cred_remove(kauth_cred_t cred)
 	}
 
 	/* did not find a match.  this should not happen! */
-	printf("%s - %d - %s - did not find a match \n", __FILE__, __LINE__, __FUNCTION__);
+	printf("%s:%s - %d - %s - did not find a match for 0x%08x\n", __FILE__, __LINE__, __FUNCTION__, current_proc()->p_comm, cred);
 	return;
 }
 
@@ -2169,11 +2211,14 @@ static void kauth_cred_remove(kauth_cred_t cred)
  *	table.
  *	NOTE - expects caller to hold KAUTH_CRED_HASH_LOCK!
  */
-kauth_cred_t kauth_cred_find(kauth_cred_t cred)
+kauth_cred_t
+kauth_cred_find(kauth_cred_t cred)
 {
 	u_long			hash_key;
 	kauth_cred_t	found_cred;
 	
+	KAUTH_CRED_HASH_LOCK_ASSERT();
+
 #if KAUTH_CRED_HASH_DEBUG
 	static int		test_count = 0; 
 
@@ -2200,12 +2245,13 @@ kauth_cred_t kauth_cred_find(kauth_cred_t cred)
 /*
  * Generates a hash key using data that makes up a credential.  Based on ElfHash.
  */
-static u_long kauth_cred_get_hashkey(kauth_cred_t cred)
+static u_long
+kauth_cred_get_hashkey(kauth_cred_t cred)
 {
 	u_long	hash_key = 0;
 	
 	hash_key = kauth_cred_hash((uint8_t *)&cred->cr_uid, 
-							   (sizeof(struct ucred) - offsetof(struct ucred, cr_uid)), 
+				   (sizeof(struct ucred) - offsetof(struct ucred, cr_uid)), 
 							   hash_key);
 	return(hash_key);
 }
diff --git a/bsd/kern/kern_descrip.c b/bsd/kern/kern_descrip.c
index e147a30bf..0f693756a 100644
--- a/bsd/kern/kern_descrip.c
+++ b/bsd/kern/kern_descrip.c
@@ -112,7 +112,6 @@ int fdgetf_noref(struct proc *p, int fd, struct fileproc **resultfp);
 void fg_drop(struct fileproc * fp);
 void fg_free(struct fileglob *fg);
 void fg_ref(struct fileproc * fp);
-int fp_getfpshm(struct proc *p, int fd, struct fileproc **resultfp, struct pshmnode  **resultpshm);
 
 static int closef_finish(struct fileproc *fp, struct fileglob *fg, struct proc *p);
 
@@ -155,13 +154,11 @@ file_lock_init(void)
 
 	/* allocate file lock group attribute and group */
 	file_lck_grp_attr= lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(file_lck_grp_attr);
 
 	file_lck_grp = lck_grp_alloc_init("file",  file_lck_grp_attr);
 
 	/* Allocate file lock attribute */
 	file_lck_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(file_lck_attr);
 
 	uipc_lock = lck_mtx_alloc_init(file_lck_grp, file_lck_attr);
 	file_iterate_lcok = lck_mtx_alloc_init(file_lck_grp, file_lck_attr);
@@ -1508,6 +1505,42 @@ fp_getfvp(p, fd, resultfp, resultvp)
 }
 
 
+
+int
+fp_getfvpandvid(p, fd, resultfp, resultvp, vidp)
+	struct proc *p;
+	int fd;
+	struct fileproc **resultfp;
+	struct vnode  **resultvp;
+	uint32_t * vidp;
+{
+	struct filedesc *fdp = p->p_fd;
+	struct fileproc *fp;
+
+	proc_fdlock(p);
+	if (fd < 0 || fd >= fdp->fd_nfiles ||
+			(fp = fdp->fd_ofiles[fd]) == NULL ||
+			(fdp->fd_ofileflags[fd] & UF_RESERVED)) {
+		proc_fdunlock(p);
+		return (EBADF);
+	}
+	if (fp->f_type != DTYPE_VNODE) {
+		proc_fdunlock(p);
+		return(ENOTSUP);
+	}
+	fp->f_iocount++;
+
+	if (resultfp)
+		*resultfp = fp;
+	if (resultvp)
+		*resultvp = (struct vnode *)fp->f_data;
+	if (vidp)
+		*vidp = (uint32_t)vnode_vid((struct vnode *)fp->f_data);
+	proc_fdunlock(p);
+
+	return (0);
+}
+
 /*
  * Returns:	EBADF			The file descriptor is invalid
  *		EOPNOTSUPP		The file descriptor is not a socket
@@ -1646,6 +1679,74 @@ fp_getfpsem(p, fd, resultfp, resultpsem)
 
 	return (0);
 }
+
+
+int
+fp_getfpipe(p, fd, resultfp, resultpipe)
+	struct proc *p;
+	int fd;
+	struct fileproc **resultfp;
+	struct pipe  **resultpipe;
+{
+	struct filedesc *fdp = p->p_fd;
+	struct fileproc *fp;
+
+	proc_fdlock(p);
+	if (fd < 0 || fd >= fdp->fd_nfiles ||
+			(fp = fdp->fd_ofiles[fd]) == NULL ||
+			(fdp->fd_ofileflags[fd] & UF_RESERVED)) {
+		proc_fdunlock(p);
+		return (EBADF);
+	}
+	if (fp->f_type != DTYPE_PIPE) {
+		proc_fdunlock(p);
+		return(EBADF);
+	}
+	fp->f_iocount++;
+
+	if (resultfp)
+		*resultfp = fp;
+	if (resultpipe)
+		*resultpipe = (struct pipe *)fp->f_data;
+	proc_fdunlock(p);
+
+	return (0);
+}
+
+
+#define DTYPE_ATALK -1
+int
+fp_getfatalk(p, fd, resultfp, resultatalk)
+	struct proc *p;
+	int fd;
+	struct fileproc **resultfp;
+	struct atalk  **resultatalk;
+{
+	struct filedesc *fdp = p->p_fd;
+	struct fileproc *fp;
+
+	proc_fdlock(p);
+	if (fd < 0 || fd >= fdp->fd_nfiles ||
+			(fp = fdp->fd_ofiles[fd]) == NULL ||
+			(fdp->fd_ofileflags[fd] & UF_RESERVED)) {
+		proc_fdunlock(p);
+		return (EBADF);
+	}
+	if (fp->f_type != (DTYPE_ATALK+1)) {
+		proc_fdunlock(p);
+		return(EBADF);
+	}
+	fp->f_iocount++;
+
+	if (resultfp)
+		*resultfp = fp;
+	if (resultatalk)
+		*resultatalk = (struct atalk *)fp->f_data;
+	proc_fdunlock(p);
+
+	return (0);
+}
+
 int
 fp_lookup(p, fd, resultfp, locked)
 	struct proc *p;
@@ -1928,17 +2029,13 @@ void
 fg_free(fg)
 	struct fileglob *fg;
 {
-	kauth_cred_t cred;
-
 	lck_mtx_lock(file_flist_lock);
 	LIST_REMOVE(fg, f_list);
 	nfiles--;
 	lck_mtx_unlock(file_flist_lock);
 
-	cred = fg->fg_cred;
-	if (cred != NOCRED) {
-		fg->fg_cred = NOCRED;
-		kauth_cred_rele(cred);
+	if (IS_VALID_CRED(fg->fg_cred)) {
+		kauth_cred_unref(&fg->fg_cred);
 	}
 	lck_mtx_destroy(&fg->fg_lock, file_lck_grp);
 
diff --git a/bsd/kern/kern_event.c b/bsd/kern/kern_event.c
index ae5b11c70..e785269ad 100644
--- a/bsd/kern/kern_event.c
+++ b/bsd/kern/kern_event.c
@@ -79,6 +79,7 @@
 #include <sys/sysproto.h>
 #include <sys/user.h>
 #include <string.h>
+#include <sys/proc_info.h>
 
 #include <kern/lock.h>
 #include <kern/clock.h>
@@ -417,6 +418,7 @@ filt_procattach(struct knote *kn)
 	}
 
 	kn->kn_flags |= EV_CLEAR;		/* automatically set */
+	kn->kn_hookid = 1;			/* mark exit not seen */
 
 	/*
 	 * internal flag indicating registration done by kernel
@@ -437,11 +439,12 @@ filt_procattach(struct knote *kn)
 
 /*
  * The knote may be attached to a different process, which may exit,
- * leaving nothing for the knote to be attached to.  So when the process
- * exits, the knote is marked as DETACHED and also flagged as ONESHOT so
- * it will be deleted when read out.  However, as part of the knote deletion,
- * this routine is called, so a check is needed to avoid actually performing
- * a detach, because the original process does not exist any more.
+ * leaving nothing for the knote to be attached to.  In that case,
+ * we wont be able to find the process from its pid.  But the exit
+ * code may still be processing the knote list for the target process.
+ * We may have to wait for that processing to complete before we can
+ * return (and presumably free the knote) without actually removing
+ * it from the dead process' knote list.
  */
 static void
 filt_procdetach(struct knote *kn)
@@ -452,66 +455,81 @@ filt_procdetach(struct knote *kn)
 	funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	p = pfind(kn->kn_id);
 
-	if (p != (struct proc *)NULL)
+	if (p != (struct proc *)NULL) {
 		KNOTE_DETACH(&p->p_klist, kn);
-
+	} else if (kn->kn_hookid != 0) {	/* if not NOTE_EXIT yet */
+		kn->kn_hookid = -1;	/* we are detaching but... */
+		assert_wait(&kn->kn_hook, THREAD_UNINT); /* have to wait */
+		thread_block(THREAD_CONTINUE_NULL);
+	}
 	thread_funnel_set(kernel_flock, funnel_state);
 }
 
 static int
 filt_proc(struct knote *kn, long hint)
 {
-	u_int event;
-	int funnel_state;
 
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+	if (hint != 0) {
+		u_int event;
 
-	/*
-	 * mask off extra data
-	 */
-	event = (u_int)hint & NOTE_PCTRLMASK;
+		/* must hold the funnel when coming from below */
+		assert(thread_funnel_get() != (funnel_t)0);
 
-	/*
-	 * if the user is interested in this event, record it.
-	 */
-	if (kn->kn_sfflags & event)
-		kn->kn_fflags |= event;
+		/*
+		 * mask off extra data
+		 */
+		event = (u_int)hint & NOTE_PCTRLMASK;
 
-	/*
-	 * process is gone, so flag the event as finished.
-	 */
-	if (event == NOTE_EXIT) {
-		kn->kn_flags |= (EV_EOF | EV_ONESHOT); 
-		thread_funnel_set(kernel_flock, funnel_state);
-		return (1);
-	}
+		/*
+		 * if the user is interested in this event, record it.
+		 */
+		if (kn->kn_sfflags & event)
+			kn->kn_fflags |= event;
 
-	/*
-	 * process forked, and user wants to track the new process,
-	 * so attach a new knote to it, and immediately report an
-	 * event with the parent's pid.
-	 */
-	if ((event == NOTE_FORK) && (kn->kn_sfflags & NOTE_TRACK)) {
-		struct kevent kev;
-		int error;
+		/*
+		 * process is gone, so flag the event as finished.
+		 *
+		 * If someone was trying to detach, but couldn't
+		 * find the proc to complete the detach, wake them
+		 * up (nothing will ever need to walk the per-proc
+		 * knote list again - so its safe for them to dump
+		 * the knote now).
+		 */
+		if (event == NOTE_EXIT) {
+			boolean_t detaching = (kn->kn_hookid == -1);
+
+			kn->kn_hookid = 0;
+			kn->kn_flags |= (EV_EOF | EV_ONESHOT); 
+			if (detaching)
+				thread_wakeup(&kn->kn_hookid);
+			return (1);
+		}
 
 		/*
-		 * register knote with new process.
+		 * process forked, and user wants to track the new process,
+		 * so attach a new knote to it, and immediately report an
+		 * event with the parent's pid.
 		 */
-		kev.ident = hint & NOTE_PDATAMASK;	/* pid */
-		kev.filter = kn->kn_filter;
-		kev.flags = kn->kn_flags | EV_ADD | EV_ENABLE | EV_FLAG1;
-		kev.fflags = kn->kn_sfflags;
-		kev.data = kn->kn_id;			/* parent */
-		kev.udata = kn->kn_kevent.udata;	/* preserve udata */
-		error = kevent_register(kn->kn_kq, &kev, NULL);
-		if (error)
-			kn->kn_fflags |= NOTE_TRACKERR;
+		if ((event == NOTE_FORK) && (kn->kn_sfflags & NOTE_TRACK)) {
+			struct kevent kev;
+			int error;
+
+			/*
+			 * register knote with new process.
+			 */
+			kev.ident = hint & NOTE_PDATAMASK;	/* pid */
+			kev.filter = kn->kn_filter;
+			kev.flags = kn->kn_flags | EV_ADD | EV_ENABLE | EV_FLAG1;
+			kev.fflags = kn->kn_sfflags;
+			kev.data = kn->kn_id;			/* parent */
+			kev.udata = kn->kn_kevent.udata;	/* preserve udata */
+			error = kevent_register(kn->kn_kq, &kev, NULL);
+			if (error)
+				kn->kn_fflags |= NOTE_TRACKERR;
+		}
 	}
-	event = kn->kn_fflags;
-	thread_funnel_set(kernel_flock, funnel_state);
 
-	return (event != 0);
+	return (kn->kn_fflags != 0); /* atomic check - no funnel needed from above */
 }
 
 /*
@@ -1357,42 +1375,46 @@ kevent_process(struct kqueue *kq,
 	       (kn = TAILQ_FIRST(&kq->kq_head)) != NULL) {
 
 		/*
-		 * move knote to the processed queue.
-		 * this is also protected by the kq lock.
-		 */
-		assert(kn->kn_tq == &kq->kq_head);
-		TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe);
-		kn->kn_tq = &kq->kq_inprocess;
-		TAILQ_INSERT_TAIL(&kq->kq_inprocess, kn, kn_tqe);
-
-		/*
+		 * Take note off the active queue.
+		 *
 		 * Non-EV_ONESHOT events must be re-validated.
 		 *
 		 * Convert our lock to a use-count and call the event's
 		 * filter routine to update.
 		 *
-		 * If the event is dropping (or no longer valid), we
-		 * already have it off the active queue, so just
-		 * finish the job of deactivating it.
+		 * If the event is valid, or triggered while the kq
+		 * is unlocked, move to the inprocess queue for processing.
 		 */
+
 		if ((kn->kn_flags & EV_ONESHOT) == 0) {
 			int result;
+			knote_deactivate(kn);
 
 			if (kqlock2knoteuse(kq, kn)) {
 				
 				/* call the filter with just a ref */
 				result = kn->kn_fop->f_event(kn, 0);
 
-				if (!knoteuse2kqlock(kq, kn) || result == 0) {
-					knote_deactivate(kn);
+				/* if it's still alive, make sure it's active */
+				if (knoteuse2kqlock(kq, kn) && result) {
+					/* may have been reactivated in filter*/
+					if (!(kn->kn_status & KN_ACTIVE)) {
+						knote_activate(kn);
+					}
+				} else {
 					continue;
 				}
 			} else {
-				knote_deactivate(kn);
 				continue;
 			}
 		}
 
+		/* knote is active: move onto inprocess queue */
+		assert(kn->kn_tq == &kq->kq_head);
+		TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe);
+		kn->kn_tq = &kq->kq_inprocess;
+		TAILQ_INSERT_TAIL(&kq->kq_inprocess, kn, kn_tqe);
+
 		/*
 		 * Got a valid triggered knote with the kqueue
 		 * still locked.  Snapshot the data, and determine
@@ -1953,13 +1975,11 @@ knote_init(void)
 
 	/* allocate kq lock group attribute and group */
 	kq_lck_grp_attr= lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(kq_lck_grp_attr);
 
 	kq_lck_grp = lck_grp_alloc_init("kqueue",  kq_lck_grp_attr);
 
 	/* Allocate kq lock attribute */
 	kq_lck_attr = lck_attr_alloc_init();
-	lck_attr_setdefault(kq_lck_attr);
 
 	/* Initialize the timer filter lock */
 	lck_mtx_init(&_filt_timerlock, kq_lck_grp, kq_lck_attr);
@@ -2053,7 +2073,6 @@ kern_event_init(void)
 	 * allocate the lock attribute for mutexes
 	 */
 	evt_mtx_attr = lck_attr_alloc_init();
-	lck_attr_setdefault(evt_mtx_attr);
   	evt_mutex = lck_mtx_alloc_init(evt_mtx_grp, evt_mtx_attr);
 	if (evt_mutex == NULL)
 			return (ENOMEM);
@@ -2278,4 +2297,23 @@ kev_control(struct socket *so,
 
 
 
+int
+fill_kqueueinfo(struct kqueue *kq, struct kqueue_info * kinfo)
+{
+	struct stat * st;
+
+	/* No need for the funnel as fd is kept alive */
+	
+	st = &kinfo->kq_stat;
+
+	st->st_size = kq->kq_count;
+	st->st_blksize = sizeof(struct kevent);
+	st->st_mode = S_IFIFO;
+	if (kq->kq_state & KQ_SEL)
+		kinfo->kq_state |=  PROC_KQUEUE_SELECT;
+	if (kq->kq_state & KQ_SLEEP)
+		kinfo->kq_state |= PROC_KQUEUE_SLEEP;
+
+	return(0);
+}
 
diff --git a/bsd/kern/kern_exec.c b/bsd/kern/kern_exec.c
index 4006a8239..b486595ed 100644
--- a/bsd/kern/kern_exec.c
+++ b/bsd/kern/kern_exec.c
@@ -172,18 +172,11 @@ static kern_return_t create_unix_stack(vm_map_t map, user_addr_t user_stack,
 static int copyoutptr(user_addr_t ua, user_addr_t ptr, int ptr_size);
 
 /* XXX forward; should be in headers, but can't be for one reason or another */
-extern int grade_binary(cpu_type_t exectype, cpu_subtype_t execsubtype);
 extern void vfork_return(thread_t th_act,
 				struct proc * p,
 				struct proc *p2,
 				register_t *retval);
 
-
-extern char classichandler[32];
-extern uint32_t classichandler_fsid;
-extern long classichandler_fileid;
-
-
 /*
  * exec_add_string
  *
@@ -290,6 +283,75 @@ exec_save_path(struct image_params *imgp, user_addr_t path, /*uio_seg*/int seg)
 	return(error);
 }
 
+#ifdef IMGPF_POWERPC
+/*
+ * exec_powerpc32_imgact
+ *
+ * Implicitly invoke the PowerPC handler for a byte-swapped image magic
+ * number.  This may happen either as a result of an attempt to invoke a
+ * PowerPC image directly, or indirectly as the interpreter used in an
+ * interpreter script.
+ *
+ * Parameters;	struct image_params *	image parameter block
+ *
+ * Returns:	-1		not an PowerPC image (keep looking)
+ *		-3		Success: exec_archhandler_ppc: relookup
+ *		>0		Failure: exec_archhandler_ppc: error number
+ *
+ * Note:	This image activator does not handle the case of a direct
+ *		invocation of the exec_archhandler_ppc, since in that case, the
+ *		exec_archhandler_ppc itself is not a PowerPC binary; instead,
+ *		binary image activators must recognize the exec_archhandler_ppc;
+ *		This is managed in exec_check_permissions().
+ *
+ * Note:	This image activator is limited to 32 bit powerpc images;
+ *		if support for 64 bit powerpc images is desired, it would
+ *		be more in line with this design to write a separate 64 bit
+ *		image activator.
+ */
+static int
+exec_powerpc32_imgact(struct image_params *imgp)
+{
+	struct mach_header *mach_header = (struct mach_header *)imgp->ip_vdata;
+	int error;
+	size_t len = 0;
+
+	/*
+	 * Make sure it's a PowerPC binary.  If we've already redirected
+	 * from an interpreted file once, don't do it again.
+	 */
+	if (mach_header->magic != MH_CIGAM)
+		return (-1);
+
+	/* If there is no exec_archhandler_ppc, we can't run it */
+	if (exec_archhandler_ppc.path[0] == 0)
+		return (EBADARCH);
+
+	/*
+	 * The PowerPC flag will be set by the exec_check_permissions()
+	 * call anyway; however, we set this flag here so that the relookup
+	 * in execve() does not follow symbolic links, as a side effect.
+	 */
+	imgp->ip_flags |= IMGPF_POWERPC;
+
+	/* impute an interpreter */
+	error = copystr(exec_archhandler_ppc.path, imgp->ip_interp_name,
+			IMG_SHSIZE, &len);
+	if (error)
+		return (error);
+
+	/*
+	 * provide a replacement string for p->p_comm; we have to use an
+	 * an alternate buffer for this, rather than replacing it directly,
+	 * since the exec may fail and return to the parent.  In that case,
+	 * we would have erroneously changed the parent p->p_comm instead.
+	 */
+	strncpy(imgp->ip_p_comm, imgp->ip_ndp->ni_cnd.cn_nameptr, MAXCOMLEN);
+	imgp->ip_p_comm[MAXCOMLEN] = '\0';
+
+	return (-3);
+}
+#endif	/* IMGPF_POWERPC */
 
 
 /*
@@ -297,7 +359,7 @@ exec_save_path(struct image_params *imgp, user_addr_t path, /*uio_seg*/int seg)
  *
  * Image activator for interpreter scripts.  If the image begins with the
  * characters "#!", then it is an interpreter script.  Verify that we are
- * not already executing in Classic mode, and that the length of the script
+ * not already executing in PowerPC mode, and that the length of the script
  * line indicating the interpreter is not in excess of the maximum allowed
  * size.  If this is the case, then break out the arguments, if any, which
  * are separated by white space, and copy them into the argument save area
@@ -325,8 +387,8 @@ exec_shell_imgact(struct image_params *imgp)
 	 * Make sure it's a shell script.  If we've already redirected
 	 * from an interpreted file once, don't do it again.
 	 *
-	 * Note: We disallow Classic, since the expectation is that we
-	 * may run a Classic interpreter, but not an interpret a Classic
+	 * Note: We disallow PowerPC, since the expectation is that we
+	 * may run a PowerPC interpreter, but not an interpret a PowerPC 
 	 * image.  This is consistent with historical behaviour.
 	 */
 	if (vdata[0] != '#' ||
@@ -335,6 +397,10 @@ exec_shell_imgact(struct image_params *imgp)
 		return (-1);
 	}
 
+#ifdef IMGPF_POWERPC
+	if ((imgp->ip_flags & IMGPF_POWERPC) != 0)
+		  return (EBADARCH);
+#endif	/* IMGPF_POWERPC */
 
 	imgp->ip_flags |= IMGPF_INTERPRET;
 
@@ -422,7 +488,7 @@ static int
 exec_fat_imgact(struct image_params *imgp)
 {
 	struct proc *p = vfs_context_proc(imgp->ip_vfs_context);
-	kauth_cred_t cred = p->p_ucred;
+	kauth_cred_t cred = kauth_cred_proc_ref(p);
 	struct fat_header *fat_header = (struct fat_header *)imgp->ip_vdata;
 	struct fat_arch fat_arch;
 	int resid, error;
@@ -466,6 +532,7 @@ exec_fat_imgact(struct image_params *imgp)
 	imgp->ip_arch_size = (user_size_t)fat_arch.size;
 
 bad:
+	kauth_cred_unref(&cred);
 	return (error);
 }
 
@@ -491,9 +558,12 @@ exec_mach_imgact(struct image_params *imgp)
 	vm_map_t old_map = VM_MAP_NULL;
 	vm_map_t map;
 	boolean_t				clean_regions = FALSE;
-    shared_region_mapping_t initial_region = NULL;
 	load_return_t		lret;
 	load_result_t		load_result;
+	shared_region_mapping_t shared_region, initial_region;
+#ifdef IMGPF_POWERPC
+	int powerpcParent, powerpcImage;
+#endif /* IMGPF_POWERPC */
 		
 	/*
 	 * make sure it's a Mach-O 1.0 or Mach-O 2.0 binary; the difference
@@ -542,11 +612,25 @@ exec_mach_imgact(struct image_params *imgp)
 	imgp->ip_strendp[2] = 0;
 	imgp->ip_strendp += (((imgp->ip_strendp - imgp->ip_strings) + NBPW-1) & ~(NBPW-1));
 
+#ifdef IMGPF_POWERPC
+	/*
+	 * XXX
+	 *
+	 * Should be factored out; this is here because we might be getting
+	 * invoked this way as the result of a shell script, and the check
+	 * in exec_check_permissions() is not interior to the jump back up
+	 * to the "encapsulated_binary:" label in execve().
+	 */
+	if (imgp->ip_vattr->va_fsid == exec_archhandler_ppc.fsid &&
+		imgp->ip_vattr->va_fileid == (uint64_t)((u_long)exec_archhandler_ppc.fileid)) {
+		imgp->ip_flags |= IMGPF_POWERPC;
+	}
+#endif	/* IMGPF_POWERPC */
 
 	if (vfexec) {
  		kern_return_t	result;
 
-		result = task_create_internal(task, FALSE, &new_task);
+		result = task_create_internal(task, FALSE, (imgp->ip_flags & IMGPF_IS_64BIT), &new_task);
 		if (result != KERN_SUCCESS)
 	    	printf("execve: task_create failed. Code: 0x%x\n", result);
 		p->task = new_task;
@@ -554,6 +638,12 @@ exec_mach_imgact(struct image_params *imgp)
 		if (p->p_nice != 0)
 			resetpriority(p);
 		map = get_task_map(new_task);
+
+		if (imgp->ip_flags & IMGPF_IS_64BIT)
+		        vm_map_set_64bit(map);
+		else
+		        vm_map_set_32bit(map);
+
 		result = thread_create(new_task, &imgp->ip_vfork_thread);
 		if (result != KERN_SUCCESS)
 	    	printf("execve: thread_create failed. Code: 0x%x\n", result);
@@ -589,7 +679,94 @@ if((imgp->ip_flags & IMGPF_IS_64BIT) == 0)
 
 	vm_get_shared_region(task, &initial_region);
 
-	
+#ifdef IMGPF_POWERPC
+	/*
+	 * If we are transitioning to/from powerpc, then we need to do extra
+	 * work here.
+	 */
+	powerpcParent = (p->p_flag & P_TRANSLATED) ? 1 : 0;
+	powerpcImage = (imgp->ip_flags & IMGPF_POWERPC) ? 1 : 0;
+
+	if (powerpcParent ^ powerpcImage) {
+		cpu_type_t cpu = (powerpcImage ? CPU_TYPE_POWERPC : cpu_type());
+		struct vnode *rootDir = p->p_fd->fd_rdir;
+
+		shared_region = lookup_default_shared_region((int)rootDir, cpu);
+		if (shared_region == NULL) {
+			shared_region_mapping_t old_region;
+			shared_region_mapping_t new_region;
+			vm_get_shared_region(current_task(), &old_region);
+			/* grrrr... this sets current_task(), not task
+			* -- they're different (usually)
+			*/
+			shared_file_boot_time_init((int)rootDir,cpu);
+			if ( current_task() != task ) {
+				vm_get_shared_region(current_task(),&new_region);
+				vm_set_shared_region(task,new_region);
+				vm_set_shared_region(current_task(),old_region);
+			}
+		} else {
+			vm_set_shared_region(task, shared_region);
+		}
+		shared_region_mapping_dealloc(initial_region);
+	} else 
+#endif	/* IMGPF_POWERPC */
+
+	{
+		struct shared_region_task_mappings map_info;
+		shared_region_mapping_t next;
+
+		shared_region_mapping_info(initial_region,
+					   &map_info.text_region,
+					   &map_info.text_size,
+					   &map_info.data_region,
+					   &map_info.data_size,
+					   &map_info.region_mappings,
+					   &map_info.client_base,
+					   &map_info.alternate_base,
+					   &map_info.alternate_next,
+					   &map_info.fs_base,
+					   &map_info.system,
+					   &map_info.flags,
+					   &next);
+		if (map_info.flags & SHARED_REGION_STANDALONE) {
+			/*
+			 * We were using a private shared region.
+			 * Try and get back to a system-wide shared region
+			 * with matching "fs_base" (for chroot) and "system"
+			 * (for CPU type).
+			 */
+			shared_region = lookup_default_shared_region(
+				map_info.fs_base,
+				map_info.system);
+			if (shared_region == NULL) {
+				/*
+				 * No system-wide default regions, stick to
+				 * our private region...
+				 */
+			} else {
+				SHARED_REGION_TRACE(
+					SHARED_REGION_TRACE_INFO,
+					("shared_region: %p [%d(%s)] "
+					 "exec(\"%s\"): "
+					 "moving from private %p[%x,%x,%x] "
+					 "to default %p\n",
+					 current_thread(),
+					 p->p_pid, p->p_comm,
+					 (imgp->ip_p_comm[0] ?
+					  imgp->ip_p_comm :
+					  imgp->ip_ndp->ni_cnd.cn_nameptr),
+					 initial_region,
+					 map_info.fs_base,
+					 map_info.system,
+					 map_info.flags,
+					 shared_region));
+				vm_set_shared_region(task, shared_region);
+				shared_region_mapping_dealloc(initial_region);
+			}
+		}
+	}
+
 	/*
 	 * NOTE: An error after this point  indicates we have potentially
 	 * destroyed or overwrote some process state while attempting an
@@ -600,6 +777,7 @@ if((imgp->ip_flags & IMGPF_IS_64BIT) == 0)
 	 * We reset the task to 64-bit (or not) here.  It may have picked up
 	 * a new map, and we need that to reflect its true 64-bit nature.
 	 */
+
 	task_set_64bit(task, 
 		       ((imgp->ip_flags & IMGPF_IS_64BIT) == IMGPF_IS_64BIT));
 
@@ -640,7 +818,6 @@ if((imgp->ip_flags & IMGPF_IS_64BIT) == 0)
 	}
 
 	if (vfexec) {
-		uthread->uu_ar0 = (void *)get_user_regs(thread);
 		old_map = vm_map_switch(get_task_map(task));
 	}
 
@@ -668,10 +845,15 @@ if((imgp->ip_flags & IMGPF_IS_64BIT) == 0)
 		/* Adjust the stack */
 		if (imgp->ip_flags & IMGPF_IS_64BIT) {
 			ap = thread_adjuserstack(thread, -8);
-			(void)copyoutptr(load_result.mach_header, ap, 8);
+			error = copyoutptr(load_result.mach_header, ap, 8);
 		} else {
 			ap = thread_adjuserstack(thread, -4);
-			(void)suword(ap, load_result.mach_header);
+			error = suword(ap, load_result.mach_header);
+		}
+		if (error) {
+			if (vfexec)
+				vm_map_switch(old_map);
+			goto badtoolate;
 		}
 	}
 
@@ -726,32 +908,42 @@ if((imgp->ip_flags & IMGPF_IS_64BIT) == 0)
 		p->p_comm[imgp->ip_ndp->ni_cnd.cn_namelen] = '\0';
 	}
 
-	{
-	  /* This is for kdebug */
-	  long dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4;
-
-	  /* Collect the pathname for tracing */
-	  kdbg_trace_string(p, &dbg_arg1, &dbg_arg2, &dbg_arg3, &dbg_arg4);
-
-
-
-	  if (vfexec)
-	  {
-		  KERNEL_DEBUG_CONSTANT1((TRACEDBG_CODE(DBG_TRACE_DATA, 2)) | DBG_FUNC_NONE,
-		                        p->p_pid ,0,0,0, (unsigned int)thread);
-	          KERNEL_DEBUG_CONSTANT1((TRACEDBG_CODE(DBG_TRACE_STRING, 2)) | DBG_FUNC_NONE,
-					dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, (unsigned int)thread);
-	  }
-	  else
-	  {
-		  KERNEL_DEBUG_CONSTANT((TRACEDBG_CODE(DBG_TRACE_DATA, 2)) | DBG_FUNC_NONE,
-		                        p->p_pid ,0,0,0,0);
-	          KERNEL_DEBUG_CONSTANT((TRACEDBG_CODE(DBG_TRACE_STRING, 2)) | DBG_FUNC_NONE,
-					dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, 0);
-	  }
+	if (kdebug_enable) {
+	          long dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4;
+
+	          /*
+		   * Collect the pathname for tracing
+		   */
+	          kdbg_trace_string(p, &dbg_arg1, &dbg_arg2, &dbg_arg3, &dbg_arg4);
+
+		  if (vfexec)
+		  {
+		          KERNEL_DEBUG_CONSTANT1((TRACEDBG_CODE(DBG_TRACE_DATA, 2)) | DBG_FUNC_NONE,
+						 p->p_pid ,0,0,0, (unsigned int)thread);
+			  KERNEL_DEBUG_CONSTANT1((TRACEDBG_CODE(DBG_TRACE_STRING, 2)) | DBG_FUNC_NONE,
+						 dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, (unsigned int)thread);
+		  }
+		  else
+		  {
+		          KERNEL_DEBUG_CONSTANT((TRACEDBG_CODE(DBG_TRACE_DATA, 2)) | DBG_FUNC_NONE,
+						p->p_pid ,0,0,0,0);
+			  KERNEL_DEBUG_CONSTANT((TRACEDBG_CODE(DBG_TRACE_STRING, 2)) | DBG_FUNC_NONE,
+						dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, 0);
+		  }
 	}
 
-		p->p_flag &= ~P_CLASSIC;
+#ifdef IMGPF_POWERPC
+	/*
+	 * Mark the process as powerpc or not.  If powerpc, set the affinity
+	 * flag, which will be used for grading binaries in future exec's
+	 * from the process.
+	 */
+	if (((imgp->ip_flags & IMGPF_POWERPC) != 0))
+		p->p_flag |= P_TRANSLATED;
+	else
+#endif	/* IMGPF_POWERPC */
+		p->p_flag &= ~P_TRANSLATED;
+	p->p_flag &= ~P_AFFINITY;
 
 	/*
 	 * mark as execed, wakeup the process that vforked (if any) and tell
@@ -795,6 +987,9 @@ struct execsw {
 } execsw[] = {
 	{ exec_mach_imgact,		"Mach-o Binary" },
 	{ exec_fat_imgact,		"Fat Binary" },
+#ifdef IMGPF_POWERPC
+	{ exec_powerpc32_imgact,	"PowerPC binary" },
+#endif	/* IMGPF_POWERPC */
 	{ exec_shell_imgact,		"Interpreter Script" },
 	{ NULL, NULL}
 };
@@ -807,7 +1002,7 @@ struct execsw {
 int
 execve(struct proc *p, struct execve_args *uap, register_t *retval)
 {
-	kauth_cred_t cred = p->p_ucred;
+	kauth_cred_t cred = kauth_cred_proc_ref(p);
 	struct image_params image_params, *imgp;
 	struct vnode_attr va;
 	struct vnode_attr origva;
@@ -819,13 +1014,13 @@ execve(struct proc *p, struct execve_args *uap, register_t *retval)
 	int numthreads;
 	int vfexec=0;
 	int once = 1;	/* save SGUID-ness for interpreted files */
-	char alt_p_comm[sizeof(p->p_comm)] = {0};	/* for Classic */
+	char alt_p_comm[sizeof(p->p_comm)] = {0};	/* for PowerPC */
 	int is_64 = IS_64BIT_PROCESS(p);
 	int seg = (is_64 ? UIO_USERSPACE64 : UIO_USERSPACE32);
 	struct vfs_context context;
 
 	context.vc_proc = p;
-	context.vc_ucred = p->p_ucred;	/* XXX must NOT be kauth_cred_get() */
+	context.vc_ucred = cred;	/* XXX must NOT be kauth_cred_get() */
 
 
 	imgp = &image_params;
@@ -840,7 +1035,7 @@ execve(struct proc *p, struct execve_args *uap, register_t *retval)
 	imgp->ip_vfs_context = &context;
 	imgp->ip_flags = (is_64 ? IMGPF_WAS_64BIT : IMGPF_NONE);
 	imgp->ip_tws_cache_name = NULL;
-	imgp->ip_p_comm = alt_p_comm;		/* for Classic */
+	imgp->ip_p_comm = alt_p_comm;		/* for PowerPC */
 
 	/*
          * XXXAUDIT: Currently, we only audit the pathname of the binary.
@@ -855,18 +1050,22 @@ execve(struct proc *p, struct execve_args *uap, register_t *retval)
 	} else {
 		if (task != kernel_task) { 
 			numthreads = get_task_numacts(task);
-			if (numthreads <= 0 )
+			if (numthreads <= 0 ) {
+			        kauth_cred_unref(&cred);
 				return(EINVAL);
+			}
 			if (numthreads > 1) {
+			        kauth_cred_unref(&cred);
 				return(ENOTSUP);
 			}
 		}
 	}
 
 	error = execargs_alloc(imgp);
-	if (error)
+	if (error) {
+	        kauth_cred_unref(&cred);
 		return(error);
-	
+	}
 	/*
 	 * XXXAUDIT: Note: the double copyin introduces an audit
 	 * race.  To correct this race, we must use a single
@@ -876,6 +1075,7 @@ execve(struct proc *p, struct execve_args *uap, register_t *retval)
 	error = exec_save_path(imgp, uap->fname, seg);
 	if (error) {
 		execargs_free(imgp);
+	        kauth_cred_unref(&cred);
 		return(error);
 	}
 
@@ -941,6 +1141,15 @@ encapsulated_binary:
 			nd.ni_cnd.cn_flags = (nd.ni_cnd.cn_flags & HASBUF) |
 						(FOLLOW | LOCKLEAF);
 
+#ifdef IMGPF_POWERPC
+			/*
+			 * PowerPC does not follow symlinks because the
+			 * code which sets exec_archhandler_ppc.fsid and
+			 * exec_archhandler_ppc.fileid doesn't follow them.
+			 */
+			if (imgp->ip_flags & IMGPF_POWERPC)
+				nd.ni_cnd.cn_flags &= ~FOLLOW;
+#endif	/* IMGPF_POWERPC */
 
 			nd.ni_segflg = UIO_SYSSPACE32;
 			nd.ni_dirp = CAST_USER_ADDR_T(imgp->ip_interp_name);
@@ -964,6 +1173,8 @@ encapsulated_binary:
 		error = ENOEXEC;
 
 bad:
+	kauth_cred_unref(&cred);
+
 	if (imgp->ip_ndp)
 		nameidone(imgp->ip_ndp);
 	if (imgp->ip_vp)
@@ -1096,6 +1307,27 @@ exec_copyout_strings(struct image_params *imgp, user_addr_t *stackp)
 
 	stack = *stackp;
 
+	unsigned patharea_len = imgp->ip_argv - imgp->ip_strings;
+	int envc_add = 0;
+	
+#ifdef IMGPF_POWERPC
+	/*
+	 * oah750 expects /usr/lib/dyld\0 as the start of the program name.
+	 * It also expects to have a certain environment variable set to 0.
+	 * 50 bytes for each to ensure we have enough space without having
+	 * to count every byte.
+	 */
+	char *progname, *envvar;
+	char progname_str[] = "/usr/lib/dyld";
+	char envvar_str[] = "OAH750_CFG_FU_STACK_SIZE=0";
+	
+	if (imgp->ip_flags & IMGPF_POWERPC) {
+		progname = progname_str;
+		envvar = envvar_str;
+		patharea_len += strlen(progname) + strlen(envvar) + 2;
+		envc_add = 1;
+	}
+#endif /* IMGPF_POWERPC */
 	/*
 	 * Set up pointers to the beginning of the string area, the beginning
 	 * of the path area, and the beginning of the pointer area (actually,
@@ -1103,8 +1335,8 @@ exec_copyout_strings(struct image_params *imgp, user_addr_t *stackp)
 	 * but we use ptr_size worth of space for it, for alignment).
 	 */
 	string_area = stack - (((imgp->ip_strendp - imgp->ip_strings) + ptr_size-1) & ~(ptr_size-1)) - ptr_size;
-	path_area = string_area - (((imgp->ip_argv - imgp->ip_strings) + ptr_size-1) & ~(ptr_size-1));
-	ptr_area = path_area - ((imgp->ip_argc + imgp->ip_envc + 4) * ptr_size) - ptr_size /*argc*/;
+	path_area = string_area - ((patharea_len + ptr_size-1) & ~(ptr_size-1));
+	ptr_area = path_area - ((imgp->ip_argc + imgp->ip_envc + 4 + envc_add) * ptr_size) - ptr_size /*argc*/;
 
 	/* Return the initial stack address: the location of argc */
 	*stackp = ptr_area;
@@ -1123,8 +1355,20 @@ exec_copyout_strings(struct image_params *imgp, user_addr_t *stackp)
 	 * copy it just before the string area.
 	 */
 	len = 0;
+#ifdef IMGPF_POWERPC
+	if (imgp->ip_flags & IMGPF_POWERPC) {
+		error = copyoutstr(progname, path_area,
+						   patharea_len,
+						   (size_t *)&len);
+		if (error)
+			goto bad;
+		error = copyoutstr(imgp->ip_strings, path_area + strlen(progname) + 1,
+						   patharea_len,
+						   (size_t *)&len);
+	} else
+#endif /* IMGPF_POWERPC */
 	error = copyoutstr(imgp->ip_strings, path_area,
-				(unsigned)(imgp->ip_argv - imgp->ip_strings),
+						   patharea_len,
 				(size_t *)&len);
 	if (error)
 		goto bad;
@@ -1167,6 +1411,27 @@ exec_copyout_strings(struct image_params *imgp, user_addr_t *stackp)
 			/* argv[n] = NULL */
 			(void)copyoutptr(0LL, ptr_area, ptr_size);
 			ptr_area += ptr_size;
+#ifdef IMGPF_POWERPC
+			if (envc_add) {
+				(void)copyoutptr(string_area, ptr_area, ptr_size);
+				
+				do {
+					if (strspace <= 0) {
+						error = E2BIG;
+						break;
+					}
+					error = copyoutstr(envvar, string_area,
+									   (unsigned)strspace,
+									   (size_t *)&len);
+					string_area += len;
+					envvar += len;
+					strspace -= len;
+				} while (error == ENAMETOOLONG);				
+				if (error == EFAULT || error == E2BIG)
+					break;
+				ptr_area += ptr_size;
+			}
+#endif /* IMGPF_POWERPC */
 		}
 		if (--stringc < 0)
 			break;
@@ -1361,6 +1626,17 @@ exec_check_permissions(struct image_params *imgp)
 	if (vp->v_writecount)
 		return (ETXTBSY);
 
+#ifdef IMGPF_POWERPC
+	/*
+	 * If the file we are about to attempt to load is the exec_handler_ppc,
+	 * which is determined by matching the vattr fields against previously
+	 * cached values, then we set the PowerPC environment flag.
+	 */
+	if (vap->va_fsid == exec_archhandler_ppc.fsid &&
+		vap->va_fileid == (uint64_t)((u_long)exec_archhandler_ppc.fileid)) {
+		imgp->ip_flags |= IMGPF_POWERPC;
+	}
+#endif	/* IMGPF_POWERPC */
 
 	/* XXX May want to indicate to underlying FS that vnode is open */
 
diff --git a/bsd/kern/kern_exit.c b/bsd/kern/kern_exit.c
index 68c245b6e..06ee02894 100644
--- a/bsd/kern/kern_exit.c
+++ b/bsd/kern/kern_exit.c
@@ -346,9 +346,10 @@ proc_exit(struct proc *p)
 				 * if we blocked.
 				 */
 				context.vc_proc = p;
-				context.vc_ucred = p->p_ucred;
+				context.vc_ucred = kauth_cred_proc_ref(p);
 				if (sp->s_ttyvp)
 					VNOP_REVOKE(sp->s_ttyvp, REVOKEALL, &context);
+				kauth_cred_unref(&context.vc_ucred);
 			}
 			ttyvp = sp->s_ttyvp;
 			sp->s_ttyvp = NULL;
@@ -579,11 +580,9 @@ reap_child_process(struct proc *parent, struct proc *child)
 	/*
 	 * Free up credentials.
 	 */
-	if (child->p_ucred != NOCRED) {
-		kauth_cred_t ucr = child->p_ucred;
-			child->p_ucred = NOCRED;
-			kauth_cred_rele(ucr);
-		}
+	if (IS_VALID_CRED(child->p_ucred)) {
+		kauth_cred_unref(&child->p_ucred);
+	}
 
 	/*
 	 * Release reference to text vnode
@@ -1163,9 +1162,10 @@ vproc_exit(struct proc *p)
 				 * if we blocked.
 				 */
 				context.vc_proc = p;
-				context.vc_ucred = p->p_ucred;
+				context.vc_ucred = kauth_cred_proc_ref(p);
 				if (sp->s_ttyvp)
 					VNOP_REVOKE(sp->s_ttyvp, REVOKEALL, &context);
+				kauth_cred_unref(&context.vc_ucred);
 			}
 			ttyvp = sp->s_ttyvp;
 			sp->s_ttyvp = NULL;
diff --git a/bsd/kern/kern_fork.c b/bsd/kern/kern_fork.c
index 71ca601e9..b4be0c5e5 100644
--- a/bsd/kern/kern_fork.c
+++ b/bsd/kern/kern_fork.c
@@ -258,9 +258,9 @@ procdup(struct proc *child, struct proc *parent)
  	kern_return_t	result;
 
 	if (parent->task == kernel_task)
-		result = task_create_internal(TASK_NULL, FALSE, &task);
+		result = task_create_internal(TASK_NULL, FALSE, FALSE, &task);
 	else
-		result = task_create_internal(parent->task, TRUE, &task);
+		result = task_create_internal(parent->task, TRUE, (parent->p_flag & P_LP64), &task);
 	if (result != KERN_SUCCESS)
 	    printf("fork/procdup: task_create failed. Code: 0x%x\n", result);
 	child->task = task;
@@ -268,15 +268,25 @@ procdup(struct proc *child, struct proc *parent)
 	set_bsdtask_info(task, child);
 	if (parent->p_flag & P_LP64) {
 		task_set_64bit(task, TRUE);
+		vm_map_set_64bit(get_task_map(task));
 		child->p_flag |= P_LP64;
-#ifdef __PPC__
                 /* LP64todo - clean up this hacked mapping of commpage */
 		pmap_map_sharedpage(task, get_map_pmap(get_task_map(task)));
                 vm_map_commpage64(get_task_map(task));
-#endif	/* __PPC__ */
 	} else {
 		task_set_64bit(task, FALSE);
+		vm_map_set_32bit(get_task_map(task));
 		child->p_flag &= ~P_LP64;
+#ifdef __i386__
+		/*
+		 * On Intel, the comm page doesn't get mapped automatically
+		 * because it goes beyond the end of the VM map in the current
+		 * 3GB/1GB address space model.
+		 * XXX This explicit mapping will probably become unnecessary
+		 * when we switch to the new 4GB/4GB address space model.
+		 */
+		vm_map_commpage32(get_task_map(task));
+#endif	/* __i386__ */
 	}
 	if (child->p_nice != 0)
 		resetpriority(child);
@@ -500,7 +510,7 @@ again:
 	 * Increase reference counts on shared objects.
 	 * The p_stats and p_sigacts substructs are set in vm_fork.
 	 */
-	p2->p_flag = (p1->p_flag & (P_LP64 | P_CLASSIC | P_AFFINITY));
+	p2->p_flag = (p1->p_flag & (P_LP64 | P_TRANSLATED | P_AFFINITY));
 	if (p1->p_flag & P_PROFIL)
 		startprofclock(p2);
 	/*
@@ -654,9 +664,14 @@ uthread_alloc(task_t task, thread_t thr_act )
 	uth_parent = (struct uthread *)get_bsdthread_info(current_thread());
 	if ((task == current_task()) && 
 	    (uth_parent != NULL) &&
-	    (uth_parent->uu_ucred != NOCRED)) {
+	    (IS_VALID_CRED(uth_parent->uu_ucred))) {
+		/*
+		 * XXX The new thread is, in theory, being created in context
+		 * XXX of parent thread, so a direct reference to the parent
+		 * XXX is OK.
+		 */
+		kauth_cred_ref(uth_parent->uu_ucred);
 		uth->uu_ucred = uth_parent->uu_ucred;
-		kauth_cred_ref(uth->uu_ucred);
 		/* the credential we just inherited is an assumed credential */
 		if (uth_parent->uu_flag & UT_SETUID)
 			uth->uu_flag |= UT_SETUID;
@@ -717,8 +732,11 @@ uthread_free(task_t task, void *uthread, void * bsd_info)
 		sel->wql = 0;
 	}
 
-	if (uth->uu_ucred != NOCRED)
-		kauth_cred_rele(uth->uu_ucred);
+	if (IS_VALID_CRED(uth->uu_ucred)) {
+		kauth_cred_t oldcred = uth->uu_ucred;
+		uth->uu_ucred = NOCRED;
+		kauth_cred_unref(&oldcred);
+	}
 
 	if ((task != kernel_task) && p) {
 		funnel_state = thread_funnel_set(kernel_flock, TRUE);
diff --git a/bsd/kern/kern_ktrace.c b/bsd/kern/kern_ktrace.c
index e640e4c69..2837fb92d 100644
--- a/bsd/kern/kern_ktrace.c
+++ b/bsd/kern/kern_ktrace.c
@@ -113,7 +113,7 @@ void
 ktrsyscall(p, code, narg, args)
 	struct proc *p;
 	int code, narg;
-	u_int64_t args[];
+	syscall_arg_t args[];
 {
 #if KTRACE
 	struct vnode *vp;
diff --git a/bsd/kern/kern_malloc.c b/bsd/kern/kern_malloc.c
index 16125c368..3be4a7ba8 100644
--- a/bsd/kern/kern_malloc.c
+++ b/bsd/kern/kern_malloc.c
@@ -87,6 +87,7 @@
 #include <sys/tty.h>
 #include <sys/quota.h>
 #include <sys/uio_internal.h>
+#include <sys/resourcevar.h>
 
 #include <ufs/ufs/inode.h>
 
diff --git a/bsd/kern/kern_mib.c b/bsd/kern/kern_mib.c
index 8598b7306..ad851a80f 100644
--- a/bsd/kern/kern_mib.c
+++ b/bsd/kern/kern_mib.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -98,6 +98,7 @@
 #include <kern/task.h>
 #include <vm/vm_kern.h>
 #include <mach/host_info.h>
+#include <kern/pms.h>
 
 extern vm_map_t bsd_pageable_map;
 
@@ -110,7 +111,15 @@ extern vm_map_t bsd_pageable_map;
 #include <machine/machine_routines.h>
 #include <machine/cpu_capabilities.h>
 
-static int	cputype, cpusubtype, cputhreadtype;
+#ifdef __i386__
+#include <i386/cpuid.h>		/* for cpuid_info() */
+#endif
+
+#ifndef MAX
+#define MAX(a,b) (a >= b ? a : b)
+#endif
+
+static int	cputype, cpusubtype, cputhreadtype, cpufamily, cacheconfig[10];;
 
 SYSCTL_NODE(, 0,	  sysctl, CTLFLAG_RW, 0,
 	"Sysctl internal magic");
@@ -271,8 +280,10 @@ sysctl_hw_generic SYSCTL_HANDLER_ARGS
 		if (epochTemp == -1)
 			return(EINVAL);
 		return(SYSCTL_RETURN(req, epochTemp));
-	case HW_VECTORUNIT:
-		return(SYSCTL_RETURN(req, cpu_info.vector_unit));
+	case HW_VECTORUNIT: {
+		int vector = cpu_info.vector_unit == 0? 0 : 1;
+		return(SYSCTL_RETURN(req, vector));
+	}
 	case HW_L2SETTINGS:
 		if (cpu_info.l2_cache_size == 0xFFFFFFFF)
 			return(EINVAL);
@@ -305,6 +316,8 @@ SYSCTL_PROC    (_hw, OID_AUTO, logicalcpu_max, CTLTYPE_INT | CTLFLAG_RD | CTLFLA
 SYSCTL_INT     (_hw, HW_BYTEORDER, byteorder, CTLFLAG_RD | CTLFLAG_KERN, NULL, BYTE_ORDER, "");
 SYSCTL_INT     (_hw, OID_AUTO, cputype, CTLFLAG_RD | CTLFLAG_KERN, &cputype, 0, "");
 SYSCTL_INT     (_hw, OID_AUTO, cpusubtype, CTLFLAG_RD | CTLFLAG_KERN, &cpusubtype, 0, "");
+SYSCTL_INT     (_hw, OID_AUTO, cpufamily, CTLFLAG_RD | CTLFLAG_KERN, &cpufamily, 0, "");
+SYSCTL_OPAQUE  (_hw, OID_AUTO, cacheconfig, CTLFLAG_RD, &cacheconfig, sizeof(cacheconfig), "I", "");
 SYSCTL_INT2QUAD(_hw, OID_AUTO, pagesize, CTLFLAG_RD | CTLFLAG_KERN, &page_size, "");
 SYSCTL_QUAD    (_hw, OID_AUTO, busfrequency, CTLFLAG_RD | CTLFLAG_KERN, &gPEClockFrequencyInfo.bus_frequency_hz, "");
 SYSCTL_QUAD    (_hw, OID_AUTO, busfrequency_min, CTLFLAG_RD | CTLFLAG_KERN, &gPEClockFrequencyInfo.bus_frequency_min_hz, "");
@@ -362,6 +375,29 @@ SYSCTL_PROC(_hw, HW_VECTORUNIT,   vectorunit, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG
 SYSCTL_PROC(_hw, HW_L2SETTINGS,   l2settings, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED, 0, HW_L2SETTINGS, sysctl_hw_generic, "I", "");
 SYSCTL_PROC(_hw, HW_L3SETTINGS,   l3settings, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED, 0, HW_L3SETTINGS, sysctl_hw_generic, "I", "");
 
+/*
+ * Debugging interface to the CPU power management code.
+ */
+static void pmsSysctl SYSCTL_HANDLER_ARGS {
+
+	pmsctl_t	ctl;
+	int		error;
+	boolean_t	intr;
+
+	if ((error = SYSCTL_IN(req, &ctl, sizeof(ctl))))
+		return(error);
+
+	intr = ml_set_interrupts_enabled(FALSE);		/* No interruptions in here */
+	error = pmsControl(ctl.request, (user_addr_t)ctl.reqaddr, ctl.reqsize);
+	(void)ml_set_interrupts_enabled(intr);			/* Restore interruptions */
+
+	return(error);
+}
+
+SYSCTL_PROC(_hw, OID_AUTO, pms, CTLTYPE_STRUCT | CTLFLAG_WR, 0, 0, pmsSysctl, "S", "Processor Power Management");
+
+
+
 /******************************************************************************
  * Generic MIB initialisation.
  *
@@ -467,7 +503,78 @@ sysctl_mib_init(void)
 		if (dcbtstreams_flag >= 0)
 			sysctl_register_oid(&sysctl__hw_optional_dcbtstreams);
 	}
-#else
+
+	/* hw.cpufamily */
+	switch (cpusubtype) {
+	case CPU_SUBTYPE_POWERPC_750:
+		cpufamily = CPUFAMILY_POWERPC_G3;
+		break;
+	case CPU_SUBTYPE_POWERPC_7400:
+	case CPU_SUBTYPE_POWERPC_7450:
+		cpufamily = CPUFAMILY_POWERPC_G4;
+		break;
+	case CPU_SUBTYPE_POWERPC_970:
+		cpufamily = CPUFAMILY_POWERPC_G5;
+		break;
+	default:
+		cpufamily = CPUFAMILY_UNKNOWN;
+	}
+
+	/* hw.cacheconfig */
+	cacheconfig[0] = 0; /* XXX not supported on PowerPC */
+
+#elif defined (__i386__)
+
+
+#define DECLARE_X86_HW_OPTIONAL_FLAGS(named, BITS, flags) { \
+	static int named##_flag = -1;	\
+	static SYSCTL_INT(_hw_optional, OID_AUTO, named, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | flags, &named##_flag, 0, "");	\
+	named##_flag = ((_get_cpu_capabilities() & BITS) == BITS)? 1 : 0;	\
+	sysctl_register_oid(&sysctl__hw_optional_##named);	\
+	}
+
+#define DECLARE_X86_HW_OPTIONAL(named, BITS) \
+	DECLARE_X86_HW_OPTIONAL_FLAGS(named, BITS, 0)
+
+#define DECLARE_X86_HW_OPTIONAL_MASKED(named, BITS) \
+	DECLARE_X86_HW_OPTIONAL_FLAGS(named, BITS, CTLFLAG_MASKED)
+
+	DECLARE_X86_HW_OPTIONAL(mmx, kHasMMX);
+	DECLARE_X86_HW_OPTIONAL(sse, kHasSSE);
+	DECLARE_X86_HW_OPTIONAL(sse2, kHasSSE2);
+	DECLARE_X86_HW_OPTIONAL(sse3, kHasSSE3);
+	if (_get_cpu_capabilities() & k64Bit)
+		DECLARE_X86_HW_OPTIONAL(x86_64, k64Bit);
+	if (_get_cpu_capabilities() & kHasSupplementalSSE3) {
+		DECLARE_X86_HW_OPTIONAL_MASKED(mni, kHasSupplementalSSE3); /* XXX */
+		DECLARE_X86_HW_OPTIONAL(supplementalsse3, kHasSupplementalSSE3);
+	}
+
+	/* hw.cpufamily */
+	switch (cpuid_info()->cpuid_family) {
+	case 6:
+		switch (cpuid_info()->cpuid_model) {
+		case 14:
+			cpufamily = CPUFAMILY_INTEL_6_14; /* Core Solo/Duo */
+			break;
+		case 15:
+			cpufamily = CPUFAMILY_INTEL_6_15;
+			break;
+		default:
+			cpufamily = CPUFAMILY_UNKNOWN;
+		}
+		break;
+	default:
+		cpufamily = CPUFAMILY_UNKNOWN;
+	}
+	/* hw.cacheconfig */
+	cacheconfig[0] = cpuid_info()->cpuid_cores_per_package;
+	cacheconfig[1] = MAX(cpuid_info()->cache_sharing[L1I], cpuid_info()->cache_sharing[L1D]);
+	cacheconfig[2] = cpuid_info()->cache_sharing[L2U];
+	cacheconfig[3] = cpuid_info()->cache_sharing[L3U];
+	cacheconfig[4] = 0;
+
+#else /* end __i386 */
 # warning we do not support this platform yet
 #endif /* __ppc__ */
 
diff --git a/bsd/kern/kern_proc.c b/bsd/kern/kern_proc.c
index 55072bf67..de3c6bcf6 100644
--- a/bsd/kern/kern_proc.c
+++ b/bsd/kern/kern_proc.c
@@ -722,7 +722,7 @@ pgrpdump(void)
 int
 proc_is_classic(struct proc *p)
 {
-    return (p->p_flag & P_CLASSIC) ? 1 : 0;
+    return (p->p_flag & P_TRANSLATED) ? 1 : 0;
 }
 
 /* XXX Why does this function exist?  Need to kill it off... */
diff --git a/bsd/kern/kern_prot.c b/bsd/kern/kern_prot.c
index 2e2a52dff..8ed96a390 100644
--- a/bsd/kern/kern_prot.c
+++ b/bsd/kern/kern_prot.c
@@ -251,21 +251,21 @@ getgroups(__unused struct proc *p, struct getgroups_args *uap, register_t *retva
 
 	if ((ngrp = uap->gidsetsize) == 0) {
 		*retval = cred->cr_ngroups;
-		kauth_cred_rele(cred);
+		kauth_cred_unref(&cred);
 		return (0);
 	}
 	if (ngrp < cred->cr_ngroups) {
-		kauth_cred_rele(cred);
+		kauth_cred_unref(&cred);
 		return (EINVAL);
 	}
 	ngrp = cred->cr_ngroups;
 	if ((error = copyout((caddr_t)cred->cr_groups,
 	    				uap->gidset, 
 	    				ngrp * sizeof(gid_t)))) {
-		kauth_cred_rele(cred);
+		kauth_cred_unref(&cred);
 		return (error);
 	}
-	kauth_cred_rele(cred);
+	kauth_cred_unref(&cred);
 	*retval = ngrp;
 	return (0);
 }
@@ -394,8 +394,11 @@ setuid(struct proc *p, struct setuid_args *uap, __unused register_t *retval)
 		my_cred = kauth_cred_proc_ref(p);
 		
 		/* 
-		 * set the credential with new info.  If there is no change we get back 
-		 * the same credential we passed in.
+		 * Set the credential with new info.  If there is no change,
+		 * we get back the same credential we passed in; if there is
+		 * a change, we drop the reference on the credential we
+		 * passed in.  The subsequent compare is safe, because it is
+		 * a pointer compare rather than a contents compare.
 		 */
 		my_new_cred = kauth_cred_setuid(my_cred, uid);
 		if (my_cred != my_new_cred) {
@@ -406,8 +409,7 @@ setuid(struct proc *p, struct setuid_args *uap, __unused register_t *retval)
 			 */
 			if (p->p_ucred != my_cred) {
 				proc_unlock(p);
-				kauth_cred_rele(my_cred);
-				kauth_cred_rele(my_new_cred);
+				kauth_cred_unref(&my_new_cred);
 				/* try again */
 				continue;
 			}
@@ -415,8 +417,8 @@ setuid(struct proc *p, struct setuid_args *uap, __unused register_t *retval)
 			p->p_flag |= P_SUGID;
 			proc_unlock(p);
 		}
-		/* drop our extra reference */
-		kauth_cred_rele(my_cred);
+		/* drop old proc reference or our extra reference */
+		kauth_cred_unref(&my_cred);
 		break;
 	}
 	
@@ -446,21 +448,25 @@ seteuid(struct proc *p, struct seteuid_args *uap, __unused register_t *retval)
 		my_cred = kauth_cred_proc_ref(p);
 	
 		/* 
-		 * set the credential with new info.  If there is no change we get back 
-		 * the same credential we passed in.
+		 * Set the credential with new info.  If there is no change,
+		 * we get back the same credential we passed in; if there is
+		 * a change, we drop the reference on the credential we
+		 * passed in.  The subsequent compare is safe, because it is
+		 * a pointer compare rather than a contents compare.
 		 */
-		my_new_cred = kauth_cred_seteuid(p->p_ucred, euid);
+		my_new_cred = kauth_cred_seteuid(my_cred, euid);
 	
 		if (my_cred != my_new_cred) {
 			proc_lock(p);
-			/* need to protect for a race where another thread also changed
-			 * the credential after we took our reference.  If p_ucred has 
-			 * changed then we should restart this again with the new cred.
+			/*
+			 * We need to protect for a race where another thread
+			 * also changed the credential after we took our
+			 * reference.  If p_ucred has changed then we should
+			 * restart this again with the new cred.
 			 */
 			if (p->p_ucred != my_cred) {
 				proc_unlock(p);
-				kauth_cred_rele(my_cred);
-				kauth_cred_rele(my_new_cred);
+				kauth_cred_unref(&my_new_cred);
 				/* try again */
 				continue;
 			}
@@ -468,8 +474,8 @@ seteuid(struct proc *p, struct seteuid_args *uap, __unused register_t *retval)
 			p->p_flag |= P_SUGID;
 			proc_unlock(p);
 		}
-		/* drop our extra reference */
-		kauth_cred_rele(my_cred);
+		/* drop old proc reference or our extra reference */
+		kauth_cred_unref(&my_cred);
 		break;
 	}
 
@@ -495,20 +501,24 @@ setgid(struct proc *p, struct setgid_args *uap, __unused register_t *retval)
 		my_cred = kauth_cred_proc_ref(p);
 		
 		/* 
-		 * set the credential with new info.  If there is no change we get back 
-		 * the same credential we passed in.
+		 * Set the credential with new info.  If there is no change,
+		 * we get back the same credential we passed in; if there is
+		 * a change, we drop the reference on the credential we
+		 * passed in.  The subsequent compare is safe, because it is
+		 * a pointer compare rather than a contents compare.
 		 */
-		my_new_cred = kauth_cred_setgid(p->p_ucred, gid);
+		my_new_cred = kauth_cred_setgid(my_cred, gid);
 		if (my_cred != my_new_cred) {
 			proc_lock(p);
-			/* need to protect for a race where another thread also changed
-			 * the credential after we took our reference.  If p_ucred has 
-			 * changed then we should restart this again with the new cred.
+			/*
+			 * We need to protect for a race where another thread
+			 * also changed the credential after we took our
+			 * reference.  If p_ucred has changed then we should
+			 * restart this again with the new cred.
 			 */
 			if (p->p_ucred != my_cred) {
 				proc_unlock(p);
-				kauth_cred_rele(my_cred);
-				kauth_cred_rele(my_new_cred);
+				kauth_cred_unref(&my_new_cred);
 				/* try again */
 				continue;
 			}
@@ -516,8 +526,8 @@ setgid(struct proc *p, struct setgid_args *uap, __unused register_t *retval)
 			p->p_flag |= P_SUGID;
 			proc_unlock(p);
 		}
-		/* drop our extra reference */
-		kauth_cred_rele(my_cred);
+		/* drop old proc reference or our extra reference */
+		kauth_cred_unref(&my_cred);
 		break;
 	}
 	
@@ -544,10 +554,13 @@ setegid(struct proc *p, struct setegid_args *uap, __unused register_t *retval)
 		my_cred = kauth_cred_proc_ref(p);
 		
 		/* 
-		 * set the credential with new info.  If there is no change we get back 
-		 * the same credential we passed in.
+		 * Set the credential with new info.  If there is no change,
+		 * we get back the same credential we passed in; if there is
+		 * a change, we drop the reference on the credential we
+		 * passed in.  The subsequent compare is safe, because it is
+		 * a pointer compare rather than a contents compare.
 		 */
-		my_new_cred = kauth_cred_setegid(p->p_ucred, egid);
+		my_new_cred = kauth_cred_setegid(my_cred, egid);
 		if (my_cred != my_new_cred) {
 			proc_lock(p);
 			/* need to protect for a race where another thread also changed
@@ -556,8 +569,7 @@ setegid(struct proc *p, struct setegid_args *uap, __unused register_t *retval)
 			 */
 			if (p->p_ucred != my_cred) {
 				proc_unlock(p);
-				kauth_cred_rele(my_cred);
-				kauth_cred_rele(my_new_cred);
+				kauth_cred_unref(&my_new_cred);
 				/* try again */
 				continue;
 			}
@@ -565,8 +577,8 @@ setegid(struct proc *p, struct setegid_args *uap, __unused register_t *retval)
 			p->p_flag |= P_SUGID;
 			proc_unlock(p);
 		}
-		/* drop our extra reference */
-		kauth_cred_rele(my_cred);
+		/* drop old proc reference or our extra reference */
+		kauth_cred_unref(&my_cred);
 		break;
 	}
 
@@ -606,7 +618,7 @@ settid(struct proc *p, struct settid_args *uap, __unused register_t *retval)
 
 		/* revert to delayed binding of process credential */
 		uc = kauth_cred_proc_ref(p);
-		kauth_cred_rele(uthread->uu_ucred);
+		kauth_cred_unref(&uthread->uu_ucred);
 		uthread->uu_ucred = uc;
 		uthread->uu_flag &= ~UT_SETUID;
 	} else {
@@ -630,7 +642,7 @@ settid(struct proc *p, struct settid_args *uap, __unused register_t *retval)
 		uthread->uu_flag |= UT_SETUID;
 
 		/* drop our extra reference */
-		kauth_cred_rele(my_cred);
+		kauth_cred_unref(&my_cred);
 	}
 	/*
 	 * XXX should potentially set per thread security token (there is
@@ -699,8 +711,8 @@ settid_with_pid(struct proc *p, struct settid_with_pid_args *uap, __unused regis
 		uthread->uu_flag |= UT_SETUID;
 		
 		/* drop our extra references */
-		kauth_cred_rele(my_cred);
-		kauth_cred_rele(my_target_cred);
+		kauth_cred_unref(&my_cred);
+		kauth_cred_unref(&my_target_cred);
 
 		return (0);
 	}
@@ -713,7 +725,7 @@ settid_with_pid(struct proc *p, struct settid_with_pid_args *uap, __unused regis
 
 	/* revert to delayed binding of process credential */
 	my_new_cred = kauth_cred_proc_ref(p);
-	kauth_cred_rele(uthread->uu_ucred);
+	kauth_cred_unref(&uthread->uu_ucred);
 	uthread->uu_ucred = my_new_cred;
 	uthread->uu_flag &= ~UT_SETUID;
 	
@@ -773,8 +785,12 @@ setgroups1(struct proc *p, u_int gidsetsize, user_addr_t gidset, uid_t gmuid, __
 			my_cred = kauth_cred_proc_ref(p);
 
 			/* 
-			 * set the credential with new info.  If there is no
-			 * change we get back the same credential we passed in.
+			 * Set the credential with new info.  If there is no
+			 * change, we get back the same credential we passed
+			 * in; if there is a change, we drop the reference on
+			 * the credential we passed in.  The subsequent
+			 * compare is safe, because it is a pointer compare
+			 * rather than a contents compare.
 			 */
 			my_new_cred = kauth_cred_setgroups(my_cred, &newgroups[0], ngrp, gmuid);
 			if (my_cred != my_new_cred) {
@@ -788,8 +804,7 @@ setgroups1(struct proc *p, u_int gidsetsize, user_addr_t gidset, uid_t gmuid, __
 				 */
 				if (p->p_ucred != my_cred) {
 					proc_unlock(p);
-					kauth_cred_rele(my_cred);
-					kauth_cred_rele(my_new_cred);
+					kauth_cred_unref(&my_new_cred);
 					/* try again */
 					continue;
 				}
@@ -797,8 +812,8 @@ setgroups1(struct proc *p, u_int gidsetsize, user_addr_t gidset, uid_t gmuid, __
 				p->p_flag |= P_SUGID;
 				proc_unlock(p);
 			}
-			/* drop our extra reference */
-			kauth_cred_rele(my_cred);
+			/* drop old proc reference or our extra reference */
+			kauth_cred_unref(&my_cred);
 			break;
 		}
 
@@ -868,7 +883,7 @@ int
 suser(kauth_cred_t cred, u_short *acflag)
 {
 #if DIAGNOSTIC
-	if (cred == NOCRED || cred == FSCRED)
+	if (!IS_VALID_CRED(cred))
 		panic("suser");
 #endif
 	if (kauth_cred_getuid(cred) == 0) {
@@ -965,7 +980,7 @@ set_security_token(struct proc * p)
 	}
 		
 	/* XXX mach_init doesn't have a p_ucred when it calls this function */
-	if (p->p_ucred != NOCRED && p->p_ucred != FSCRED) {
+	if (IS_VALID_CRED(p->p_ucred)) {
 		sec_token.val[0] = kauth_cred_getuid(p->p_ucred);
 		sec_token.val[1] = p->p_ucred->cr_gid;
 	} else {
diff --git a/bsd/kern/kern_resource.c b/bsd/kern/kern_resource.c
index 21da076ba..c12b5bbd8 100644
--- a/bsd/kern/kern_resource.c
+++ b/bsd/kern/kern_resource.c
@@ -103,7 +103,7 @@ rlim_t maxsmap = MAXSSIZ;	/* XXX */
  *
  * Note: would be in kern/subr_param.c in FreeBSD.
  */
-int maxprocperuid = CHILD_MAX;		/* max # of procs per user */
+extern int maxprocperuid;		/* max # of procs per user */
 int maxfilesperproc = OPEN_MAX;		/* per-proc open files limit */
 
 SYSCTL_INT( _kern, KERN_MAXPROCPERUID, maxprocperuid, CTLFLAG_RW,
diff --git a/bsd/kern/kern_shutdown.c b/bsd/kern/kern_shutdown.c
index 1fcd585ae..592368bb4 100644
--- a/bsd/kern/kern_shutdown.c
+++ b/bsd/kern/kern_shutdown.c
@@ -147,6 +147,10 @@ boot(paniced, howto, command)
 	if (paniced == RB_PANIC)
 		hostboot_option = HOST_REBOOT_HALT;
 
+    if (howto & RB_UPSDELAY) {
+        hostboot_option = HOST_REBOOT_UPSDELAY;
+    }
+
 	/*
 	 * if we're going to power down due to a halt,
 	 * give the disks a chance to finish getting
diff --git a/bsd/kern/kern_sig.c b/bsd/kern/kern_sig.c
index 1492505f3..2333f91fb 100644
--- a/bsd/kern/kern_sig.c
+++ b/bsd/kern/kern_sig.c
@@ -612,8 +612,11 @@ execsigs(p, thr_act)
 {
 	register struct sigacts *ps = p->p_sigacts;
 	register int nc, mask;
-	struct uthread *ut;
+	struct uthread *ut = (struct uthread *)0;
 
+	if (thr_act){
+		ut = (struct uthread *)get_bsdthread_info(thr_act);
+	}
 	/*
 	 * Reset caught signals.  Held signals remain held
 	 * through p_sigmask (unless they were caught,
@@ -627,7 +630,6 @@ execsigs(p, thr_act)
 			if (nc != SIGCONT)
 				p->p_sigignore |= mask;
 			if (thr_act){
-			 	ut = (struct uthread *)get_bsdthread_info(thr_act);
 				ut->uu_siglist &= ~mask;
 				p->p_siglist &= ~mask;
 			} else
@@ -643,6 +645,13 @@ execsigs(p, thr_act)
 	ps->ps_sigstk.ss_size = 0;
 	ps->ps_sigstk.ss_sp = USER_ADDR_NULL;
 	ps->ps_flags = 0;
+	if (thr_act) {
+		ut->uu_sigstk.ss_flags = SA_DISABLE;
+		ut->uu_sigstk.ss_size = 0;
+		ut->uu_sigstk.ss_sp = USER_ADDR_NULL;
+		ut->uu_flag &= ~UT_ALTSTACK;
+	}
+	ps->ps_sigonstack = 0;
 }
 
 /*
@@ -1073,18 +1082,31 @@ int
 sigaltstack(struct proc *p, register struct sigaltstack_args *uap, __unused register_t *retval)
 {
 	struct sigacts *psp;
+	struct user_sigaltstack *pstk;
 	struct user_sigaltstack ss;
+	struct uthread *uth;
+	int uthsigaltstack = 0;
 	int error;
 
+	uth = (struct uthread *)get_bsdthread_info(current_thread());
+	uthsigaltstack = p->p_lflag & P_LTHSIGSTACK;
+
 	psp = p->p_sigacts;
-	if ((psp->ps_flags & SAS_ALTSTACK) == 0)
-		psp->ps_sigstk.ss_flags |= SA_DISABLE;
+	if (uthsigaltstack != 0)  {
+		pstk = &uth->uu_sigstk;
+		if ((uth->uu_flag & UT_ALTSTACK) == 0)
+			uth->uu_sigstk.ss_flags |= SA_DISABLE;
+	} else {
+		pstk = &psp->ps_sigstk;
+		if ((psp->ps_flags & SAS_ALTSTACK) == 0)
+			psp->ps_sigstk.ss_flags |= SA_DISABLE;
+	}
 	if (uap->oss) {
 		if (IS_64BIT_PROCESS(p)) {
-			error = copyout(&psp->ps_sigstk, uap->oss, sizeof(struct user_sigaltstack));
+			error = copyout(pstk, uap->oss, sizeof(struct user_sigaltstack));
 		} else {
 			struct sigaltstack ss32;
-			sigaltstack_64to32(&psp->ps_sigstk, &ss32);
+			sigaltstack_64to32(pstk, &ss32);
 			error = copyout(&ss32, uap->oss, sizeof(struct sigaltstack));
 		}
 		if (error)
@@ -1106,18 +1128,32 @@ sigaltstack(struct proc *p, register struct sigaltstack_args *uap, __unused regi
 	}
 
 	if (ss.ss_flags & SA_DISABLE) {
-		if (psp->ps_sigstk.ss_flags & SA_ONSTACK)
-			return (EINVAL);
-		psp->ps_flags &= ~SAS_ALTSTACK;
-		psp->ps_sigstk.ss_flags = ss.ss_flags;
+		if (uthsigaltstack != 0)  {
+			/* if we are here we are not in the signal handler ;so no need to check */
+			if (uth->uu_sigstk.ss_flags & SA_ONSTACK)
+				return (EINVAL);
+			uth->uu_flag &= ~UT_ALTSTACK;
+			uth->uu_sigstk.ss_flags = ss.ss_flags;
+		} else {
+			if (psp->ps_sigstk.ss_flags & SA_ONSTACK)
+				return (EINVAL);
+			psp->ps_flags &= ~SAS_ALTSTACK;
+			psp->ps_sigstk.ss_flags = ss.ss_flags;
+		}
+		
 		return (0);
 	}
 /* The older stacksize was 8K, enforce that one so no compat problems */
 #define OLDMINSIGSTKSZ 8*1024
 	if (ss.ss_size < OLDMINSIGSTKSZ)
 		return (ENOMEM);
-	psp->ps_flags |= SAS_ALTSTACK;
-	psp->ps_sigstk= ss;
+	if (uthsigaltstack != 0)  {
+		uth->uu_flag |= UT_ALTSTACK;
+		uth->uu_sigstk= ss;
+	} else {
+		psp->ps_flags |= SAS_ALTSTACK;
+		psp->ps_sigstk= ss;
+	}
 	return (0);
 }
 
@@ -1465,7 +1501,7 @@ psignal_lock(p, signum, withlock)
 	register int signum;
 	register int withlock;
 {
-	register int s, prop;
+	register int prop;
 	register sig_t action;
 	thread_t	sig_thread_act;
 	register task_t		sig_task;
@@ -1500,9 +1536,7 @@ psignal_lock(p, signum, withlock)
 		return;
 	}
 
-        s = splhigh();
         KNOTE(&p->p_klist, NOTE_SIGNAL | signum);
-        splx(s);
 
 	/*
 	 * do not send signals to the process that has the thread
@@ -2638,12 +2672,15 @@ static int
 filt_sigattach(struct knote *kn)
 {
 	struct proc *p = current_proc();
+	boolean_t funnel_state;
 
 	kn->kn_ptr.p_proc = p;
 	kn->kn_flags |= EV_CLEAR;		/* automatically set */
 
-	/* XXX lock the proc here while adding to the list? */
+	/* Take the funnel to protect the proc while adding to the list */
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	KNOTE_ATTACH(&p->p_klist, kn);
+	thread_funnel_set(kernel_flock, funnel_state);
 
 	return (0);
 }
@@ -2652,8 +2689,11 @@ static void
 filt_sigdetach(struct knote *kn)
 {
 	struct proc *p = kn->kn_ptr.p_proc;
+	boolean_t funnel_state;
 
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	KNOTE_DETACH(&p->p_klist, kn);
+	thread_funnel_set(kernel_flock, funnel_state);
 }
 
 /*
diff --git a/bsd/kern/kern_symfile.c b/bsd/kern/kern_symfile.c
index 983cf38c4..7d25e43aa 100644
--- a/bsd/kern/kern_symfile.c
+++ b/bsd/kern/kern_symfile.c
@@ -95,7 +95,7 @@ static int
 output_kernel_symbols(struct proc *p)
 {
     struct vnode		*vp;
-    kauth_cred_t		cred = p->p_ucred;	/* XXX */
+    kauth_cred_t		cred = p->p_ucred;	/* XXX unsafe */
     struct vnode_attr		va;
     struct vfs_context		context;
     struct load_command		*cmd;
@@ -120,6 +120,9 @@ output_kernel_symbols(struct proc *p)
     orig_mh	= NULL;
     orig_st	= NULL;
     
+    context.vc_proc = p;
+    context.vc_ucred = kauth_cred_proc_ref(p);
+
     // Dispose of unnecessary gumf, the booter doesn't need to load these
     rc_mh = IODTGetLoaderInfo("Kernel-__HEADER",
 				(void **)&orig_mh, &orig_mhsize);
@@ -140,9 +143,6 @@ output_kernel_symbols(struct proc *p)
     if (rootdevice[0] == 'e' && rootdevice[1] == 'n')
 	goto out;
 
-    context.vc_proc = p;
-    context.vc_ucred = cred;
-
     if ((error = vnode_open("mach.sym", (O_CREAT | FWRITE), (S_IRUSR | S_IRGRP | S_IROTH), 0, &vp, &context)))
         goto out;
 
@@ -331,6 +331,7 @@ out:
 	if (!error) error = error1;
     }
 
+    kauth_cred_unref(&context.vc_ucred);
     return(error);
 }
 /*
@@ -379,7 +380,6 @@ kern_open_file_for_direct_io(const char * name,
     struct kern_direct_file_io_ref_t * ref;
 
     struct proc 		*p;
-    struct ucred 		*cred;
     struct vnode_attr		va;
     int				error;
     off_t			f_offset;
@@ -403,9 +403,8 @@ kern_open_file_for_direct_io(const char * name,
 
     ref->vp = NULL;
     p = current_proc();		// kernproc;
-    cred = p->p_ucred;
     ref->context.vc_proc = p;
-    ref->context.vc_ucred = cred;
+    ref->context.vc_ucred = kauth_cred_proc_ref(p);
 
     if ((error = vnode_open(name, (O_CREAT | FWRITE), (0), 0, &ref->vp, &ref->context)))
         goto out;
@@ -542,13 +541,14 @@ out:
     kprintf("kern_open_file_for_direct_io(%d)\n", error);
 
     if (error && ref) {
-	if (ref->vp) {
-	    vnode_close(ref->vp, FWRITE, &ref->context);
-		ref->vp = NULLVP;
-	}
-
-	kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
-	ref = NULL;
+		if (ref->vp) {
+			vnode_close(ref->vp, FWRITE, &ref->context);
+			ref->vp = NULLVP;
+		}
+
+		kauth_cred_unref(&ref->context.vc_ucred);
+		kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
+		ref = NULL;
     }
 
     return(ref);
@@ -576,6 +576,7 @@ kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref)
 	    kprintf("vnode_close(%d)\n", error);
 		ref->vp = NULLVP;
 	}
+	kauth_cred_unref(&ref->context.vc_ucred);
 	kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
     }
 }
diff --git a/bsd/kern/kern_sysctl.c b/bsd/kern/kern_sysctl.c
index 0d1b7d533..292085e1d 100644
--- a/bsd/kern/kern_sysctl.c
+++ b/bsd/kern/kern_sysctl.c
@@ -108,9 +108,14 @@ extern vm_map_t bsd_pageable_map;
 #include <pexpert/pexpert.h>
 
 #include <machine/machine_routines.h>
+#include <machine/exec.h>
 
 #include <vm/vm_protos.h>
 
+#ifdef __i386__
+#include <i386/cpuid.h>
+#endif
+
 sysctlfn kern_sysctl;
 #ifdef DEBUG
 sysctlfn debug_sysctl;
@@ -122,10 +127,10 @@ extern sysctlfn cpu_sysctl;
 extern int aio_max_requests;  				
 extern int aio_max_requests_per_process;	
 extern int aio_worker_threads;				
-extern int maxprocperuid;
 extern int maxfilesperproc;
 extern int lowpri_IO_window_msecs;
 extern int lowpri_IO_delay_msecs;
+extern int nx_enabled;
 
 static void
 fill_eproc(struct proc *p, struct eproc *ep);
@@ -319,7 +324,7 @@ __sysctl(struct proc *p, struct __sysctl_args *uap, __unused register_t *retval)
 	if (uap->new != USER_ADDR_NULL
 	    && ((name[0] == CTL_KERN
 		&& !(name[1] == KERN_IPC || name[1] == KERN_PANICINFO || name[1] == KERN_PROCDELAYTERM || 
-		     name[1] == KERN_PROC_LOW_PRI_IO))
+		     name[1] == KERN_PROC_LOW_PRI_IO || name[1] == KERN_PROCNAME || name[1] == KERN_THALTSTACK))
 	    || (name[0] == CTL_HW)
 	    || (name[0] == CTL_VM)
 		|| (name[0] == CTL_VFS))
@@ -433,9 +438,6 @@ __sysctl(struct proc *p, struct __sysctl_args *uap, __unused register_t *retval)
 /*
  * Attributes stored in the kernel.
  */
-extern char classichandler[32];
-extern uint32_t classichandler_fsid;
-extern long classichandler_fileid;
 __private_extern__ char corefilename[MAXPATHLEN+1];
 __private_extern__ int do_coredump;
 __private_extern__ int sugid_coredump;
@@ -474,8 +476,9 @@ sysctl_affinity(
 	return (ENOTSUP);
 }
 
+
 static int
-sysctl_classic(
+sysctl_translate(
 	int *name,
 	u_int namelen,
 	user_addr_t oldBuf,
@@ -498,11 +501,60 @@ sysctl_classic(
 		return (EPERM);
 
 	return sysctl_rdint(oldBuf, oldSize, newBuf,
-		                (p->p_flag & P_CLASSIC) ? 1 : 0);
+		                (p->p_flag & P_TRANSLATED) ? 1 : 0);
+}
+
+int
+set_archhandler(struct proc *p, int arch)
+{
+	int error;
+	struct nameidata nd;
+	struct vnode_attr va;
+	struct vfs_context context;
+	char *archhandler;
+
+	switch(arch) {
+	case CPU_TYPE_POWERPC:
+		archhandler = exec_archhandler_ppc.path;
+		break;
+	default:
+		return (EBADARCH);
+	}
+
+	context.vc_proc = p;
+	context.vc_ucred = kauth_cred_get();
+	
+	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE32,
+		   CAST_USER_ADDR_T(archhandler), &context);
+	error = namei(&nd);
+	if (error)
+		return (error);
+	nameidone(&nd);
+	
+	/* Check mount point */
+	if ((nd.ni_vp->v_mount->mnt_flag & MNT_NOEXEC) ||
+		(nd.ni_vp->v_type != VREG)) {
+		vnode_put(nd.ni_vp);
+		return (EACCES);
+	}
+	
+	VATTR_INIT(&va);
+	VATTR_WANTED(&va, va_fsid);
+	VATTR_WANTED(&va, va_fileid);
+	error = vnode_getattr(nd.ni_vp, &va, &context);
+	if (error) {
+		vnode_put(nd.ni_vp);
+		return (error);
+	}
+	vnode_put(nd.ni_vp);
+	
+	exec_archhandler_ppc.fsid = va.va_fsid;
+	exec_archhandler_ppc.fileid = (u_long)va.va_fileid;
+	return 0;
 }
 
 static int
-sysctl_classichandler(
+sysctl_exec_archhandler_ppc(
 	__unused int *name,
 	__unused u_int namelen,
 	user_addr_t oldBuf,
@@ -515,18 +567,18 @@ sysctl_classichandler(
 	size_t len;
 	struct nameidata nd;
 	struct vnode_attr va;
-	char handler[sizeof(classichandler)];
+	char handler[sizeof(exec_archhandler_ppc.path)];
 	struct vfs_context context;
 
 	context.vc_proc = p;
 	context.vc_ucred = kauth_cred_get();
 
 	if (oldSize) {
-		len = strlen(classichandler) + 1;
+		len = strlen(exec_archhandler_ppc.path) + 1;
 		if (oldBuf) {
 			if (*oldSize < len)
 				return (ENOMEM);
-			error = copyout(classichandler, oldBuf, len);
+			error = copyout(exec_archhandler_ppc.path, oldBuf, len);
 			if (error)
 				return (error);
 		}
@@ -536,44 +588,26 @@ sysctl_classichandler(
 		error = suser(context.vc_ucred, &p->p_acflag);
 		if (error)
 			return (error);
-		if (newSize >= sizeof(classichandler))
+		if (newSize >= sizeof(exec_archhandler_ppc.path))
 			return (ENAMETOOLONG);
 		error = copyin(newBuf, handler, newSize);
 		if (error)
 			return (error);
 		handler[newSize] = 0;
-
-		NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE32,
-				CAST_USER_ADDR_T(handler), &context);
-		error = namei(&nd);
+		strcpy(exec_archhandler_ppc.path, handler);
+		error = set_archhandler(p, CPU_TYPE_POWERPC);
 		if (error)
 			return (error);
-		nameidone(&nd);
-
-		/* Check mount point */
-		if ((nd.ni_vp->v_mount->mnt_flag & MNT_NOEXEC) ||
-			(nd.ni_vp->v_type != VREG)) {
-			vnode_put(nd.ni_vp);
-			return (EACCES);
-		}
-
-		VATTR_INIT(&va);
-		VATTR_WANTED(&va, va_fsid);
-		VATTR_WANTED(&va, va_fileid);
-		error = vnode_getattr(nd.ni_vp, &va, &context);
-		if (error) {
-			vnode_put(nd.ni_vp);
-			return (error);
-		}
-		vnode_put(nd.ni_vp);
-
-		classichandler_fsid = va.va_fsid;
-		classichandler_fileid = (u_long)va.va_fileid;
-		strcpy(classichandler, handler);
 	}
 	return 0;
 }
 
+SYSCTL_NODE(_kern, KERN_EXEC, exec, CTLFLAG_RD, 0, "");
+
+SYSCTL_NODE(_kern_exec, OID_AUTO, archhandler, CTLFLAG_RD, 0, "");
+
+SYSCTL_STRING(_kern_exec_archhandler, OID_AUTO, powerpc, CTLFLAG_RD,
+		exec_archhandler_ppc.path, 0, "");
 
 extern int get_kernel_symfile( struct proc *, char **);
 __private_extern__ int 
@@ -601,9 +635,11 @@ kern_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
 			|| name[0] == KERN_IPC
 			|| name[0] == KERN_SYSV
 			|| name[0] == KERN_AFFINITY
-			|| name[0] == KERN_CLASSIC
+			|| name[0] == KERN_TRANSLATE
+			|| name[0] == KERN_EXEC
 			|| name[0] == KERN_PANICINFO
-			|| name[0] == KERN_POSIX)
+			|| name[0] == KERN_POSIX
+			|| name[0] == KERN_TFP)
 		)
 		return (ENOTDIR);		/* overloaded */
 
@@ -719,12 +755,12 @@ kern_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
 	case KERN_AFFINITY:
 		return sysctl_affinity(name+1, namelen-1, oldp, oldlenp,
 									newp, newlen, p);
-	case KERN_CLASSIC:
-		return sysctl_classic(name+1, namelen-1, oldp, oldlenp,
-								newp, newlen, p);
+	case KERN_TRANSLATE:
+		return sysctl_translate(name+1, namelen-1, oldp, oldlenp, newp,
+				      newlen, p);
 	case KERN_CLASSICHANDLER:
-		return sysctl_classichandler(name+1, namelen-1, oldp, oldlenp,
-										newp, newlen, p);
+		return sysctl_exec_archhandler_ppc(name+1, namelen-1, oldp,
+						   oldlenp, newp, newlen, p);
 	case KERN_AIOMAX:
 		return( sysctl_aiomax( oldp, oldlenp, newp, newlen ) );
 	case KERN_AIOPROCMAX:
@@ -851,9 +887,76 @@ kern_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
         	}
 		return(error);
 	}
+	case KERN_NX_PROTECTION:
+	{
+		int	 old_value, new_value;
+
+		error = 0;
+		if (oldp && *oldlenp < sizeof(old_value) )
+			return (ENOMEM);
+		if ( newp && newlen != sizeof(new_value) )
+			return(EINVAL);
+		*oldlenp = sizeof(old_value);
+
+		old_value = nx_enabled;
+
+		if (oldp && (error = copyout( &old_value, oldp, *oldlenp)))
+			return(error);
+#ifdef __i386__
+		/*
+		 * Only allow setting if NX is supported on the chip
+		 */
+		if (cpuid_extfeatures() & CPUID_EXTFEATURE_XD) {
+#endif
+        		if (error == 0 && newp)
+                		error = copyin(newp, &new_value,
+					       sizeof(newlen));
+        		if (error == 0 && newp)
+		        	nx_enabled = new_value;
+#ifdef __i386__
+        	} else if (newp) {
+			error = ENOTSUP;
+		}
+#endif
+		return(error);
+	}
 	case KERN_SHREG_PRIVATIZABLE:
 		/* this kernel does implement shared_region_make_private_np() */
 		return (sysctl_rdint(oldp, oldlenp, newp, 1));
+	case KERN_PROCNAME:
+		error = sysctl_trstring(oldp, oldlenp, newp, newlen,
+		    &p->p_name[0], (2*MAXCOMLEN+1));
+		return (error);
+	case KERN_THALTSTACK:
+	{
+		int	 old_value, new_value;
+
+		error = 0;
+		if (oldp && *oldlenp < sizeof(int))
+			return (ENOMEM);
+		if ( newp && newlen != sizeof(int) )
+			return(EINVAL);
+		*oldlenp = sizeof(int);
+		old_value = (p->p_lflag & P_LTHSIGSTACK)? 1: 0;
+		if (oldp && (error = copyout( &old_value, oldp, sizeof(int))))
+			return(error);
+        	if (error == 0 && newp )
+                	error = copyin( newp, &new_value, sizeof(int) );
+        	if (error == 0 && newp) {
+                	if (new_value) {
+							/* we cannot swich midstream if inuse */
+							if ((p->p_sigacts->ps_flags & SAS_ALTSTACK) == SAS_ALTSTACK)
+								return(EPERM);
+                        	p->p_lflag |=  P_LTHSIGSTACK;
+                	} else {
+							/* we cannot swich midstream */
+							if ((p->p_lflag & P_LTHSIGSTACK) == P_LTHSIGSTACK)
+								return(EPERM);
+							p->p_lflag &=  ~P_LTHSIGSTACK;
+					}
+        	}
+		return(error);
+	}
 	default:
 		return (ENOTSUP);
 	}
@@ -2095,3 +2198,98 @@ sysctl_maxproc(user_addr_t oldp, size_t *oldlenp,
 	return( error );
 	
 } /* sysctl_maxproc */
+
+#if __i386__
+static int
+sysctl_sysctl_exec_affinity SYSCTL_HANDLER_ARGS
+{
+	struct proc *cur_proc = req->p;
+	int error;
+	
+	if (req->oldptr != USER_ADDR_NULL) {
+		cpu_type_t oldcputype = (cur_proc->p_flag & P_AFFINITY) ? CPU_TYPE_POWERPC : CPU_TYPE_I386;
+		if ((error = SYSCTL_OUT(req, &oldcputype, sizeof(oldcputype))))
+			return error;
+	}
+
+	if (req->newptr != USER_ADDR_NULL) {
+		cpu_type_t newcputype;
+		if ((error = SYSCTL_IN(req, &newcputype, sizeof(newcputype))))
+			return error;
+		if (newcputype == CPU_TYPE_I386)
+			cur_proc->p_flag &= ~P_AFFINITY;
+		else if (newcputype == CPU_TYPE_POWERPC)
+			cur_proc->p_flag |= P_AFFINITY;
+		else
+			return (EINVAL);
+	}
+	
+	return 0;
+}
+SYSCTL_PROC(_sysctl, OID_AUTO, proc_exec_affinity, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY, 0, 0, sysctl_sysctl_exec_affinity ,"I","proc_exec_affinity");
+#endif
+
+static int
+fetch_process_cputype(
+	struct proc *cur_proc,
+	int *name,
+	u_int namelen,
+	cpu_type_t *cputype)
+{
+	struct proc *p = NULL;
+	cpu_type_t ret = 0;
+	
+	if (namelen == 0)
+		p = cur_proc;
+	else if (namelen == 1) {
+		p = pfind(name[0]);
+		if (p == NULL)
+			return (EINVAL);
+		if ((kauth_cred_getuid(p->p_ucred) != kauth_cred_getuid(kauth_cred_get())) 
+			&& suser(kauth_cred_get(), &cur_proc->p_acflag))
+			return (EPERM);
+	} else {
+		return EINVAL;
+	}
+
+#if __i386__
+	if (p->p_flag & P_TRANSLATED) {
+		ret = CPU_TYPE_POWERPC;
+	}
+	else
+#endif
+	{
+		ret = cpu_type();
+		if (IS_64BIT_PROCESS(p))
+			ret |= CPU_ARCH_ABI64;
+	}
+	*cputype = ret;
+	
+	return 0;
+}
+
+static int
+sysctl_sysctl_native SYSCTL_HANDLER_ARGS
+{
+	int error;
+	cpu_type_t proc_cputype = 0;
+	if ((error = fetch_process_cputype(req->p, (int *)arg1, arg2, &proc_cputype)) != 0)
+		return error;
+	int res = 1;
+	if ((proc_cputype & ~CPU_ARCH_MASK) != (cpu_type() & ~CPU_ARCH_MASK))
+		res = 0;
+	return SYSCTL_OUT(req, &res, sizeof(res));
+}	
+SYSCTL_PROC(_sysctl, OID_AUTO, proc_native, CTLTYPE_NODE|CTLFLAG_RD, 0, 0, sysctl_sysctl_native ,"I","proc_native");
+
+static int
+sysctl_sysctl_cputype SYSCTL_HANDLER_ARGS
+{
+	int error;
+	cpu_type_t proc_cputype = 0;
+	if ((error = fetch_process_cputype(req->p, (int *)arg1, arg2, &proc_cputype)) != 0)
+		return error;
+	return SYSCTL_OUT(req, &proc_cputype, sizeof(proc_cputype));
+}
+SYSCTL_PROC(_sysctl, OID_AUTO, proc_cputype, CTLTYPE_NODE|CTLFLAG_RD, 0, 0, sysctl_sysctl_cputype ,"I","proc_cputype");
+
diff --git a/bsd/kern/kern_time.c b/bsd/kern/kern_time.c
index 2948aff98..906ba9ead 100644
--- a/bsd/kern/kern_time.c
+++ b/bsd/kern/kern_time.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -89,14 +89,6 @@ static void		setthetime(
 
 void time_zone_slock_init(void);
 
-int gettimeofday(struct proc *p,
-#ifdef __ppc__
-			 struct ppc_gettimeofday_args *uap, 
-#else			 
-			 struct gettimeofday_args *uap, 
-#endif
-			 register_t *retval);
-
 /* 
  * Time of day and interval timer support.
  *
@@ -105,52 +97,29 @@ int gettimeofday(struct proc *p,
  * here provide support for adding and subtracting timeval structures
  * and decrementing interval timers, optionally reloading the interval
  * timers when they expire.
- *
- * XXX Y2038 bug because of clock_get_calendar_microtime() first argument
  */
 /* ARGSUSED */
 int
-gettimeofday(__unused struct proc *p,
-#ifdef __ppc__
-			 register struct ppc_gettimeofday_args *uap, 
-#else			 
-			 register struct gettimeofday_args *uap, 
-#endif
-			 __unused register_t *retval)
+gettimeofday(
+__unused	struct proc	*p,
+			struct gettimeofday_args *uap, 
+			register_t *retval)
 {
-	struct timeval atv;
 	int error = 0;
 	struct timezone ltz; /* local copy */
 
-/*  NOTE THIS implementation is for non ppc architectures only */
-
-	if (uap->tp) {
-		clock_get_calendar_microtime((uint32_t *)&atv.tv_sec, &atv.tv_usec);
-		if (IS_64BIT_PROCESS(p)) {
-			struct user_timeval user_atv;
-			user_atv.tv_sec = atv.tv_sec;
-			user_atv.tv_usec = atv.tv_usec;
-			/*
-			 * This cast is not necessary for PPC, but is
-			 * mostly harmless.
-			 */
-			error = copyout(&user_atv, CAST_USER_ADDR_T(uap->tp), sizeof(struct user_timeval));
-		} else {
-			error = copyout(&atv, CAST_USER_ADDR_T(uap->tp), sizeof(struct timeval));
-		}
-		if (error)
-			return(error);
-	}
+	if (uap->tp)
+		clock_gettimeofday(&retval[0], &retval[1]);
 	
 	if (uap->tzp) {
 		lck_spin_lock(tz_slock);
 		ltz = tz;
 		lck_spin_unlock(tz_slock);
-		error = copyout((caddr_t)&ltz, CAST_USER_ADDR_T(uap->tzp),
-		    sizeof (tz));
+
+		error = copyout((caddr_t)&ltz, CAST_USER_ADDR_T(uap->tzp), sizeof (tz));
 	}
 
-	return(error);
+	return (error);
 }
 
 /*
@@ -611,13 +580,11 @@ time_zone_slock_init(void)
 {
 	/* allocate lock group attribute and group */
 	tz_slock_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(tz_slock_grp_attr);
 
 	tz_slock_grp =  lck_grp_alloc_init("tzlock", tz_slock_grp_attr);
 
 	/* Allocate lock attribute */
 	tz_slock_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(tz_slock_attr);
 
 	/* Allocate the spin lock */
 	tz_slock = lck_spin_alloc_init(tz_slock_grp, tz_slock_attr);
diff --git a/bsd/kern/kpi_mbuf.c b/bsd/kern/kpi_mbuf.c
index 61ac09968..9ea0ae567 100644
--- a/bsd/kern/kpi_mbuf.c
+++ b/bsd/kern/kpi_mbuf.c
@@ -678,12 +678,10 @@ mbuf_tag_id_find_internal(
 		lck_mtx_t		*new_lock = NULL;
 		
 		grp_attrib = lck_grp_attr_alloc_init();
-		lck_grp_attr_setdefault(grp_attrib);
 		lck_group = lck_grp_alloc_init("mbuf_tag_allocate_id", grp_attrib);
 		lck_grp_attr_free(grp_attrib);
 		lck_attrb = lck_attr_alloc_init();
-		lck_attr_setdefault(lck_attrb);
-		lck_attr_setdebug(lck_attrb);
+
 		new_lock = lck_mtx_alloc_init(lck_group, lck_attrb);
 		if (!OSCompareAndSwap((UInt32)0, (UInt32)new_lock, (UInt32*)&mtag_id_lock)) {
 			/*
diff --git a/bsd/kern/kpi_socketfilter.c b/bsd/kern/kpi_socketfilter.c
index 9e3c9e59a..835e07079 100644
--- a/bsd/kern/kpi_socketfilter.c
+++ b/bsd/kern/kpi_socketfilter.c
@@ -52,12 +52,9 @@ sflt_init(void)
 	
 	/* Allocate a spin lock */
 	grp_attrib = lck_grp_attr_alloc_init();
-	lck_grp_attr_setdefault(grp_attrib);
 	lck_group = lck_grp_alloc_init("socket filter lock", grp_attrib);
 	lck_grp_attr_free(grp_attrib);
 	lck_attrib = lck_attr_alloc_init();
-	lck_attr_setdefault(lck_attrib);
-	lck_attr_setdebug(lck_attrib);
 	sock_filter_lock = lck_mtx_alloc_init(lck_group, lck_attrib);
 	lck_grp_free(lck_group);
 	lck_attr_free(lck_attrib);
diff --git a/bsd/kern/mach_fat.c b/bsd/kern/mach_fat.c
index a9418aa8f..677ae0008 100644
--- a/bsd/kern/mach_fat.c
+++ b/bsd/kern/mach_fat.c
@@ -44,13 +44,8 @@
 #include <kern/cpu_number.h>
 #include <mach-o/fat.h>
 #include <kern/mach_loader.h>
-#include <architecture/byte_order.h>
-
-/* XXX should be in common header */
-extern int grade_binary(cpu_type_t exectype, cpu_subtype_t execsubtype);
-
-#define CPU_TYPE_NATIVE		(cpu_type())
-#define CPU_TYPE_CLASSIC	CPU_TYPE_POWERPC
+#include <libkern/OSByteOrder.h>
+#include <machine/exec.h>
 
 /**********************************************************************
  * Routine:	fatfile_getarch2()
@@ -106,7 +101,7 @@ fatfile_getarch2(
 	 *	Map portion that must be accessible directly into
 	 *	kernel's map.
 	 */
-	nfat_arch = NXSwapBigLongToHost(header->nfat_arch);
+	nfat_arch = OSSwapBigToHostInt32(header->nfat_arch);
 
 	end_of_archs = (off_t)nfat_arch * sizeof(struct fat_arch) + 
 		sizeof(struct fat_header);
@@ -124,6 +119,7 @@ fatfile_getarch2(
 	if (end_of_archs > PAGE_SIZE ||
 		end_of_archs < (sizeof(struct fat_header)+sizeof(struct fat_arch)))
 		return(LOAD_BADMACHO);
+
 	/*
 	 * 	Round size of fat_arch structures up to page boundry.
 	 */
@@ -143,15 +139,15 @@ fatfile_getarch2(
 		/*
 		 *	Check to see if right cpu type.
 		 */
-		if(((cpu_type_t)NXSwapBigIntToHost(arch->cputype) & ~mask_bits) != req_cpu_type)
+		if(((cpu_type_t)OSSwapBigToHostInt32(arch->cputype) & ~mask_bits) != req_cpu_type)
 			continue;
 
 		/*
 		 * 	Get the grade of the cpu subtype.
 		 */
 		grade = grade_binary(
-			    NXSwapBigIntToHost(arch->cputype),
-			    NXSwapBigIntToHost(arch->cpusubtype));
+			    OSSwapBigToHostInt32(arch->cputype),
+			    OSSwapBigToHostInt32(arch->cpusubtype));
 
 		/*
 		 *	Remember it if it's the best we've seen.
@@ -169,15 +165,15 @@ fatfile_getarch2(
 		lret = LOAD_BADARCH;
 	} else {
 		archret->cputype	=
-			    NXSwapBigIntToHost(best_arch->cputype);
+			    OSSwapBigToHostInt32(best_arch->cputype);
 		archret->cpusubtype	=
-			    NXSwapBigIntToHost(best_arch->cpusubtype);
+			    OSSwapBigToHostInt32(best_arch->cpusubtype);
 		archret->offset		=
-			    NXSwapBigLongToHost(best_arch->offset);
+			    OSSwapBigToHostInt32(best_arch->offset);
 		archret->size		=
-			    NXSwapBigLongToHost(best_arch->size);
+			    OSSwapBigToHostInt32(best_arch->size);
 		archret->align		=
-			    NXSwapBigLongToHost(best_arch->align);
+			    OSSwapBigToHostInt32(best_arch->align);
 
 		lret = LOAD_SUCCESS;
 	}
@@ -188,8 +184,6 @@ fatfile_getarch2(
 	return(lret);
 }
 
-extern char classichandler[];
-
 load_return_t
 fatfile_getarch_affinity(
 		struct vnode		*vp,
@@ -198,15 +192,15 @@ fatfile_getarch_affinity(
 		int 				affinity)
 {
 		load_return_t lret;
-		int handler = (classichandler[0] != 0);
+		int handler = (exec_archhandler_ppc.path[0] != 0);
 		cpu_type_t primary_type, fallback_type;
 
 		if (handler && affinity) {
-				primary_type = CPU_TYPE_CLASSIC;
-				fallback_type = CPU_TYPE_NATIVE;
+				primary_type = CPU_TYPE_POWERPC;
+				fallback_type = cpu_type();
 		} else {
-				primary_type = CPU_TYPE_NATIVE;
-				fallback_type = CPU_TYPE_CLASSIC;
+				primary_type = cpu_type();
+				fallback_type = CPU_TYPE_POWERPC;
 		}
 		/*
 		 * Ignore the architectural bits when determining if an image
@@ -240,7 +234,7 @@ fatfile_getarch(
 	vm_offset_t 	data_ptr,
 	struct fat_arch		*archret)
 {
-	return fatfile_getarch2(vp, data_ptr, CPU_TYPE_NATIVE, 0, archret);
+	return fatfile_getarch2(vp, data_ptr, cpu_type(), 0, archret);
 }
 
 /**********************************************************************
@@ -265,6 +259,6 @@ fatfile_getarch_with_bits(
 	vm_offset_t 	data_ptr,
 	struct fat_arch		*archret)
 {
-	return fatfile_getarch2(vp, data_ptr, archbits | CPU_TYPE_NATIVE, 0, archret);
+	return fatfile_getarch2(vp, data_ptr, archbits | cpu_type(), 0, archret);
 }
 
diff --git a/bsd/kern/mach_loader.c b/bsd/kern/mach_loader.c
index 5f10965a1..42a182a9b 100644
--- a/bsd/kern/mach_loader.c
+++ b/bsd/kern/mach_loader.c
@@ -82,7 +82,7 @@
  * XXX vm/pmap.h should not treat these prototypes as MACH_KERNEL_PRIVATE
  * when KERNEL is defined.
  */
-extern pmap_t	pmap_create(vm_map_size_t size);
+extern pmap_t	pmap_create(vm_map_size_t size, boolean_t is_64bit);
 extern void	pmap_switch(pmap_t);
 extern void	pmap_map_sharedpage(task_t task, pmap_t pmap);
 
@@ -98,7 +98,6 @@ extern kern_return_t    thread_state_initialize(thread_t thread);
 
 
 /* XXX should have prototypes in a shared header file */
-extern int grade_binary(cpu_type_t exectype, cpu_subtype_t execsubtype);
 extern int	get_map_nentries(vm_map_t);
 extern kern_return_t	thread_userstack(thread_t, int, thread_state_t,
 				unsigned int, mach_vm_offset_t *, int *);
@@ -202,7 +201,8 @@ load_dylinker(
 	thread_t			thr_act,
 	int						depth,
 	load_result_t			*result,
-	boolean_t			clean_regions
+	boolean_t			clean_regions,
+	boolean_t			is_64bit
 );
 
 static load_return_t
@@ -242,19 +242,22 @@ load_machfile(
 
 	if (create_map) {
 		old_map = current_map();
-#ifdef i386
+#ifdef NO_NESTED_PMAP
 		pmap = get_task_pmap(current_task());
 		pmap_reference(pmap);
-#else
-		pmap = pmap_create((vm_map_size_t) 0);
-#endif
+#else	/* NO_NESTED_PMAP */
+		pmap = pmap_create((vm_map_size_t) 0, (imgp->ip_flags & IMGPF_IS_64BIT));
+#endif	/* NO_NESTED_PMAP */
 		map = vm_map_create(pmap,
-				get_map_min(old_map),
-				get_map_max(old_map),
-				TRUE); /**** FIXME ****/
+				0,
+				vm_compute_max_offset((imgp->ip_flags & IMGPF_IS_64BIT)),
+				TRUE);
 	} else
 		map = new_map;
-		
+
+	if ( (header->flags & MH_ALLOW_STACK_EXECUTION) )
+	        vm_map_disable_NX(map);
+
 	if (!result)
 		result = &myresult;
 
@@ -271,6 +274,15 @@ load_machfile(
 		return(lret);
 	}
 
+	/*
+	 * For 64-bit users, check for presence of a 4GB page zero
+	 * which will enable the kernel to share the user's address space
+	 * and hence avoid TLB flushes on kernel entry/exit
+	 */ 
+	if ((imgp->ip_flags & IMGPF_IS_64BIT) &&
+	     vm_map_has_4GB_pagezero(map))
+		vm_map_set_4GB_pagezero(map);
+
 	/*
 	 *	Commit to new map.  First make sure that the current
 	 *	users of the task get done with it, and that we clean
@@ -282,13 +294,15 @@ load_machfile(
 	 *	That lets us get off the pmap associated with it, and
 	 *	then we can release it.
 	 */
+
 	 if (create_map) {
 		task_halt(current_task());
 
 		old_map = swap_task_map(current_task(), map);
-#ifndef i386
+		vm_map_clear_4GB_pagezero(old_map);
+#ifndef NO_NESTED_PMAP
 		pmap_switch(pmap);	/* Make sure we are using the new pmap */
-#endif
+#endif	/* !NO_NESTED_PMAP */
 		vm_map_deallocate(old_map);
 	}
 	return(LOAD_SUCCESS);
@@ -628,19 +642,29 @@ RedoLookup:
 		}
             }
 	    if (dlp != 0)
-			ret = load_dylinker(dlp, dlarchbits, map, thr_act, depth, result, clean_regions);
+			ret = load_dylinker(dlp, dlarchbits, map, thr_act, depth, result, clean_regions, abi64);
 
 	    if(depth == 1) {
 		if (result->thread_count == 0)
 			ret = LOAD_FAILURE;
-#ifdef __ppc__
 		else if ( abi64 ) {
 			/* Map in 64-bit commpage */
 			/* LP64todo - make this clean */
 			pmap_map_sharedpage(current_task(), get_map_pmap(map));
                         vm_map_commpage64(map);
+		} else {
+#ifdef __i386__
+			/*
+			 * On Intel, the comm page doesn't get mapped
+			 * automatically because it goes beyond the current end
+			 * of the VM map in the current 3GB/1GB address space
+			 * model.
+			 * XXX This will probably become unnecessary when we
+			 * switch to the 4GB/4GB address space model.
+			 */
+			vm_map_commpage32(map);
+#endif	/* __i386__ */
 		}
-#endif
 	    }
 	}
 
@@ -653,6 +677,62 @@ RedoLookup:
 	return(ret);
 }
 
+#ifndef SG_PROTECTED_VERSION_1
+#define SG_PROTECTED_VERSION_1 0x8
+#endif /* SG_PROTECTED_VERSION_1 */
+
+#ifdef __i386__
+
+#define	APPLE_UNPROTECTED_HEADER_SIZE	(3 * PAGE_SIZE_64)
+
+static load_return_t
+unprotect_segment_64(
+	uint64_t	file_off,
+	uint64_t	file_size,
+	vm_map_t	map,
+	vm_map_offset_t	map_addr,
+	vm_map_size_t	map_size)
+{
+	kern_return_t	kr;
+
+	/*
+	 * The first APPLE_UNPROTECTED_HEADER_SIZE bytes (from offset 0 of
+	 * this part of a Universal binary) are not protected...
+	 * The rest needs to be "transformed".
+	 */
+	if (file_off <= APPLE_UNPROTECTED_HEADER_SIZE &&
+	    file_off + file_size <= APPLE_UNPROTECTED_HEADER_SIZE) {
+		/* it's all unprotected, nothing to do... */
+		kr = KERN_SUCCESS;
+	} else {
+		if (file_off <= APPLE_UNPROTECTED_HEADER_SIZE) {
+			/*
+			 * We start mapping in the unprotected area.
+			 * Skip the unprotected part...
+			 */
+			vm_map_offset_t	delta;
+
+			delta = APPLE_UNPROTECTED_HEADER_SIZE;
+			delta -= file_off;
+			map_addr += delta;
+			map_size -= delta;
+		}
+		/* ... transform the rest of the mapping. */
+		kr = vm_map_apple_protected(map,
+					    map_addr,
+					    map_addr + map_size);
+	}
+
+	if (kr != KERN_SUCCESS) {
+		return LOAD_FAILURE;
+	}
+	return LOAD_SUCCESS;
+}
+#else	/* __i386__ */
+#define unprotect_segment_64(file_off, file_size, map, map_addr, map_size) \
+	LOAD_SUCCESS
+#endif	/* __i386__ */
+
 static
 load_return_t
 load_segment(
@@ -688,6 +768,27 @@ load_segment(
 	map_size = round_page(scp->filesize);
 	map_addr = trunc_page(scp->vmaddr);
 
+#if 0	/* XXX (4596982) this interferes with Rosetta */
+	if (map_addr == 0 &&
+	    map_size == 0 &&
+	    seg_size != 0 &&
+	    (scp->initprot & VM_PROT_ALL) == VM_PROT_NONE &&
+	    (scp->maxprot & VM_PROT_ALL) == VM_PROT_NONE) {
+		/*
+		 * This is a "page zero" segment:  it starts at address 0,
+		 * is not mapped from the binary file and is not accessible.
+		 * User-space should never be able to access that memory, so
+		 * make it completely off limits by raising the VM map's
+		 * minimum offset.
+		 */
+		ret = vm_map_raise_min_offset(map, (vm_map_offset_t) seg_size);
+		if (ret != KERN_SUCCESS) {
+			return LOAD_FAILURE;
+		}
+		return LOAD_SUCCESS;
+	}
+#endif
+
 	map_offset = pager_offset + scp->fileoff;
 
 	if (map_size > 0) {
@@ -738,28 +839,28 @@ load_segment(
 	if (delta_size > 0) {
 		vm_offset_t	tmp = map_addr + map_size;
 
-		ret = vm_allocate(map, &tmp, delta_size, VM_FLAGS_FIXED);
+		ret = vm_map(map, &tmp, delta_size, 0, VM_FLAGS_FIXED,
+			     NULL, 0, FALSE,
+			     scp->initprot, scp->maxprot,
+			     VM_INHERIT_DEFAULT);
 		if (ret != KERN_SUCCESS)
 			return(LOAD_NOSPACE);
 	}
 
-	/*
-	 *	Set protection values. (Note: ignore errors!)
-	 */
-
-	if (scp->maxprot != VM_PROT_DEFAULT) {
-		(void) vm_protect(map,
-					map_addr, seg_size,
-					TRUE, scp->maxprot);
-	}
-	if (scp->initprot != VM_PROT_DEFAULT) {
-		(void) vm_protect(map,
-				      map_addr, seg_size,
-				      FALSE, scp->initprot);
-	}
 	if ( (scp->fileoff == 0) && (scp->filesize != 0) )
 		result->mach_header = map_addr;
-	return(LOAD_SUCCESS);
+
+	if (scp->flags & SG_PROTECTED_VERSION_1) {
+		ret = unprotect_segment_64((uint64_t) scp->fileoff,
+					   (uint64_t) scp->filesize,
+					   map,
+					   (vm_map_offset_t) map_addr,
+					   (vm_map_size_t) map_size);
+	} else {
+		ret = LOAD_SUCCESS;
+	}
+
+	return ret;
 }
 
 static
@@ -797,6 +898,25 @@ load_segment_64(
 	map_size = round_page_64(scp64->filesize);	/* limited to 32 bits */
 	map_addr = round_page_64(scp64->vmaddr);
 
+	if (map_addr == 0 &&
+	    map_size == 0 &&
+	    seg_size != 0 &&
+	    (scp64->initprot & VM_PROT_ALL) == VM_PROT_NONE &&
+	    (scp64->maxprot & VM_PROT_ALL) == VM_PROT_NONE) {
+		/*
+		 * This is a "page zero" segment:  it starts at address 0,
+		 * is not mapped from the binary file and is not accessible.
+		 * User-space should never be able to access that memory, so
+		 * make it completely off limits by raising the VM map's
+		 * minimum offset.
+		 */
+		ret = vm_map_raise_min_offset(map, seg_size);
+		if (ret != KERN_SUCCESS) {
+			return LOAD_FAILURE;
+		}
+		return LOAD_SUCCESS;
+	}
+
 	map_offset = pager_offset + scp64->fileoff;	/* limited to 32 bits */
 
 	if (map_size > 0) {
@@ -847,28 +967,28 @@ load_segment_64(
 	if (delta_size > 0) {
 		mach_vm_offset_t tmp = map_addr + map_size;
 
-		ret = mach_vm_allocate(map, &tmp, delta_size, VM_FLAGS_FIXED);
+		ret = mach_vm_map(map, &tmp, delta_size, 0, VM_FLAGS_FIXED,
+				  NULL, 0, FALSE,
+				  scp64->initprot, scp64->maxprot,
+				  VM_INHERIT_DEFAULT);
 		if (ret != KERN_SUCCESS)
 			return(LOAD_NOSPACE);
 	}
 
-	/*
-	 *	Set protection values. (Note: ignore errors!)
-	 */
-
-	if (scp64->maxprot != VM_PROT_DEFAULT) {
-		(void) mach_vm_protect(map,
-					map_addr, seg_size,
-					TRUE, scp64->maxprot);
-	}
-	if (scp64->initprot != VM_PROT_DEFAULT) {
-		(void) mach_vm_protect(map,
-				      map_addr, seg_size,
-				      FALSE, scp64->initprot);
-	}
 	if ( (scp64->fileoff == 0) && (scp64->filesize != 0) )
 		result->mach_header = map_addr;
-	return(LOAD_SUCCESS);
+
+	if (scp64->flags & SG_PROTECTED_VERSION_1) {
+		ret = unprotect_segment_64(scp64->fileoff,
+					   scp64->filesize,
+					   map,
+					   map_addr,
+					   map_size);
+	} else {
+		ret = LOAD_SUCCESS;
+	}
+
+	return ret;
 }
 
 static
@@ -1110,7 +1230,8 @@ load_dylinker(
 	thread_t	thr_act,
 	int			depth,
 	load_result_t		*result,
-	boolean_t		clean_regions
+	boolean_t		clean_regions,
+	boolean_t		is_64bit
 )
 {
 	char			*name;
@@ -1144,7 +1265,8 @@ load_dylinker(
 	 *	Load the Mach-O.
 	 *	Use a temporary map to do the work.
 	 */
-	copy_map = vm_map_create(pmap_create(vm_map_round_page(macho_size)),
+	copy_map = vm_map_create(pmap_create(vm_map_round_page(macho_size),
+					     is_64bit),
 				 get_map_min(map), get_map_max(map), TRUE);
 	if (VM_MAP_NULL == copy_map) {
 		ret = LOAD_RESOURCE;
diff --git a/bsd/kern/mach_process.c b/bsd/kern/mach_process.c
index a79d9de65..6ecd242b1 100644
--- a/bsd/kern/mach_process.c
+++ b/bsd/kern/mach_process.c
@@ -87,7 +87,6 @@
 
 #include <kern/task.h>
 #include <kern/thread.h>
-#include <mach/machine/thread_status.h>
 
 
 /* Macros to clear/set/test flags. */
@@ -96,12 +95,8 @@
 #define	ISSET(t, f)	((t) & (f))
 
 extern thread_t	port_name_to_thread(mach_port_name_t port_name);
-extern kern_return_t thread_getstatus(thread_t thread, int flavor, thread_state_t tstate, mach_msg_type_number_t *count);
 extern thread_t get_firstthread(task_t);
 
-#if defined (ppc)
-extern kern_return_t thread_setstatus(thread_t thread, int flavor, thread_state_t tstate, mach_msg_type_number_t count);
-#endif
 
 /*
  * sys-trace system call.
@@ -117,15 +112,6 @@ ptrace(p, uap, retval)
 	task_t		task;
 	thread_t	th_act;
 	struct uthread 	*ut;
-	int		*locr0;
-#if defined(ppc)
-	struct ppc_thread_state64 statep;
-#elif	defined(i386)
-	struct i386_saved_state statep;
-#else
-#error architecture not supported
-#endif
-	unsigned long state_count;
 	int tr_sigexc = 0;
 
 	AUDIT_ARG(cmd, uap->req);
@@ -273,39 +259,16 @@ ptrace(p, uap, retval)
 		th_act = (thread_t)get_firstthread(task);
 		if (th_act == THREAD_NULL)
 			goto errorLabel;
-		ut = (uthread_t)get_bsdthread_info(th_act);
-		locr0 = ut->uu_ar0;
-#if defined(i386)
-		state_count = i386_NEW_THREAD_STATE_COUNT;
-		if (thread_getstatus(th_act, i386_NEW_THREAD_STATE, &statep, &state_count)  != KERN_SUCCESS) {
-			goto errorLabel;
-		}	
-#elif defined(ppc)
-		state_count = PPC_THREAD_STATE64_COUNT;
-		if (thread_getstatus(th_act, PPC_THREAD_STATE64, (thread_state_t)&statep, (mach_msg_type_number_t *)&state_count)  != KERN_SUCCESS) {
-			goto errorLabel;
-		}	
-#else
-#error architecture not supported
-#endif
+
 		if (uap->addr != (user_addr_t)1) {
-#if	defined(i386)
-			locr0[PC] = (int)uap->addr;
-#elif	defined(ppc)
+#if defined(ppc)
 #define ALIGNED(addr,size)	(((unsigned)(addr)&((size)-1))==0)
-		if (!ALIGNED((int)uap->addr, sizeof(int)))
-			return (ERESTART);
-
-		statep.srr0 = uap->addr;
-		state_count = PPC_THREAD_STATE64_COUNT;
-		if (thread_setstatus(th_act, PPC_THREAD_STATE64, (thread_state_t)&statep, state_count)  != KERN_SUCCESS) {
-			goto errorLabel;
-		}	
+		        if (!ALIGNED((int)uap->addr, sizeof(int)))
+			        return (ERESTART);
 #undef 	ALIGNED
-#else
-#error architecture not implemented!
 #endif
-		} /* uap->addr != (user_addr_t)1 */
+		        thread_setentrypoint(th_act, uap->addr);
+		}
 
 		if ((unsigned)uap->data >= NSIG)
 			goto errorLabel;
@@ -313,37 +276,18 @@ ptrace(p, uap, retval)
 		if (uap->data != 0) {
 			psignal_lock(t, uap->data, 0);
                 }
-#if defined(ppc)
-		state_count = PPC_THREAD_STATE64_COUNT;
-		if (thread_getstatus(th_act, PPC_THREAD_STATE64, (thread_state_t)&statep, (mach_msg_type_number_t *)&state_count)  != KERN_SUCCESS) {
-			goto errorLabel;
-		}	
-#endif
-
-#define MSR_SE_BIT	21
 
 		if (uap->req == PT_STEP) {
-#if	defined(i386)
-			locr0[PS] |= PSL_T;
-#elif 	defined(ppc)
-			statep.srr1 |= MASK(MSR_SE);
-#else
-#error architecture not implemented!
-#endif
-		} /* uap->req == PT_STEP */
-		else {  /* PT_CONTINUE - clear trace bit if set */
-#if defined(i386)
-			locr0[PS] &= ~PSL_T;
-#elif defined(ppc)
-			statep.srr1 &= ~MASK(MSR_SE);
-#endif
+		        /*
+			 * set trace bit
+			 */
+		        thread_setsinglestep(th_act, 1);
+		} else {
+		        /*
+			 * clear trace bit if on
+			 */
+		        thread_setsinglestep(th_act, 0);
 		}
-#if defined (ppc)
-		state_count = PPC_THREAD_STATE64_COUNT;
-		if (thread_setstatus(th_act, PPC_THREAD_STATE64, (thread_state_t)&statep, state_count)  != KERN_SUCCESS) {
-			goto errorLabel;
-		}	
-#endif
 	resume:
 		t->p_xstat = uap->data;
 		t->p_stat = SRUN;
diff --git a/bsd/kern/makesyscalls.sh b/bsd/kern/makesyscalls.sh
index 185b117ed..57b680fb8 100755
--- a/bsd/kern/makesyscalls.sh
+++ b/bsd/kern/makesyscalls.sh
@@ -138,13 +138,8 @@ s/\$//g
 		printf "#include <mach/shared_memory_server.h>\n" > sysarg
 		printf "\n#ifdef KERNEL\n" > sysarg
 		printf "#ifdef __APPLE_API_PRIVATE\n" > sysarg
-		printf "#ifdef __ppc__\n" > sysarg
 		printf "#define\tPAD_(t)\t(sizeof(uint64_t) <= sizeof(t) \\\n " > sysarg
 		printf "\t\t? 0 : sizeof(uint64_t) - sizeof(t))\n" > sysarg
-		printf "#else\n" > sysarg
-		printf "#define\tPAD_(t)\t(sizeof(register_t) <= sizeof(t) \\\n " > sysarg
-		printf "\t\t? 0 : sizeof(register_t) - sizeof(t))\n" > sysarg
-		printf "#endif\n" > sysarg
 		printf "#if BYTE_ORDER == LITTLE_ENDIAN\n"> sysarg
 		printf "#define\tPADL_(t)\t0\n" > sysarg
 		printf "#define\tPADR_(t)\tPAD_(t)\n" > sysarg
@@ -155,7 +150,6 @@ s/\$//g
 		printf "\n__BEGIN_DECLS\n" > sysarg
 		printf "#ifndef __MUNGE_ONCE\n" > sysarg
 		printf "#define __MUNGE_ONCE\n" > sysarg
-		printf "#ifdef __ppc__\n" > sysarg
 		printf "void munge_w(const void *, void *);  \n" > sysarg
 		printf "void munge_ww(const void *, void *);  \n" > sysarg
 		printf "void munge_www(const void *, void *);  \n" > sysarg
@@ -164,31 +158,25 @@ s/\$//g
 		printf "void munge_wwwwww(const void *, void *);  \n" > sysarg
 		printf "void munge_wwwwwww(const void *, void *);  \n" > sysarg
 		printf "void munge_wwwwwwww(const void *, void *);  \n" > sysarg
-		printf "void munge_d(const void *, void *);  \n" > sysarg
-		printf "void munge_dd(const void *, void *);  \n" > sysarg
-		printf "void munge_ddd(const void *, void *);  \n" > sysarg
-		printf "void munge_dddd(const void *, void *);  \n" > sysarg
-		printf "void munge_ddddd(const void *, void *);  \n" > sysarg
-		printf "void munge_dddddd(const void *, void *);  \n" > sysarg
-		printf "void munge_ddddddd(const void *, void *);  \n" > sysarg
-		printf "void munge_dddddddd(const void *, void *);  \n" > sysarg
 		printf "void munge_wl(const void *, void *);  \n" > sysarg
 		printf "void munge_wlw(const void *, void *);  \n" > sysarg
 		printf "void munge_wwwl(const void *, void *);  \n" > sysarg
+		printf "void munge_wwwlww(const void *, void *); \n" > sysarg
 		printf "void munge_wwwwl(const void *, void *);  \n" > sysarg
 		printf "void munge_wwwwwl(const void *, void *);  \n" > sysarg
 		printf "void munge_wsw(const void *, void *);  \n" > sysarg
 		printf "void munge_wws(const void *, void *);  \n" > sysarg
 		printf "void munge_wwwsw(const void *, void *);  \n" > sysarg
+		printf "#ifdef __ppc__\n" > sysarg
+		printf "void munge_d(const void *, void *);  \n" > sysarg
+		printf "void munge_dd(const void *, void *);  \n" > sysarg
+		printf "void munge_ddd(const void *, void *);  \n" > sysarg
+		printf "void munge_dddd(const void *, void *);  \n" > sysarg
+		printf "void munge_ddddd(const void *, void *);  \n" > sysarg
+		printf "void munge_dddddd(const void *, void *);  \n" > sysarg
+		printf "void munge_ddddddd(const void *, void *);  \n" > sysarg
+		printf "void munge_dddddddd(const void *, void *);  \n" > sysarg
 		printf "#else \n" > sysarg
-		printf "#define munge_w  NULL \n" > sysarg
-		printf "#define munge_ww  NULL \n" > sysarg
-		printf "#define munge_www  NULL \n" > sysarg
-		printf "#define munge_wwww  NULL \n" > sysarg
-		printf "#define munge_wwwww  NULL \n" > sysarg
-		printf "#define munge_wwwwww  NULL \n" > sysarg
-		printf "#define munge_wwwwwww  NULL \n" > sysarg
-		printf "#define munge_wwwwwwww  NULL \n" > sysarg
 		printf "#define munge_d  NULL \n" > sysarg
 		printf "#define munge_dd  NULL \n" > sysarg
 		printf "#define munge_ddd  NULL \n" > sysarg
@@ -197,14 +185,6 @@ s/\$//g
 		printf "#define munge_dddddd  NULL \n" > sysarg
 		printf "#define munge_ddddddd  NULL \n" > sysarg
 		printf "#define munge_dddddddd  NULL \n" > sysarg
-		printf "#define munge_wl  NULL \n" > sysarg
-		printf "#define munge_wlw  NULL \n" > sysarg
-		printf "#define munge_wwwl  NULL \n" > sysarg
-		printf "#define munge_wwwwl  NULL \n" > sysarg
-		printf "#define munge_wwwwwl  NULL \n" > sysarg
-		printf "#define munge_wsw  NULL \n" > sysarg
-		printf "#define munge_wws  NULL \n" > sysarg
-		printf "#define munge_wwwsw  NULL \n" > sysarg
 		printf "#endif // __ppc__\n" > sysarg
 		printf "#endif /* !__MUNGE_ONCE */\n" > sysarg
 		
@@ -663,11 +643,7 @@ s/\$//g
 	}
 
 	END {
-		printf "#ifdef __ppc__\n" > sysinc
 		printf "#define AC(name) (sizeof(struct name) / sizeof(uint64_t))\n" > sysinc
-		printf "#else\n" > sysinc
-		printf "#define AC(name) (sizeof(struct name) / sizeof(register_t))\n" > sysinc
-		printf "#endif\n" > sysinc
 		printf "\n" > sysinc
 
 		printf("\n__END_DECLS\n") > sysprotoend
diff --git a/bsd/kern/md5c.c b/bsd/kern/md5c.c
index 80fce2d7a..616779655 100644
--- a/bsd/kern/md5c.c
+++ b/bsd/kern/md5c.c
@@ -22,7 +22,7 @@
  * These notices must be retained in any copies of any part of this
  * documentation and/or software.
  *
- * $Id: md5c.c,v 1.2 2000/09/14 20:34:44 lindak Exp $
+ * $Id: md5c.c,v 1.2.4880.1 2005/06/24 01:47:07 lindak Exp $
  *
  * This code is the same as the code published by RSA Inc.  It has been
  * edited for clarity and style only.
diff --git a/bsd/kern/netboot.c b/bsd/kern/netboot.c
index 97ca556cf..cd594de26 100644
--- a/bsd/kern/netboot.c
+++ b/bsd/kern/netboot.c
@@ -813,7 +813,7 @@ netboot_setup(struct proc * p)
 	struct vfs_context context;
 
 	context.vc_proc = p;
-	context.vc_ucred = proc_ucred(p);	/* XXX kauth_cred_get() ??? proxy */
+	context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? proxy */
 
 	/* Get the vnode for '/'.  Set fdp->fd_fd.fd_cdir to reference it. */
 	if (VFS_ROOT(mountlist.tqh_last, &newdp, &context))
@@ -828,6 +828,7 @@ netboot_setup(struct proc * p)
 	TAILQ_REMOVE(&mountlist, TAILQ_FIRST(&mountlist), mnt_list);
 	mount_list_unlock();
 	mountlist.tqh_first->mnt_flag |= MNT_ROOTFS;
+	kauth_cred_unref(&context.vc_ucred);
     }
  done:
     netboot_info_free(&S_netboot_info_p);
diff --git a/bsd/kern/posix_sem.c b/bsd/kern/posix_sem.c
index 1dcc200d2..1c00fa91a 100644
--- a/bsd/kern/posix_sem.c
+++ b/bsd/kern/posix_sem.c
@@ -58,6 +58,7 @@
 #include <sys/malloc.h>
 #include <sys/semaphore.h>
 #include <sys/sysproto.h>
+#include <sys/proc_info.h>
 
 #include <bsm/audit_kernel.h>
 
@@ -187,12 +188,10 @@ psem_lock_init( void )
 {
 
     psx_sem_subsys_lck_grp_attr = lck_grp_attr_alloc_init();
-    lck_grp_attr_setstat(psx_sem_subsys_lck_grp_attr);
 
     psx_sem_subsys_lck_grp = lck_grp_alloc_init("posix shared memory", psx_sem_subsys_lck_grp_attr);
 
     psx_sem_subsys_lck_attr = lck_attr_alloc_init();
-    /* lck_attr_setdebug(psx_sem_subsys_lck_attr); */
     lck_mtx_init(& psx_sem_subsys_mutex, psx_sem_subsys_lck_grp, psx_sem_subsys_lck_attr);
 }
 
@@ -475,6 +474,8 @@ sem_open(struct proc *p, struct sem_open_args *uap, user_addr_t *retval)
 		pinfo->psem_mode = cmode;
 		pinfo->psem_uid = kauth_cred_getuid(kauth_cred_get());
 		pinfo->psem_gid = kauth_cred_get()->cr_gid;
+		bcopy(pnbuf, &pinfo->psem_name[0], PSEMNAMLEN);
+		pinfo->psem_name[PSEMNAMLEN]= 0;
 		PSEM_SUBSYS_UNLOCK();
    		kret = semaphore_create(kernel_task, &pinfo->psem_semobject,
                             SYNC_POLICY_FIFO, value);
@@ -1033,3 +1034,35 @@ psem_kqfilter(__unused struct fileproc *fp, __unused struct knote *kn,
 	return (ENOTSUP);
 }
 
+int
+fill_pseminfo(struct psemnode *pnode, struct psem_info * info)
+{
+	register struct pseminfo *pinfo;
+	struct stat *sb;
+
+	PSEM_SUBSYS_LOCK();
+	if ((pinfo = pnode->pinfo) == PSEMINFO_NULL){
+		PSEM_SUBSYS_UNLOCK();
+		return(EINVAL);
+	}
+
+#if 0
+	if ((pinfo->psem_flags & PSEM_ALLOCATED) != PSEM_ALLOCATED) {
+		PSEM_SUBSYS_UNLOCK();
+		return(EINVAL);
+	}
+#endif
+
+	sb = &info->psem_stat;
+	bzero(sb, sizeof(struct stat));
+
+    	sb->st_mode = pinfo->psem_mode;
+    	sb->st_uid = pinfo->psem_uid;
+    	sb->st_gid = pinfo->psem_gid;
+    	sb->st_size = pinfo->psem_usecount;
+	bcopy(&pinfo->psem_name[0], &info->psem_name[0], PSEMNAMLEN+1);
+
+	PSEM_SUBSYS_UNLOCK();
+	return(0);
+}
+
diff --git a/bsd/kern/posix_shm.c b/bsd/kern/posix_shm.c
index f413dab1b..f5f71f022 100644
--- a/bsd/kern/posix_shm.c
+++ b/bsd/kern/posix_shm.c
@@ -59,6 +59,7 @@
 #include <sys/mman.h>
 #include <sys/stat.h>
 #include <sys/sysproto.h>
+#include <sys/proc_info.h>
 
 #include <bsm/audit_kernel.h>
 
@@ -87,6 +88,7 @@
 #define f_data f_fglob->fg_data
 #define	PSHMNAMLEN	31	/* maximum name segment length we bother with */
 
+
 struct pshminfo {
 	unsigned int	pshm_flags;
 	unsigned int	pshm_usecount;
@@ -196,12 +198,10 @@ pshm_lock_init( void )
 {
 
     psx_shm_subsys_lck_grp_attr = lck_grp_attr_alloc_init();
-    lck_grp_attr_setstat(psx_shm_subsys_lck_grp_attr);
 
     psx_shm_subsys_lck_grp = lck_grp_alloc_init("posix shared memory", psx_shm_subsys_lck_grp_attr);
 
     psx_shm_subsys_lck_attr = lck_attr_alloc_init();
-    /* lck_attr_setdebug(psx_shm_subsys_lck_attr); */
     lck_mtx_init(& psx_shm_subsys_mutex, psx_shm_subsys_lck_grp, psx_shm_subsys_lck_attr);
 }
 
@@ -485,6 +485,8 @@ shm_open(struct proc *p, struct shm_open_args *uap, register_t *retval)
                     pinfo->pshm_mode = cmode;
                     pinfo->pshm_uid = kauth_cred_getuid(kauth_cred_get());
                     pinfo->pshm_gid = kauth_cred_get()->cr_gid;
+			bcopy(pnbuf, &pinfo->pshm_name[0], PSHMNAMLEN);
+			pinfo->pshm_name[PSHMNAMLEN]=0;
                 } else {
                     /*  already exists */
                         if( pinfo->pshm_flags & PSHM_INDELETE) {
@@ -593,9 +595,9 @@ pshm_truncate(__unused struct proc *p, struct fileproc *fp, __unused int fd,
 	struct pshminfo * pinfo;
 	struct pshmnode * pnode ;
 	kern_return_t kret;
-	vm_offset_t user_addr;
+	mach_vm_offset_t user_addr;
 	mem_entry_name_port_t mem_object;
-	vm_size_t size;
+	mach_vm_size_t size;
 
 	if (fp->f_type != DTYPE_PSXSHM) {
 		return(EINVAL);
@@ -618,17 +620,17 @@ pshm_truncate(__unused struct proc *p, struct fileproc *fp, __unused int fd,
 
 	PSHM_SUBSYS_UNLOCK();
 	size = round_page_64(length);
-	kret = vm_allocate(current_map(), &user_addr, size, VM_FLAGS_ANYWHERE);
+	kret = mach_vm_allocate(current_map(), &user_addr, size, VM_FLAGS_ANYWHERE);
 	if (kret != KERN_SUCCESS) 
 		goto out;
 
-	kret = mach_make_memory_entry (current_map(), &size,
+	kret = mach_make_memory_entry_64 (current_map(), &size,
 			user_addr, VM_PROT_DEFAULT, &mem_object, 0);
 
 	if (kret != KERN_SUCCESS) 
 		goto out;
 	
-	vm_deallocate(current_map(), user_addr, size);
+	mach_vm_deallocate(current_map(), user_addr, size);
 
 	PSHM_SUBSYS_LOCK();
 	pinfo->pshm_flags &= ~PSHM_DEFINED;
@@ -1022,3 +1024,32 @@ pshm_kqfilter(__unused struct fileproc *fp, __unused struct knote *kn,
 {
 	return(ENOTSUP);
 }
+
+int
+fill_pshminfo(struct pshmnode * pshm, struct pshm_info * info)
+{
+	struct pshminfo *pinfo;
+	struct stat *sb;
+	
+	PSHM_SUBSYS_LOCK();
+	if ((pinfo = pshm->pinfo) == PSHMINFO_NULL){
+		PSHM_SUBSYS_UNLOCK();
+		return(EINVAL);
+	}
+
+	sb = &info->pshm_stat;
+
+	bzero(sb, sizeof(struct stat)); 
+	sb->st_mode = pinfo->pshm_mode;
+	sb->st_uid = pinfo->pshm_uid;
+	sb->st_gid = pinfo->pshm_gid;
+	sb->st_size = pinfo->pshm_length;
+
+	info->pshm_mappaddr = pshm->mapp_addr;
+	bcopy(&pinfo->pshm_name[0], &info->pshm_name[0], PSHMNAMLEN+1); 
+
+	PSHM_SUBSYS_UNLOCK();
+	return(0);
+}
+
+
diff --git a/bsd/kern/proc_info.c b/bsd/kern/proc_info.c
new file mode 100644
index 000000000..5d9b31089
--- /dev/null
+++ b/bsd/kern/proc_info.c
@@ -0,0 +1,1064 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * sysctl system call.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/proc_internal.h>
+#include <sys/kauth.h>
+#include <sys/file_internal.h>
+#include <sys/vnode_internal.h>
+#include <sys/unistd.h>
+#include <sys/buf.h>
+#include <sys/ioctl.h>
+#include <sys/namei.h>
+#include <sys/tty.h>
+#include <sys/disklabel.h>
+#include <sys/vm.h>
+#include <sys/sysctl.h>
+#include <sys/user.h>
+#include <sys/aio_kern.h>
+
+#include <bsm/audit_kernel.h>
+
+#include <mach/machine.h>
+#include <mach/mach_types.h>
+#include <mach/vm_param.h>
+#include <kern/task.h>
+#include <kern/lock.h>
+#include <kern/kalloc.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_map.h>
+#include <mach/host_info.h>
+#include <mach/task_info.h>
+#include <mach/thread_info.h>
+#include <mach/vm_region.h>
+
+#include <sys/mount_internal.h>
+#include <sys/proc_info.h>
+#include <sys/bsdtask_info.h>
+#include <sys/kdebug.h>
+#include <sys/sysproto.h>
+#include <sys/msgbuf.h>
+
+
+#include <machine/machine_routines.h>
+
+#include <vm/vm_protos.h>
+
+struct pshmnode;
+struct psemnode;
+struct pipe;
+struct kqueue;
+struct atalk;
+
+int proc_info_internal(int callnum, int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t buffersize, register_t * retval);
+
+/* protos for proc_info calls */
+int proc_listpids(uint32_t type, uint32_t tyoneinfo, user_addr_t buffer, uint32_t buffersize, register_t * retval);
+int proc_pidinfo(int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t buffersize, register_t * retval);
+int proc_pidfdinfo(int pid, int flavor,int fd, user_addr_t buffer, uint32_t buffersize, register_t * retval);
+int proc_kernmsgbuf(user_addr_t buffer, uint32_t buffersize, register_t * retval);
+
+/* protos for procpidinfo calls */
+int proc_pidfdlist(proc_t p, user_addr_t buffer, uint32_t buffersize, register_t *retval);
+int proc_pidbsdinfo(proc_t p, struct proc_bsdinfo *pbsd);
+int proc_pidtaskinfo(proc_t p, struct proc_taskinfo *ptinfo);
+int proc_pidallinfo(proc_t p, int flavor, uint64_t arg, user_addr_t buffer, uint32_t buffersize, register_t *retval);
+int proc_pidthreadinfo(proc_t p, uint64_t arg,  struct proc_threadinfo *pthinfo);
+int proc_pidlistthreads(proc_t p,  user_addr_t buffer, uint32_t buffersize, register_t *retval);
+int proc_pidregioninfo(proc_t p, uint64_t arg, user_addr_t buffer, uint32_t buffersize, register_t *retval);
+int proc_pidregionpathinfo(proc_t p,  uint64_t arg, user_addr_t buffer, uint32_t buffersize, register_t *retval);
+int proc_pidvnodepathinfo(proc_t p,  uint64_t arg, user_addr_t buffer, uint32_t buffersize, register_t *retval);
+
+
+/* protos for proc_pidfdinfo calls */
+int pid_vnodeinfo(vnode_t vp, uint32_t vid, struct fileproc * fp, user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+int pid_vnodeinfopath(vnode_t vp, uint32_t vid, struct fileproc * fp, user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+int pid_socketinfo(socket_t  so, struct fileproc *fp, user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+int pid_pseminfo(struct psemnode * psem, struct fileproc * fp,  user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+int pid_pshminfo(struct pshmnode * pshm, struct fileproc * fp,  user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+int pid_pipeinfo(struct pipe * p, struct fileproc * fp,  user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+int pid_kqueueinfo(struct kqueue * kq, struct fileproc * fp,  user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+int pid_atalkinfo(struct atalk  * at, struct fileproc * fp,  user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+
+
+/* protos for misc */
+
+int fill_vnodeinfo(vnode_t vp, struct vnode_info *vinfo);
+void  fill_fileinfo(struct fileproc * fp, struct proc_fileinfo * finfo);
+static int proc_security_policy(proc_t p);
+
+
+/***************************** proc_info ********************/
+
+int
+proc_info(__unused struct proc *p, struct proc_info_args * uap, register_t *retval)
+{
+	return(proc_info_internal(uap->callnum, uap->pid, uap->flavor, uap->arg, uap->buffer, uap->buffersize, retval));
+}
+
+
+int 
+proc_info_internal(int callnum, int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t  buffersize, register_t * retval)
+{
+
+	switch(callnum) {
+		case 1: /* proc_listpids */
+			/* pid contains type and flavor contains typeinfo */
+			return(proc_listpids(pid, flavor, buffer, buffersize, retval));
+		case 2: /* proc_pidinfo */
+			return(proc_pidinfo(pid, flavor, arg, buffer, buffersize, retval));
+		case 3: /* proc_pidfdinfo */
+			return(proc_pidfdinfo(pid, flavor, (int)arg, buffer, buffersize, retval));
+		case 4: /* proc_kernmsgbuf */
+			return(proc_kernmsgbuf( buffer, buffersize, retval));
+		default:
+				return(EINVAL);
+	}
+
+	return(EINVAL);
+}
+
+/******************* proc_listpids routine ****************/
+int
+proc_listpids(uint32_t type, uint32_t typeinfo, user_addr_t buffer, uint32_t  buffersize, register_t * retval)
+{
+	boolean_t funnel_state;
+	int numprocs, wantpids;
+	char * kbuf;
+	int * ptr;
+	int n, skip;
+	struct proc * p;
+	int error = 0;
+
+	/* if the buffer is null, return num of procs */
+	if (buffer == (user_addr_t)0) {
+		*retval = ((nprocs+20) * sizeof(int));
+		return(0);
+	}
+
+	if (buffersize < sizeof(int)) {
+		return(ENOMEM);
+	}
+	wantpids = buffersize/sizeof(int);
+	numprocs = nprocs+20;
+	if (numprocs > wantpids)
+		numprocs = wantpids;
+
+	kbuf = (char *)kalloc((vm_size_t)(numprocs * sizeof(int)));
+	bzero(kbuf, sizeof(int));
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+	
+	n = 0;
+	ptr = (int *)kbuf;
+	LIST_FOREACH(p, &allproc, p_list) {
+		skip = 0;
+		switch (type) {
+			case PROC_PGRP_ONLY:
+				if (p->p_pgrp->pg_id != (pid_t)typeinfo)
+					skip = 1;
+			  	break;
+			case PROC_ALL_PIDS:
+				skip = 0;
+			  	break;
+			case PROC_TTY_ONLY:
+				if ((p->p_flag & P_CONTROLT) == 0 ||
+					(p->p_session == NULL) ||
+			    	p->p_session->s_ttyp == NULL ||
+			    	p->p_session->s_ttyp->t_dev != (dev_t)typeinfo)
+					skip = 1;
+			  	break;
+			case PROC_UID_ONLY:
+				if ((p->p_ucred == NULL) ||
+					(kauth_cred_getuid(p->p_ucred) != (uid_t)typeinfo))
+					skip = 1;
+			  	break;
+			case PROC_RUID_ONLY:
+				if ((p->p_ucred == NULL) ||
+					(p->p_ucred->cr_ruid != (uid_t)typeinfo))
+			  	break;
+			default:
+			  skip = 1;
+			  break;
+		};
+
+		/* Do we have permission to look into this ? */
+		if (proc_security_policy(p) != 0) {
+			skip = 1;
+		}
+
+		if(skip == 0) {
+			*ptr++ = p->p_pid;
+			n++;
+		}
+		if (n >= numprocs)
+			break;
+	}
+	
+	if (n < numprocs) {
+		LIST_FOREACH(p, &zombproc, p_list) {
+			*ptr++ = p->p_pid;
+			n++;
+			if (n >= numprocs)
+				break;
+		}
+	}
+	
+	thread_funnel_set(kernel_flock, funnel_state);
+
+	ptr = (int *)kbuf;
+	error = copyout((caddr_t)ptr, buffer, n * sizeof(int));
+	if (error == 0)
+		*retval = (n * sizeof(int));
+	kfree((void *)kbuf, (vm_size_t)(numprocs * sizeof(int)));
+
+	return(error);
+}
+
+
+/********************************** proc_pidinfo routines ********************************/
+
+int 
+proc_pidfdlist(proc_t p, user_addr_t buffer, uint32_t  buffersize, register_t *retval)
+{
+		int numfds, needfds;
+		char * kbuf;
+		struct proc_fdinfo * pfd;
+		struct fileproc * fp;
+		int n;
+		int count = 0;
+		int error = 0;
+		
+	 	numfds = p->p_fd->fd_nfiles;	
+
+		if (buffer == (user_addr_t) 0) {
+			numfds += 20;
+			*retval = (numfds * sizeof(struct proc_fdinfo));
+			return(0);
+		}
+
+		/* buffersize is big enough atleast for one struct */
+		needfds = buffersize/sizeof(struct proc_fdinfo);
+
+		if (numfds > needfds)
+			numfds = needfds;
+
+		kbuf = (char *)kalloc((vm_size_t)(numfds * sizeof(struct proc_fdinfo)));
+		bzero(kbuf, numfds * sizeof(struct proc_fdinfo));
+
+		proc_fdlock(p);
+
+		pfd = (struct proc_fdinfo *)kbuf;
+
+		for (n = 0; ((n < numfds) && (n < p->p_fd->fd_nfiles)); n++) {
+			if (((fp = p->p_fd->fd_ofiles[n]) != 0) 
+			     && ((p->p_fd->fd_ofileflags[n] & UF_RESERVED) == 0)) {
+				pfd->proc_fd = n;
+				pfd->proc_fdtype = fp->f_fglob->fg_type;	
+				count++;
+				pfd++;
+			}
+		}
+		proc_fdunlock(p);
+
+		error = copyout(kbuf, buffer, count * sizeof(struct proc_fdinfo));
+		kfree((void *)kbuf, (vm_size_t)(numfds * sizeof(struct proc_fdinfo)));
+		if (error == 0)
+			*retval = (count * sizeof(struct proc_fdinfo));
+		return(error);		
+}
+
+
+int 
+proc_pidbsdinfo(proc_t p, struct proc_bsdinfo * pbsd)
+{
+	register struct tty *tp;
+	struct  session *sessionp = NULL;
+
+	bzero(pbsd, sizeof(struct proc_bsdinfo));
+	pbsd->pbi_status = p->p_stat;
+	pbsd->pbi_xstatus = p->p_xstat;
+	pbsd->pbi_pid = p->p_pid;
+	pbsd->pbi_ppid = p->p_pptr->p_pid;
+	pbsd->pbi_uid = p->p_ucred->cr_uid;
+	pbsd->pbi_gid = p->p_ucred->cr_uid; /* XXX */
+	pbsd->pbi_ruid =  p->p_ucred->cr_ruid;
+	pbsd->pbi_rgid = p->p_ucred->cr_rgid;
+	pbsd->pbi_svuid =  p->p_ucred->cr_svuid;
+	pbsd->pbi_svgid = p->p_ucred->cr_svgid;
+	
+	pbsd->pbi_nice = p->p_nice;
+	if (p->p_stats)
+		pbsd->pbi_start = p->p_stats->p_start;
+	bcopy(&p->p_comm, &pbsd->pbi_comm[0], MAXCOMLEN);
+	bcopy(&p->p_name, &pbsd->pbi_name[0], 2* MAXCOMLEN);
+
+	pbsd->pbi_flags = 0;	
+	if ((p->p_flag & P_SYSTEM) == P_SYSTEM) 
+		pbsd->pbi_flags |= PROC_FLAG_SYSTEM;
+	if ((p->p_flag & P_TRACED) == P_TRACED) 
+		pbsd->pbi_flags |= PROC_FLAG_TRACED;
+	if ((p->p_flag & P_WEXIT) == P_WEXIT) 
+		pbsd->pbi_flags |= PROC_FLAG_INEXIT;
+	if ((p->p_flag & P_PPWAIT) == P_PPWAIT) 
+		pbsd->pbi_flags |= PROC_FLAG_PPWAIT;
+	if ((p->p_flag & P_LP64) == P_LP64) 
+		pbsd->pbi_flags |= PROC_FLAG_LP64;
+	if ((p->p_flag & P_CONTROLT) == P_CONTROLT) 
+		pbsd->pbi_flags |= PROC_FLAG_CONTROLT;
+
+	if (SESS_LEADER(p))
+		pbsd->pbi_flags |= PROC_FLAG_SLEADER;
+	if (p->p_pgrp->pg_session && p->p_pgrp->pg_session->s_ttyvp)
+		pbsd->pbi_flags |= PROC_FLAG_CTTY;
+		
+	pbsd->pbi_nfiles = p->p_fd->fd_nfiles;
+	if (p->p_pgrp) {
+		sessionp = p->p_pgrp->pg_session;
+		pbsd->pbi_pgid = p->p_pgrp->pg_id;
+		pbsd->pbi_pjobc = p->p_pgrp->pg_jobc;
+		if ((p->p_flag & P_CONTROLT) && (sessionp) && (tp = sessionp->s_ttyp)) {
+			pbsd->e_tdev = tp->t_dev;
+			pbsd->e_tpgid = tp->t_pgrp ? tp->t_pgrp->pg_id : 0;
+		}
+	} 
+	return(0);
+}
+
+
+int 
+proc_pidtaskinfo(proc_t p, struct proc_taskinfo * ptinfo)
+{
+	task_t task;
+	
+	task = p->task;
+
+	bzero(ptinfo, sizeof(struct proc_taskinfo));
+	fill_taskprocinfo(task, (struct proc_taskinfo_internal *)ptinfo);
+
+	return(0);
+}
+
+
+
+int 
+proc_pidthreadinfo(proc_t p, uint64_t arg,  struct proc_threadinfo *pthinfo)
+{
+	int error = 0;
+	uint64_t threadaddr = (uint64_t)arg;
+
+	bzero(pthinfo, sizeof(struct proc_threadinfo));
+
+	error = fill_taskthreadinfo(p->task, threadaddr, (struct proc_threadinfo_internal *)pthinfo);
+	if (error)
+		return(ESRCH);
+	else
+		return(0);
+
+}
+
+
+int 
+proc_pidlistthreads(proc_t p,  user_addr_t buffer, uint32_t  buffersize, register_t *retval)
+{
+	int count = 0;	
+	int ret = 0;
+	int error = 0;
+	void * kbuf;
+	int numthreads;
+
+	
+	count = buffersize/(sizeof(uint64_t));
+	numthreads = get_numthreads(p->task);
+
+	numthreads += 10;
+
+	if (numthreads > count)
+		numthreads = count;
+
+	kbuf = (void *)kalloc(numthreads * sizeof(uint64_t));
+	bzero(kbuf, numthreads * sizeof(uint64_t));
+	
+	ret = fill_taskthreadlist(p->task, kbuf, numthreads);
+	
+	error = copyout(kbuf, buffer, ret);
+	kfree(kbuf, numthreads * sizeof(uint64_t));
+	if (error == 0)
+		*retval = ret;
+	return(error);
+	
+}
+
+
+int 
+proc_pidregioninfo(proc_t p, uint64_t arg, user_addr_t buffer, __unused uint32_t  buffersize, register_t *retval)
+{
+	struct proc_regioninfo preginfo;
+	int ret, error = 0;
+
+	bzero(&preginfo, sizeof(struct proc_regioninfo));
+	ret = fill_procregioninfo( p->task, arg, (struct proc_regioninfo_internal *)&preginfo, (uint32_t *)0, (uint32_t *)0);
+	if (ret == 0)
+		return(EINVAL);
+	error = copyout(&preginfo, buffer, sizeof(struct proc_regioninfo));
+	if (error == 0)
+		*retval = sizeof(struct proc_regioninfo);
+	return(error);
+}
+
+
+int 
+proc_pidregionpathinfo(proc_t p, uint64_t arg, user_addr_t buffer, __unused uint32_t  buffersize, register_t *retval)
+{
+	struct proc_regionwithpathinfo preginfo;
+	int ret, error = 0;
+	uint32_t vnodeaddr= 0;
+	uint32_t vnodeid= 0;
+	vnode_t vp;
+	int count;
+
+	bzero(&preginfo, sizeof(struct proc_regionwithpathinfo));
+
+	ret = fill_procregioninfo( p->task, arg, (struct proc_regioninfo_internal *)&preginfo.prp_prinfo, (uint32_t *)&vnodeaddr, (uint32_t *)&vnodeid);
+	if (ret == 0)
+		return(EINVAL);
+	if (vnodeaddr) {
+		vp = (vnode_t)vnodeaddr;
+		if ((vnode_getwithvid(vp, vnodeid)) == 0) {
+			/* FILL THE VNODEINFO */
+			error = fill_vnodeinfo(vp, &preginfo.prp_vip.vip_vi);
+			count = MAXPATHLEN;
+			vn_getpath(vp, &preginfo.prp_vip.vip_path[0], &count);
+			/* Always make sure it is null terminated */
+			preginfo.prp_vip.vip_path[MAXPATHLEN-1] = 0;
+			vnode_put(vp);
+		}
+	}
+	error = copyout(&preginfo, buffer, sizeof(struct proc_regionwithpathinfo));
+	if (error == 0)
+		*retval = sizeof(struct proc_regionwithpathinfo);
+	return(error);
+}
+
+int 
+proc_pidvnodepathinfo(proc_t p, __unused uint64_t arg, user_addr_t buffer, __unused uint32_t  buffersize, register_t *retval)
+{
+	struct proc_vnodepathinfo pvninfo;
+	int error = 0;
+	vnode_t vncdirvp = NULLVP;
+	uint32_t vncdirid=0;
+	vnode_t vnrdirvp = NULLVP;
+	uint32_t vnrdirid=0;
+	int count;
+
+	bzero(&pvninfo, sizeof(struct proc_vnodepathinfo));
+
+	proc_fdlock(p);
+	if (p->p_fd->fd_cdir) {
+		vncdirvp = p->p_fd->fd_cdir;
+		vncdirid = p->p_fd->fd_cdir->v_id;
+	}
+	if (p->p_fd->fd_rdir) {
+		vnrdirvp = p->p_fd->fd_rdir;
+		vnrdirid = p->p_fd->fd_rdir->v_id;
+	}
+	proc_fdunlock(p);
+
+	if (vncdirvp != NULLVP) {
+		if ((error = vnode_getwithvid(vncdirvp, vncdirid)) == 0) {
+			/* FILL THE VNODEINFO */
+			error = fill_vnodeinfo(vncdirvp, &pvninfo.pvi_cdir.vip_vi);
+			if ( error == 0) {
+				count = MAXPATHLEN;
+				vn_getpath(vncdirvp, &pvninfo.pvi_cdir.vip_path[0], &count);
+				pvninfo.pvi_cdir.vip_path[MAXPATHLEN-1] = 0;
+			}	
+			vnode_put(vncdirvp);
+		} else {
+			goto out;
+		}
+	}
+
+	if ((error == 0) && (vnrdirvp != NULLVP)) {
+		if ((error = vnode_getwithvid(vnrdirvp, vnrdirid)) == 0) {
+			/* FILL THE VNODEINFO */
+			error = fill_vnodeinfo(vnrdirvp, &pvninfo.pvi_rdir.vip_vi);
+			if ( error == 0) {
+				count = MAXPATHLEN;
+				vn_getpath(vnrdirvp, &pvninfo.pvi_rdir.vip_path[0], &count);
+				pvninfo.pvi_rdir.vip_path[MAXPATHLEN-1] = 0;
+			}	
+			vnode_put(vnrdirvp);
+		} else {
+			goto out;
+		}
+	}
+	if (error == 0) {
+		error = copyout(&pvninfo, buffer, sizeof(struct proc_vnodepathinfo));
+		if (error == 0)
+			*retval = sizeof(struct proc_vnodepathinfo);
+	}
+out:
+	return(error);
+}
+
+/********************************** proc_pidinfo ********************************/
+
+
+int
+proc_pidinfo(int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t  buffersize, register_t * retval)
+{
+	struct proc * p = PROC_NULL;
+	int error = ENOTSUP;
+	int gotref = 0;
+	int findzomb = 0;
+	boolean_t funnel_state;
+	uint32_t size;
+
+	switch (flavor) {
+		case PROC_PIDLISTFDS:
+			size = PROC_PIDLISTFD_SIZE;
+			if (buffer == (user_addr_t)0)
+				size = 0;
+			break;
+		case PROC_PIDTBSDINFO:
+			size = PROC_PIDTBSDINFO_SIZE;
+			break;
+		case PROC_PIDTASKINFO:
+			size = PROC_PIDTASKINFO_SIZE;
+			break;
+		case PROC_PIDTASKALLINFO:
+			size = PROC_PIDTASKALLINFO_SIZE;
+			break;
+		case PROC_PIDTHREADINFO:
+			size = PROC_PIDTHREADINFO_SIZE;
+			break;
+		case PROC_PIDLISTTHREADS:
+			size = PROC_PIDLISTTHREADS_SIZE;
+			break;
+		case PROC_PIDREGIONINFO:
+			size = PROC_PIDREGIONINFO_SIZE;
+			break;
+		case PROC_PIDREGIONPATHINFO:
+			size = PROC_PIDREGIONPATHINFO_SIZE;
+			break;
+		case PROC_PIDVNODEPATHINFO:
+			size = PROC_PIDVNODEPATHINFO_SIZE;
+			break;
+		default:
+			return(EINVAL);
+	}
+
+	if (buffersize < size) 
+		return(ENOMEM);
+
+	if (flavor != PROC_PIDTBSDINFO) {
+		if ((p = proc_findref(pid)) == PROC_NULL) {
+				error = ESRCH;
+				goto out;
+			} else {
+				gotref = 1;
+
+				/* Do we have permission to look into this ? */
+				if ((error = proc_security_policy(p)) != 0) {
+					goto out;
+				}
+			}
+	}
+	switch (flavor) {
+		case PROC_PIDLISTFDS: {
+			error = proc_pidfdlist(p, buffer, buffersize, retval);
+		}
+		break;
+
+		case PROC_PIDTBSDINFO: {
+			struct proc_bsdinfo pbsd;
+
+			if (arg)
+				findzomb = 1;
+			funnel_state = thread_funnel_set(kernel_flock, TRUE);
+			p = pfind(pid);
+			if (p == PROC_NULL) {
+				if (findzomb) 
+					p = pzfind(pid);
+				if (p == NULL) {
+					error = ESRCH;
+					thread_funnel_set(kernel_flock, funnel_state);
+					goto out;	
+				}
+			}
+			/* Do we have permission to look into this ? */
+			if ((error = proc_security_policy(p)) != 0) {
+				thread_funnel_set(kernel_flock, funnel_state);
+				goto out;
+			}
+			error = proc_pidbsdinfo(p, &pbsd);
+			thread_funnel_set(kernel_flock, funnel_state);	
+			if (error == 0) {
+				error = copyout(&pbsd, buffer, sizeof(struct proc_bsdinfo));
+				if (error == 0)
+					*retval = sizeof(struct proc_bsdinfo);
+			}	
+		}
+		break;
+
+		case PROC_PIDTASKINFO: {
+			struct proc_taskinfo ptinfo;
+
+			error =  proc_pidtaskinfo(p, &ptinfo);
+			if (error == 0) {
+				error = copyout(&ptinfo, buffer, sizeof(struct proc_taskinfo));
+				if (error == 0)
+					*retval = sizeof(struct proc_taskinfo);
+			}	
+		}
+		break;
+
+		case PROC_PIDTASKALLINFO: {
+		struct proc_taskallinfo pall;
+
+			error = proc_pidbsdinfo(p, &pall.pbsd);
+			error =  proc_pidtaskinfo(p, &pall.ptinfo);
+			if (error == 0) {
+				error = copyout(&pall, buffer, sizeof(struct proc_taskallinfo));
+				if (error == 0)
+					*retval = sizeof(struct proc_taskallinfo);
+			}	
+		}
+		break;
+
+		case PROC_PIDTHREADINFO:{
+		struct proc_threadinfo pthinfo;
+
+			error  = proc_pidthreadinfo(p,  arg, &pthinfo);
+			if (error == 0) {
+				error = copyout(&pthinfo, buffer, sizeof(struct proc_threadinfo));
+				if (error == 0)
+					*retval = sizeof(struct proc_threadinfo);
+			}	
+		}
+		break;
+
+		case PROC_PIDLISTTHREADS:{
+			error =  proc_pidlistthreads(p,  buffer, buffersize, retval);
+		}
+		break;
+
+		case PROC_PIDREGIONINFO:{
+			error =  proc_pidregioninfo(p,  arg, buffer, buffersize, retval);
+		}
+		break;
+
+
+		case PROC_PIDREGIONPATHINFO:{
+			error =  proc_pidregionpathinfo(p, arg, buffer, buffersize, retval);
+		}
+		break;
+
+		case PROC_PIDVNODEPATHINFO:{
+			error =  proc_pidvnodepathinfo(p, arg, buffer, buffersize, retval);
+		}
+		break;
+
+		default:
+			error = ENOTSUP;
+	}
+	
+out:
+	if (gotref)
+		proc_dropref(p);
+	return(error);
+}
+
+
+int 
+pid_vnodeinfo(vnode_t vp, uint32_t vid, struct fileproc * fp, user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval) 
+{
+	struct vnode_fdinfo vfi;
+	int error= 0;
+
+	if ((error = vnode_getwithvid(vp, vid)) != 0) {
+		return(error);
+	}
+	bzero(&vfi, sizeof(struct vnode_fdinfo));
+	fill_fileinfo(fp, &vfi.pfi);
+	error = fill_vnodeinfo(vp, &vfi.pvi);
+	vnode_put(vp);
+	if (error == 0) {
+		error = copyout((caddr_t)&vfi, buffer, sizeof(struct vnode_fdinfo));
+		if (error == 0)
+			*retval = sizeof(struct vnode_fdinfo);
+	}
+	return(error);
+}
+
+int 
+pid_vnodeinfopath(vnode_t vp, uint32_t vid, struct fileproc * fp, user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval) 
+{
+	struct vnode_fdinfowithpath vfip;
+	int count, error= 0;
+
+	if ((error = vnode_getwithvid(vp, vid)) != 0) {
+		return(error);
+	}
+	bzero(&vfip, sizeof(struct vnode_fdinfowithpath));
+	fill_fileinfo(fp, &vfip.pfi);
+	error = fill_vnodeinfo(vp, &vfip.pvip.vip_vi) ;
+	if (error == 0) {
+		count = MAXPATHLEN;
+		vn_getpath(vp, &vfip.pvip.vip_path[0], &count);
+		vfip.pvip.vip_path[MAXPATHLEN-1] = 0;
+		vnode_put(vp);
+		error = copyout((caddr_t)&vfip, buffer, sizeof(struct vnode_fdinfowithpath));
+		if (error == 0)
+			*retval = sizeof(struct vnode_fdinfowithpath);
+	} else 
+		vnode_put(vp);
+	return(error);
+}
+
+void  
+fill_fileinfo(struct fileproc * fp, struct proc_fileinfo * fproc)
+{
+		fproc->fi_openflags = fp->f_fglob->fg_flag;
+		fproc->fi_status = fp->f_flags;
+		fproc->fi_offset = fp->f_fglob->fg_offset;
+		fproc->fi_type = fp->f_fglob->fg_type;
+}
+
+
+
+int
+fill_vnodeinfo(vnode_t vp, struct vnode_info *vinfo)
+{
+		vfs_context_t context;
+		struct stat * sb;
+		int error = 0;
+
+		sb = &vinfo->vi_stat;
+
+		context = vfs_context_create((vfs_context_t)0);
+		error = vn_stat(vp, sb, NULL, context);
+		(void)vfs_context_rele(context);
+
+		if (error != 0)
+			goto out;
+
+		if (vp->v_mount != dead_mountp) {
+			vinfo->vi_fsid = vp->v_mount->mnt_vfsstat.f_fsid;
+		} else {
+			vinfo->vi_fsid.val[0] = 0;
+			vinfo->vi_fsid.val[1] = 0;
+		}
+		vinfo->vi_type = vp->v_type;
+out:
+		return(error);
+}
+
+int
+pid_socketinfo(socket_t so, struct fileproc *fp, user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval)
+{
+	struct socket_fdinfo s;
+	int error = 0;
+
+	bzero(&s, sizeof(struct socket_fdinfo));
+	fill_fileinfo(fp, &s.pfi);
+	if ((error = fill_socketinfo(so, &s.psi)) == 0) {
+		if ((error = copyout(&s, buffer, sizeof(struct socket_fdinfo))) == 0)
+				*retval = sizeof(struct socket_fdinfo);
+	}
+	
+	return (error);
+}
+
+int
+pid_pseminfo(struct psemnode *psem, struct fileproc *fp,  user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval)
+{
+	struct psem_fdinfo pseminfo;
+	int error = 0;
+ 
+	bzero(&pseminfo, sizeof(struct psem_fdinfo));
+	fill_fileinfo(fp, &pseminfo.pfi);
+
+	if ((error = fill_pseminfo(psem, &pseminfo.pseminfo)) == 0) {
+		if ((error = copyout(&pseminfo, buffer, sizeof(struct psem_fdinfo))) == 0)
+				*retval = sizeof(struct psem_fdinfo);
+	}
+
+	return(error);
+}
+
+int
+pid_pshminfo(struct pshmnode *pshm, struct fileproc *fp,  user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval)
+{
+	struct pshm_fdinfo pshminfo;
+	int error = 0;
+ 
+	bzero(&pshminfo, sizeof(struct pshm_fdinfo));
+	fill_fileinfo(fp, &pshminfo.pfi);
+
+	if ((error = fill_pshminfo(pshm, &pshminfo.pshminfo)) == 0) {
+		if ((error = copyout(&pshminfo, buffer, sizeof(struct pshm_fdinfo))) == 0)
+				*retval = sizeof(struct pshm_fdinfo);
+	}
+
+	return(error);
+}
+
+int
+pid_pipeinfo(struct pipe *  p, struct fileproc *fp,  user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval)
+{
+	struct pipe_fdinfo pipeinfo;
+	int error = 0;
+
+	bzero(&pipeinfo, sizeof(struct pipe_fdinfo));
+	fill_fileinfo(fp, &pipeinfo.pfi);
+	if ((error = fill_pipeinfo(p, &pipeinfo.pipeinfo)) == 0) {
+		if ((error = copyout(&pipeinfo, buffer, sizeof(struct pipe_fdinfo))) == 0)
+				*retval = sizeof(struct pipe_fdinfo);
+	}
+
+	return(error);
+}
+
+int
+pid_kqueueinfo(struct kqueue * kq, struct fileproc *fp,  user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval)
+{
+	struct kqueue_fdinfo kqinfo;
+	int error = 0;
+	
+	bzero(&kqinfo, sizeof(struct kqueue_fdinfo));
+ 
+	fill_fileinfo(fp, &kqinfo.pfi);
+
+	if ((error = fill_kqueueinfo(kq, &kqinfo.kqueueinfo)) == 0) {
+		if ((error = copyout(&kqinfo, buffer, sizeof(struct kqueue_fdinfo))) == 0)
+				*retval = sizeof(struct kqueue_fdinfo);
+	}
+
+	return(error);
+}
+
+int
+pid_atalkinfo(__unused struct atalk * at, __unused struct fileproc *fp,  __unused user_addr_t  buffer, __unused uint32_t buffersize, __unused register_t * retval)
+{
+	return ENOTSUP;
+}
+
+
+
+/************************** proc_pidfdinfo routine ***************************/
+int
+proc_pidfdinfo(int pid, int flavor,  int fd, user_addr_t buffer, uint32_t buffersize, register_t * retval)
+{
+	proc_t p;
+	int error = ENOTSUP;
+	struct fileproc * fp;
+	uint32_t size;
+	
+
+	switch (flavor) {
+		case PROC_PIDFDVNODEINFO:
+			size = PROC_PIDFDVNODEINFO_SIZE;
+			break;
+		case PROC_PIDFDVNODEPATHINFO:
+			size = PROC_PIDFDVNODEPATHINFO_SIZE;
+			break;
+		case PROC_PIDFDSOCKETINFO:
+			size = PROC_PIDFDSOCKETINFO_SIZE;
+			break;
+		case PROC_PIDFDPSEMINFO:
+			size = PROC_PIDFDPSEMINFO_SIZE;
+			break;
+		case PROC_PIDFDPSHMINFO:
+			size = PROC_PIDFDPSHMINFO_SIZE;
+			break;
+		case PROC_PIDFDPIPEINFO:
+			size = PROC_PIDFDPIPEINFO_SIZE;
+			break;
+		case PROC_PIDFDKQUEUEINFO:
+			size = PROC_PIDFDKQUEUEINFO_SIZE;
+			break;
+		case PROC_PIDFDATALKINFO:
+			size = PROC_PIDFDATALKINFO_SIZE;
+			break;
+
+		default:
+			return(EINVAL);
+
+	}
+
+	if (buffersize < size)
+		return(ENOMEM);
+
+	if ((p = proc_findref(pid)) == PROC_NULL) {
+		error = ESRCH;
+		goto out;
+	}
+	/* Do we have permission to look into this ? */
+	if ((error = proc_security_policy(p)) != 0) {
+		goto out1;
+	}
+
+	switch (flavor) {
+		case PROC_PIDFDVNODEINFO: {
+			vnode_t vp;
+			uint32_t vid=0;
+
+			if ((error = fp_getfvpandvid(p, fd, &fp,  &vp, &vid)) !=0) {
+				goto out1;
+			}
+			error =  pid_vnodeinfo(vp, vid, fp, buffer, buffersize, retval);
+		}
+		break;
+
+		case PROC_PIDFDVNODEPATHINFO: {
+			vnode_t vp;
+			uint32_t vid=0;
+
+			if ((error = fp_getfvpandvid(p, fd, &fp,  &vp, &vid)) !=0) {
+				goto out1;
+			}
+
+			error =  pid_vnodeinfopath(vp, vid, fp, buffer, buffersize, retval);
+		}
+		break;
+
+		case PROC_PIDFDSOCKETINFO: {
+			socket_t so; 
+
+			if ((error = fp_getfsock(p, fd, &fp,  &so)) !=0) {
+				goto out1;
+			}
+			error =  pid_socketinfo(so, fp, buffer, buffersize, retval);
+		}
+		break;
+
+		case PROC_PIDFDPSEMINFO: {
+			struct psemnode * psem;
+
+			if ((error = fp_getfpsem(p, fd, &fp,  &psem)) !=0) {
+				goto out1;
+			}
+			error =  pid_pseminfo(psem, fp, buffer, buffersize, retval);
+		}
+		break;
+
+		case PROC_PIDFDPSHMINFO: {
+			struct pshmnode * pshm;
+
+			if ((error = fp_getfpshm(p, fd, &fp,  &pshm)) !=0) {
+				goto out1;
+			}
+			error =  pid_pshminfo(pshm, fp, buffer, buffersize, retval);
+		}
+		break;
+
+		case PROC_PIDFDPIPEINFO: {
+			struct pipe * cpipe;
+
+			if ((error = fp_getfpipe(p, fd, &fp,  &cpipe)) !=0) {
+				goto out1;
+			}
+			error =  pid_pipeinfo(cpipe, fp, buffer, buffersize, retval);
+		}
+		break;
+
+		case PROC_PIDFDKQUEUEINFO: {
+			struct kqueue * kq;
+
+			if ((error = fp_getfkq(p, fd, &fp,  &kq)) !=0) {
+				goto out1;
+			}
+			error =  pid_kqueueinfo(kq, fp, buffer, buffersize, retval);
+		}
+		break;
+
+		case PROC_PIDFDATALKINFO: {
+			struct atalk * at;
+
+			if ((error = fp_getfatalk(p, fd, &fp,  &at)) !=0) {
+				goto out1;
+			}
+
+			error =  pid_atalkinfo(at, fp, buffer, buffersize, retval);
+		}
+		break;
+
+		default: {
+			error = EINVAL;
+		}
+		break;
+
+	}
+
+	fp_drop(p, fd, fp , 0); 	
+out1 :
+	proc_dropref(p);
+out:
+	return(error);
+}
+
+
+static int
+proc_security_policy(proc_t p)
+{
+	if ((kauth_cred_getuid(p->p_ucred) != kauth_cred_getuid(kauth_cred_get())) 
+		&& suser(kauth_cred_get(), (u_short *)0)) {
+			return(EPERM);
+		}
+
+	return(0);
+}
+
+
+/* Temporary hack to get dmesg to work. In Leopard this will disappear */
+int 
+proc_kernmsgbuf(user_addr_t buffer, uint32_t buffersize, register_t * retval)
+{
+	int error;
+
+	if (buffersize  < sizeof(struct msgbuf))
+		return(ENOMEM);
+
+	if (suser(kauth_cred_get(), (u_short *)0) == 0) {
+		error = copyout(msgbufp, buffer, sizeof(struct msgbuf));
+		if (error ==  0)
+			*retval = sizeof(struct msgbuf);
+		return(error);
+	} else
+		return(EPERM);
+}
+
diff --git a/bsd/kern/socket_info.c b/bsd/kern/socket_info.c
new file mode 100644
index 000000000..01ab24d71
--- /dev/null
+++ b/bsd/kern/socket_info.c
@@ -0,0 +1,247 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <sys/types.h>
+#include <sys/kernel_types.h>
+#include <sys/errno.h>
+#include <sys/kernel.h>
+#include <sys/file_internal.h>
+#include <sys/stat.h>
+#include <sys/select.h>
+#include <sys/pipe.h>
+#include <sys/proc_info.h>
+#include <sys/domain.h>
+#include <sys/protosw.h>
+#include <sys/domain.h>
+#include <sys/socketvar.h>
+#include <sys/unpcb.h>
+#include <sys/sys_domain.h>
+#include <sys/kern_event.h>
+#include <net/ndrv_var.h>
+#include <netinet/in_pcb.h>
+#include <netinet/tcp_var.h>
+#include <string.h>
+
+static void fill_sockbuf_info(struct sockbuf *sb, struct sockbuf_info *sbi);
+static void fill_common_sockinfo(struct socket *so, struct socket_info *si);
+
+static void
+fill_sockbuf_info(struct sockbuf *sb, struct sockbuf_info *sbi)
+{
+	sbi->sbi_cc = sb->sb_cc;
+	sbi->sbi_hiwat = sb->sb_hiwat;
+	sbi->sbi_mbcnt = sb->sb_mbcnt;
+	sbi->sbi_mbmax = sb->sb_mbmax;
+	sbi->sbi_lowat = sb->sb_lowat;
+	sbi->sbi_flags = sb->sb_flags;
+	sbi->sbi_timeo = (u_long)(sb->sb_timeo.tv_sec * hz) + sb->sb_timeo.tv_usec / tick;
+	if (sbi->sbi_timeo == 0 && sb->sb_timeo.tv_usec != 0)
+		sbi->sbi_timeo = 1;
+}
+
+static void
+fill_common_sockinfo(struct socket *so, struct socket_info *si)
+{
+	si->soi_so = (u_int64_t)((uintptr_t)so);
+	si->soi_type = so->so_type;
+	si->soi_options = so->so_options;
+	si->soi_linger = so->so_linger;
+	si->soi_state = so->so_state;
+	si->soi_pcb = (u_int64_t)((uintptr_t)so->so_pcb);
+	if (so->so_proto) {
+		si->soi_protocol = so->so_proto->pr_protocol;
+		if (so->so_proto->pr_domain)
+			si->soi_family = so->so_proto->pr_domain->dom_family;
+		else
+			si->soi_family = 0;
+	} else
+		si->soi_protocol = si->soi_family = 0;
+	si->soi_qlen = so->so_qlen;
+	si->soi_incqlen = so->so_incqlen;
+	si->soi_qlimit = so->so_qlimit;
+	si->soi_timeo = so->so_timeo;
+	si->soi_error = so->so_error;
+	si->soi_oobmark = so->so_oobmark;
+	fill_sockbuf_info(&so->so_snd, &si->soi_snd);
+	fill_sockbuf_info(&so->so_rcv, &si->soi_rcv);
+	
+}
+
+errno_t
+fill_socketinfo(struct socket *so, struct socket_info *si)
+{
+	errno_t error = 0;
+	int family;
+	short type;
+	short protocol;
+		
+	socket_lock(so, 0);
+
+	si->soi_kind = SOCKINFO_GENERIC;
+
+	fill_common_sockinfo(so, si);
+
+	if (so->so_pcb == 0 || so->so_proto == 0 || so->so_proto->pr_domain == 0)
+		goto out;
+
+	/* The kind of socket is determined by the triplet {family, type, protocol} */
+	family = so->so_proto->pr_domain->dom_family;
+	type =  so->so_proto->pr_type;
+	protocol =  so->so_proto->pr_protocol;
+	switch (family) {
+		case AF_INET: 
+		case AF_INET6: {
+			struct in_sockinfo *insi = &si->soi_proto.pri_in;
+			struct inpcb *inp = (struct inpcb *)so->so_pcb;
+			
+			si->soi_kind = SOCKINFO_IN;
+			
+			insi->insi_fport = inp->inp_fport;
+			insi->insi_lport = inp->inp_lport;
+			insi->insi_gencnt = inp->inp_gencnt;
+			insi->insi_flags = inp->inp_flags;
+			insi->insi_vflag = inp->inp_vflag;
+			insi->insi_ip_ttl = inp->inp_ip_ttl;
+			insi->insi_faddr.ina_6 = inp->inp_dependfaddr.inp6_foreign;
+			insi->insi_laddr.ina_6 = inp->inp_dependladdr.inp6_local;
+			insi->insi_v4.in4_tos = inp->inp_depend4.inp4_ip_tos;
+			insi->insi_v6.in6_hlim = inp->inp_depend6.inp6_hlim;
+			insi->insi_v6.in6_cksum = inp->inp_depend6.inp6_cksum;
+			insi->insi_v6.in6_ifindex = inp->inp6_ifindex;
+			insi->insi_v6.in6_hops = inp->inp_depend6.inp6_hops;
+
+			if (type == SOCK_STREAM && (protocol == 0 || protocol == IPPROTO_TCP) && inp->inp_ppcb != 0) {
+				struct tcp_sockinfo *tcpsi = &si->soi_proto.pri_tcp;
+				struct tcpcb *tp= (struct tcpcb *)inp->inp_ppcb;
+				
+				si->soi_kind = SOCKINFO_TCP;
+
+				tcpsi->tcpsi_state = tp->t_state;
+				tcpsi->tcpsi_timer[TCPT_REXMT] = tp->t_timer[TCPT_REXMT];
+				tcpsi->tcpsi_timer[TCPT_PERSIST] = tp->t_timer[TCPT_PERSIST];
+				tcpsi->tcpsi_timer[TCPT_KEEP] = tp->t_timer[TCPT_KEEP];
+				tcpsi->tcpsi_timer[TCPT_2MSL] = tp->t_timer[TCPT_2MSL];
+				tcpsi->tcpsi_mss = tp->t_maxseg;
+				tcpsi->tcpsi_flags = tp->t_flags;
+				tcpsi->tcpsi_tp = (u_int64_t)((uintptr_t)tp);
+			}
+			break;
+		}
+		case AF_UNIX: {
+			struct unpcb *unp = (struct unpcb *)so->so_pcb;
+			struct un_sockinfo *unsi = &si->soi_proto.pri_un;
+			
+			si->soi_kind = SOCKINFO_UN;
+			
+			unsi->unsi_conn_pcb = (uint64_t)((uintptr_t)unp->unp_conn);
+			if (unp->unp_conn)
+				unsi->unsi_conn_so = (uint64_t)((uintptr_t)unp->unp_conn->unp_socket);
+				
+			
+			if (unp->unp_addr) {
+				size_t	addrlen = unp->unp_addr->sun_len;
+				
+				if (addrlen > SOCK_MAXADDRLEN)
+					addrlen = SOCK_MAXADDRLEN;
+				bcopy(unp->unp_addr, &unsi->unsi_addr, addrlen);
+			}
+			if (unp->unp_conn && unp->unp_conn->unp_addr) {
+				size_t	addrlen = unp->unp_conn->unp_addr->sun_len;
+				
+				if (addrlen > SOCK_MAXADDRLEN)
+					addrlen = SOCK_MAXADDRLEN;
+				bcopy(unp->unp_conn->unp_addr, &unsi->unsi_caddr, addrlen);
+			}
+			break;
+		}
+		case AF_NDRV: {
+			struct ndrv_cb *ndrv_cb = (struct ndrv_cb *)so->so_pcb;
+			struct ndrv_info *ndrvsi = &si->soi_proto.pri_ndrv;
+
+			si->soi_kind = SOCKINFO_NDRV;
+
+			/* TDB lock ifnet ???? */
+			if (ndrv_cb->nd_if != 0) {
+				struct ifnet *ifp = ndrv_cb->nd_if;
+				
+				ndrvsi->ndrvsi_if_family = ifp->if_family;
+				ndrvsi->ndrvsi_if_unit = ifp->if_unit;
+				strncpy(ndrvsi->ndrvsi_if_name, ifp->if_name, IFNAMSIZ);
+			}
+
+			break;
+		}
+		case AF_SYSTEM:
+			if (so->so_proto->pr_protocol == SYSPROTO_EVENT) {
+				struct kern_event_pcb *ev_pcb = (struct kern_event_pcb *)so->so_pcb;
+				struct kern_event_info *kesi = &si->soi_proto.pri_kern_event;
+				
+				si->soi_kind = SOCKINFO_KERN_EVENT;
+				
+				kesi->kesi_vendor_code_filter = ev_pcb->vendor_code_filter;
+				kesi->kesi_class_filter = ev_pcb->class_filter;
+				kesi->kesi_subclass_filter = ev_pcb->subclass_filter;
+				
+			} else if (so->so_proto->pr_protocol == SYSPROTO_CONTROL) {
+				struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
+				struct kern_ctl_info *kcsi = &si->soi_proto.pri_kern_ctl;
+				struct kctl *kctl = kcb->kctl;
+				
+				
+				si->soi_kind = SOCKINFO_KERN_CTL;
+				
+				if (kctl == 0)
+					break;
+				kcsi->kcsi_id = kctl->id;
+				kcsi->kcsi_reg_unit = kctl->id;
+				kcsi->kcsi_flags = kctl->flags;
+				kcsi->kcsi_recvbufsize = kctl->recvbufsize;
+				kcsi->kcsi_sendbufsize = kctl->sendbufsize;
+				kcsi->kcsi_unit = kcb->unit;
+				strncpy(kcsi->kcsi_name, kctl->name, MAX_KCTL_NAME);
+			}
+			break;
+
+		case AF_APPLETALK:
+			break;
+			
+		case AF_ROUTE:
+			break;
+
+		case AF_PPP:
+			break;
+		
+		default:
+			break;
+	}
+out:	
+	socket_unlock(so, 0);
+	
+	return error;
+}
+
diff --git a/bsd/kern/subr_prof.c b/bsd/kern/subr_prof.c
index 5d6468c88..b57b14249 100644
--- a/bsd/kern/subr_prof.c
+++ b/bsd/kern/subr_prof.c
@@ -143,7 +143,6 @@ kmstartup(void)
 	
 	mcount_lock_grp = lck_grp_alloc_init("MCOUNT", LCK_GRP_ATTR_NULL);
 	mcount_lock_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(mcount_lock_attr);
 	mcount_lock = lck_spin_alloc_init(mcount_lock_grp, mcount_lock_attr);
 
 }
diff --git a/bsd/kern/sys_generic.c b/bsd/kern/sys_generic.c
index 02286d88d..bb215dfdd 100644
--- a/bsd/kern/sys_generic.c
+++ b/bsd/kern/sys_generic.c
@@ -1706,8 +1706,6 @@ selthreadclear(sip)
 
 
 
-#define DBG_EVENT	0x10
-
 #define DBG_POST	0x10
 #define DBG_WATCH	0x11
 #define DBG_WAIT	0x12
diff --git a/bsd/kern/sys_pipe.c b/bsd/kern/sys_pipe.c
index f7c0fa30c..4a117ac97 100644
--- a/bsd/kern/sys_pipe.c
+++ b/bsd/kern/sys_pipe.c
@@ -111,6 +111,7 @@
 #include <sys/signalvar.h>
 #include <sys/pipe.h>
 #include <sys/sysproto.h>
+#include <sys/proc_info.h>
 
 #include <bsm/audit_kernel.h>
 
@@ -266,14 +267,12 @@ pipeinit(void *dummy __unused)
 	 * allocate lock group attribute and group for pipe mutexes
 	 */
 	pipe_mtx_grp_attr = lck_grp_attr_alloc_init();
-	//lck_grp_attr_setstat(pipe_mtx_grp_attr);
 	pipe_mtx_grp = lck_grp_alloc_init("pipe", pipe_mtx_grp_attr);
 
 	/*
 	 * allocate the lock attribute for pipe mutexes
 	 */
 	pipe_mtx_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(pipe_mtx_attr);
 }
 
 
@@ -1416,7 +1415,7 @@ pipe_free_kmem(struct pipe *cpipe)
 	if (cpipe->pipe_buffer.buffer != NULL) {
 		if (cpipe->pipe_buffer.size > PIPE_SIZE)
 			OSAddAtomic(-1, (SInt32 *)&nbigpipe);
-		OSAddAtomic(cpipe->pipe_buffer.size, (SInt32 *)&amountpipekva);
+		OSAddAtomic(-(cpipe->pipe_buffer.size), (SInt32 *)&amountpipekva);
 		OSAddAtomic(-1, (SInt32 *)&amountpipes);
 
 		kmem_free(kernel_map, (vm_offset_t)cpipe->pipe_buffer.buffer,
@@ -1650,3 +1649,66 @@ filt_pipewrite(struct knote *kn, long hint)
 	return (kn->kn_data >= ((kn->kn_sfflags & NOTE_LOWAT) ?
 	                         kn->kn_sdata : PIPE_BUF));
 }
+
+int
+fill_pipeinfo(struct pipe * cpipe, struct pipe_info * pinfo)
+{
+#ifdef MAC
+        int error;
+#endif
+	struct timeval now;
+	struct stat * ub;
+
+	if (cpipe == NULL)
+	        return (EBADF);
+#ifdef MAC
+	PIPE_LOCK(cpipe);
+	error = mac_check_pipe_stat(active_cred, cpipe);
+	PIPE_UNLOCK(cpipe);
+	if (error)
+	        return (error);
+#endif
+	if (cpipe->pipe_buffer.buffer == 0) {
+	        /*
+		 * must be stat'ing the write fd
+		 */
+	        cpipe = cpipe->pipe_peer;
+
+		if (cpipe == NULL)
+		        return (EBADF);
+	}
+
+	ub = &pinfo->pipe_stat;
+
+	bzero(ub, sizeof(*ub));
+	ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP;
+	ub->st_blksize = cpipe->pipe_buffer.size;
+	ub->st_size = cpipe->pipe_buffer.cnt;
+	if (ub->st_blksize != 0);
+		ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize;
+	ub->st_nlink = 1;
+
+	ub->st_uid = kauth_getuid();
+	ub->st_gid = kauth_getgid();
+
+	microtime(&now);
+	ub->st_atimespec.tv_sec  = now.tv_sec;
+	ub->st_atimespec.tv_nsec = now.tv_usec * 1000;
+
+	ub->st_mtimespec.tv_sec  = now.tv_sec;
+	ub->st_mtimespec.tv_nsec = now.tv_usec * 1000;
+
+	ub->st_ctimespec.tv_sec  = now.tv_sec;
+	ub->st_ctimespec.tv_nsec = now.tv_usec * 1000;
+
+	/*
+	 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen, st_uid, st_gid.
+	 * XXX (st_dev, st_ino) should be unique.
+	 */
+
+	pinfo->pipe_handle = (uint64_t)((uintptr_t)cpipe);
+	pinfo->pipe_peerhandle = (uint64_t)((uintptr_t)(cpipe->pipe_peer));
+	pinfo->pipe_status = cpipe->pipe_state;
+	return (0);
+}
+
diff --git a/bsd/kern/syscalls.c b/bsd/kern/syscalls.c
index ac1f91442..b0abc908f 100644
--- a/bsd/kern/syscalls.c
+++ b/bsd/kern/syscalls.c
@@ -140,11 +140,7 @@ const char *syscallnames[] = {
 	"getpriority",        /* 100 = getpriority */
 	"#101",               /* 101 = old send */
 	"#102",               /* 102 = old recv */
-#ifdef __ppc__
 	"#103",               /* 103 = old sigreturn */
-#else
-	"sigreturn",          /* 103 = sigreturn */
-#endif
 	"bind",               /* 104 = bind */
 	"setsockopt",         /* 105 = setsockopt */
 	"listen",             /* 106 = listen */
@@ -157,11 +153,7 @@ const char *syscallnames[] = {
 	"#113",               /* 113 = old recvmsg */
 	"#114",               /* 114 = old sendmsg */
 	"#115",               /* 115 = old vtrace */
-#ifdef __ppc__
-	"ppc_gettimeofday",   /* 116 = ppc_gettimeofday */
-#else
 	"gettimeofday",       /* 116 = gettimeofday */
-#endif
 	"getrusage",          /* 117 = getrusage */
 	"getsockopt",         /* 118 = getsockopt */
 	"#119",               /* 119 = old resuba */
@@ -237,12 +229,8 @@ const char *syscallnames[] = {
 	"setgid",             /* 181 = setgid */
 	"setegid",            /* 182 = setegid */
 	"seteuid",            /* 183 = seteuid */
-#ifdef __ppc__
 	"sigreturn",          /* 184 = sigreturn */
-#else
-	"#184",               /* 184 = */
-#endif
-	"#185",               /* 185 = */
+	"chud",               /* 185 = chud */
 	"#186",               /* 186 = */
 	"#187",               /* 187 = */
 	"stat",               /* 188 = stat */
@@ -267,16 +255,6 @@ const char *syscallnames[] = {
 	"mlock",              /* 203 = mlock */
 	"munlock",            /* 204 = munlock */
 	"undelete",           /* 205 = undelete */
-#ifdef __ppc__
-	"ATsocket",           /* 206 = ATsocket */
-	"ATgetmsg",           /* 207 = ATgetmsg */
-	"ATputmsg",           /* 208 = ATputmsg */
-	"ATPsndreq",          /* 209 = ATPsndreq */
-	"ATPsndrsp",          /* 210 = ATPsndrsp */
-	"ATPgetreq",          /* 211 = ATPgetreq */
-	"ATPgetrsp",          /* 212 = ATPgetrsp */
-	"#213",               /* 213 = Reserved for AppleTalk */
-#else
 	"ATsocket",           /* 206 = ATsocket */
 	"ATgetmsg",           /* 207 = ATgetmsg */
 	"ATputmsg",           /* 208 = ATputmsg */
@@ -285,7 +263,6 @@ const char *syscallnames[] = {
 	"ATPgetreq",          /* 211 = ATPgetreq */
 	"ATPgetrsp",          /* 212 = ATPgetrsp */
 	"#213",               /* 213 = Reserved for AppleTalk */
-#endif /* __ppc__ */
 	"kqueue_from_portset_np",  /* 214 = kqueue_from_portset_np */
 	"kqueue_portset_np",  /* 215 = kqueue_portset_np */
 	"mkcomplex",          /* 216 = mkcomplex soon to be obsolete */
@@ -417,7 +394,7 @@ const char *syscallnames[] = {
 	"__pthread_canceled",  /* 333 = __pthread_canceled */
 	"__semwait_signal",   /* 334 = __semwait_signal */
 	"utrace",             /* 335 = utrace */
-	"#336",               /* 336 = */
+	"proc_info",          /* 336 = proc_info */
 	"#337",               /* 337 = */
 	"#338",               /* 338 = */
 	"#339",               /* 339 = */
@@ -446,7 +423,7 @@ const char *syscallnames[] = {
 	"kqueue",             /* 362 = kqueue */
 	"kevent",             /* 363 = kevent */
 	"lchown",             /* 364 = lchown */
-	"#365",               /* 365 = */
+	"stack_snapshot",     /* 365 = stack_snapshot */
 	"#366",               /* 366 = */
 	"#367",               /* 367 = */
 	"#368",               /* 368 = */
diff --git a/bsd/kern/syscalls.master b/bsd/kern/syscalls.master
index a051ba6cf..0c0008058 100644
--- a/bsd/kern/syscalls.master
+++ b/bsd/kern/syscalls.master
@@ -136,13 +136,7 @@
 100	NONE	KERN	ALL	{ int getpriority(int which, int who); } 
 101	NONE	NONE	ALL	{ int nosys(void); }   { old send }
 102	NONE	NONE	ALL	{ int nosys(void); }   { old recv }
-
-#ifdef __ppc__
 103	NONE	NONE	ALL	{ int nosys(void); }   { old sigreturn }
-#else
-103	NONE	KERN	UALL	{ int sigreturn(struct sigcontext *sigcntxp); } 
-#endif
-
 104	NONE	NONE	ALL	{ int bind(int s, caddr_t name, socklen_t namelen); } 
 105	NONE	NONE	ALL	{ int setsockopt(int s, int level, int name, caddr_t val, socklen_t valsize); } 
 106	NONE	NONE	ALL	{ int listen(int s, int backlog); } 
@@ -155,13 +149,7 @@
 113	NONE	NONE	ALL	{ int nosys(void); }   { old recvmsg }
 114	NONE	NONE	ALL	{ int nosys(void); }   { old sendmsg }	
 115	NONE	NONE	ALL	{ int nosys(void); }   { old vtrace }
-
-#ifdef __ppc__
-116	NONE	NONE	ALL	{ int ppc_gettimeofday(struct timeval *tp, struct timezone *tzp); } 
-#else
 116	NONE	NONE	ALL	{ int gettimeofday(struct timeval *tp, struct timezone *tzp); } 
-#endif
-
 117	NONE	KERN	ALL	{ int getrusage(int who, struct rusage *rusage); } 
 118	NONE	NONE	ALL	{ int getsockopt(int s, int level, int name, caddr_t val, socklen_t *avalsize); } 
 119	NONE	NONE	ALL	{ int nosys(void); }   { old resuba }
@@ -241,14 +229,8 @@
 181	NONE	KERN	ALL	{ int setgid(gid_t gid); } 
 182	NONE	KERN	ALL	{ int setegid(gid_t egid); } 
 183	NONE	KERN	ALL	{ int seteuid(uid_t euid); } 
-
-#ifdef __ppc__
 184	NONE	KERN	ALL	{ int sigreturn(struct ucontext *uctx, int infostyle); } 
-#else
-184	NONE	NONE	ALL	{ int nosys(void); } 
-#endif
-
-185	NONE	NONE	ALL	{ int nosys(void); } 
+185	NONE	NONE	UALL { int chud(int code, int arg1, int arg2, int arg3, int arg4, int arg5); } 
 186	NONE	NONE	ALL	{ int nosys(void); } 
 187	NONE	NONE	ALL	{ int nosys(void); } 
 188	NONE	NONE	ALL	{ int stat(user_addr_t path, user_addr_t ub); } 
@@ -276,7 +258,6 @@
 204	NONE	NONE	ALL	{ int munlock(caddr_t addr, size_t len); } 
 205	NONE	NONE	ALL	{ int undelete(user_addr_t path); } 
 
-#ifdef __ppc__
 206	NONE	NONE	ALL	{ int ATsocket(int proto); } 
 207	NONE	NONE	UALL	{ int ATgetmsg(int fd, void *ctlptr, void *datptr, int *flags); } 
 208	NONE	NONE	UALL	{ int ATputmsg(int fd, void *ctlptr, void *datptr, int flags); } 
@@ -285,16 +266,6 @@
 211	NONE	NONE	UALL	{ int ATPgetreq(int fd, unsigned char *buf, int buflen); } 
 212	NONE	NONE	UALL	{ int ATPgetrsp(int fd, unsigned char *bdsp); } 
 213	NONE	NONE	ALL	{ int nosys(void); } 	{ Reserved for AppleTalk }
-#else
-206	NONE	NONE	HN	{ int ATsocket(int proto); } 
-207	NONE	NONE	UHN	{ int ATgetmsg(int fd, void *ctlptr, void *datptr, int *flags); } 
-208	NONE	NONE	UHN	{ int ATputmsg(int fd, void *ctlptr, void *datptr, int flags); } 
-209	NONE	NONE	UHN	{ int ATPsndreq(int fd, unsigned char *buf, int len, int nowait); } 
-210	NONE	NONE	UHN	{ int ATPsndrsp(int fd, unsigned char *respbuff, int resplen, int datalen); } 
-211	NONE	NONE	UHN	{ int ATPgetreq(int fd, unsigned char *buf, int buflen); } 
-212	NONE	NONE	UHN	{ int ATPgetrsp(int fd, unsigned char *bdsp); } 
-213	NONE	NONE	ALL	{ int nosys(void); } 	{ Reserved for AppleTalk }
-#endif /* __ppc__ */
 
 214	NONE	KERN	ALL	{ int kqueue_from_portset_np(int portset); } 
 215	NONE	KERN	ALL	{ int kqueue_portset_np(int fd); } 
@@ -438,7 +409,7 @@
 333	NONE	NONE	ALL	{ int __pthread_canceled(int  action); } 
 334	POST	NONE	ALL	{ int __semwait_signal(int cond_sem, int mutex_sem, int timeout, int relative, time_t tv_sec, int32_t tv_nsec); } 
 335	NONE	KERN	ALL	{ int utrace(const void *addr, size_t len); } 
-336	NONE	NONE	ALL	{ int nosys(void); } 
+336	NONE	NONE	ALL	{ int proc_info(int32_t callnum,int32_t pid,uint32_t flavor, uint64_t arg,user_addr_t buffer,int32_t buffersize); } 
 337	NONE	NONE	ALL	{ int nosys(void); } 
 338	NONE	NONE	ALL	{ int nosys(void); } 
 339	NONE	NONE	ALL	{ int nosys(void); } 
@@ -467,7 +438,7 @@
 362	NONE	NONE	ALL	{ int kqueue(void); } 
 363	NONE	NONE	ALL	{ int kevent(int fd, const struct kevent *changelist, int nchanges, struct kevent *eventlist, int nevents, const struct timespec *timeout); } 
 364	NONE	NONE	ALL	{ int lchown(user_addr_t path, uid_t owner, gid_t group); }
-365	NONE	NONE	ALL	{ int nosys(void); } 
+365	NONE	NONE	ALL	{ int stack_snapshot(pid_t pid, user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t options); }
 366	NONE	NONE	ALL	{ int nosys(void); } 
 367	NONE	NONE	ALL	{ int nosys(void); } 
 368	NONE	NONE	ALL	{ int nosys(void); } 
diff --git a/bsd/kern/sysctl_init.c b/bsd/kern/sysctl_init.c
index 89674a5bb..4284aa42e 100644
--- a/bsd/kern/sysctl_init.c
+++ b/bsd/kern/sysctl_init.c
@@ -56,9 +56,12 @@ extern struct sysctl_oid sysctl__hw_logicalcpu_max;
 extern struct sysctl_oid sysctl__hw_byteorder;
 extern struct sysctl_oid sysctl__hw_cputype;
 extern struct sysctl_oid sysctl__hw_cpusubtype;
+extern struct sysctl_oid sysctl__hw_cpufamily;
+extern struct sysctl_oid sysctl__hw_cacheconfig;
 extern struct sysctl_oid sysctl__hw_physmem;
 extern struct sysctl_oid sysctl__hw_usermem;
 extern struct sysctl_oid sysctl__hw_pagesize;
+extern struct sysctl_oid sysctl__hw_pms;
 extern struct sysctl_oid sysctl__hw_epoch;
 extern struct sysctl_oid sysctl__hw_vectorunit;
 extern struct sysctl_oid sysctl__hw_busfrequency;
@@ -90,6 +93,12 @@ extern struct sysctl_oid sysctl__hw_l2cachesize_compat;
 extern struct sysctl_oid sysctl__hw_l3cachesize_compat;
 extern struct sysctl_oid sysctl__hw_tbfrequency_compat;
 
+#if __i386__
+extern struct sysctl_oid sysctl__sysctl_proc_exec_affinity;
+#endif
+extern struct sysctl_oid sysctl__sysctl_proc_cputype;
+extern struct sysctl_oid sysctl__sysctl_proc_native;
+
 extern struct sysctl_oid sysctl__kern_sysv_shmmax;
 extern struct sysctl_oid sysctl__kern_sysv_shmmin;
 extern struct sysctl_oid sysctl__kern_sysv_shmmni;
@@ -106,6 +115,10 @@ extern struct sysctl_oid sysctl__kern_sysv_semmnu;
 extern struct sysctl_oid sysctl__kern_sysv_semmsl;
 extern struct sysctl_oid sysctl__kern_sysv_semume;
 
+extern struct sysctl_oid sysctl__kern_tfp_policy;
+extern struct sysctl_oid sysctl__kern_tfp_read_group;
+extern struct sysctl_oid sysctl__kern_tfp_rw_group;
+
 extern struct sysctl_oid sysctl__kern_dummy;
 extern struct sysctl_oid sysctl__kern_ipc_maxsockbuf;
 extern struct sysctl_oid sysctl__kern_ipc_mbstat;
@@ -119,8 +132,13 @@ extern struct sysctl_oid sysctl__kern_posix;
 extern struct sysctl_oid sysctl__kern_posix_sem;
 extern struct sysctl_oid sysctl__kern_posix_sem_max;
 extern struct sysctl_oid sysctl__kern_sugid_scripts;
+extern struct sysctl_oid sysctl__kern_nbuf;
+extern struct sysctl_oid sysctl__kern_maxnbuf;
 extern struct sysctl_oid sysctl__kern_always_do_fullfsync;
 extern struct sysctl_oid sysctl__kern_flush_cache_on_write;
+extern struct sysctl_oid sysctl__kern_exec;
+extern struct sysctl_oid sysctl__kern_exec_archhandler;
+extern struct sysctl_oid sysctl__kern_exec_archhandler_powerpc;
 extern struct sysctl_oid sysctl__net_inet_icmp_icmplim;
 extern struct sysctl_oid sysctl__net_inet_icmp_maskrepl;
 extern struct sysctl_oid sysctl__net_inet_icmp_timestamp;
@@ -326,6 +344,7 @@ extern struct sysctl_oid sysctl__vfs_generic_noremotehang;
 
 extern struct sysctl_oid sysctl__kern_ipc;
 extern struct sysctl_oid sysctl__kern_sysv;
+extern struct sysctl_oid sysctl__kern_tfp;
 
 extern struct sysctl_oid sysctl__net_inet;
 
@@ -382,6 +401,7 @@ extern struct sysctl_oid sysctl__machdep;
 extern struct sysctl_oid sysctl__net;
 extern struct sysctl_oid sysctl__debug;
 extern struct sysctl_oid sysctl__vfs;
+extern struct sysctl_oid sysctl__vm;
 extern struct sysctl_oid sysctl__sysctl;
 
 #if INET6
@@ -481,6 +501,7 @@ extern struct sysctl_oid sysctl__net_key_natt_keepalive_interval;
 extern struct sysctl_oid sysctl__net_key_pfkeystat;
 #endif
 
+extern struct sysctl_oid sysctl__vm_shared_region_trace_level;
 
 struct sysctl_oid *newsysctl_list[] =
 {
@@ -490,6 +511,7 @@ struct sysctl_oid *newsysctl_list[] =
     &sysctl__net,
     &sysctl__debug,
     &sysctl__vfs,
+    &sysctl__vm,
     &sysctl__sysctl,
     &sysctl__debug_bpf_bufsize,
     &sysctl__debug_bpf_maxbufsize,
@@ -509,6 +531,12 @@ struct sysctl_oid *newsysctl_list[] =
     ,&sysctl__kern_posix_sem
     ,&sysctl__kern_posix_sem_max
 
+#if __i386__
+    ,&sysctl__sysctl_proc_exec_affinity
+#endif
+    ,&sysctl__sysctl_proc_cputype
+    ,&sysctl__sysctl_proc_native
+	
     ,&sysctl__kern_sysv_shmmax
     ,&sysctl__kern_sysv_shmmin
     ,&sysctl__kern_sysv_shmmni
@@ -525,6 +553,11 @@ struct sysctl_oid *newsysctl_list[] =
     ,&sysctl__kern_sysv_semume
     ,&sysctl__kern_dummy
 
+    ,&sysctl__kern_tfp
+    ,&sysctl__kern_tfp_policy
+    ,&sysctl__kern_tfp_read_group
+    ,&sysctl__kern_tfp_rw_group
+
     ,&sysctl__kern_ipc_maxsockbuf
     ,&sysctl__kern_ipc_mbstat
     ,&sysctl__kern_ipc_nmbclusters
@@ -537,6 +570,11 @@ struct sysctl_oid *newsysctl_list[] =
     ,&sysctl__kern_sugid_scripts
     ,&sysctl__kern_always_do_fullfsync
     ,&sysctl__kern_flush_cache_on_write
+    ,&sysctl__kern_exec
+    ,&sysctl__kern_exec_archhandler
+    ,&sysctl__kern_exec_archhandler_powerpc
+    ,&sysctl__kern_nbuf
+    ,&sysctl__kern_maxnbuf
 
     ,&sysctl__hw_machine
     ,&sysctl__hw_model
@@ -549,6 +587,8 @@ struct sysctl_oid *newsysctl_list[] =
     ,&sysctl__hw_byteorder
     ,&sysctl__hw_cputype
     ,&sysctl__hw_cpusubtype
+    ,&sysctl__hw_cpufamily
+    ,&sysctl__hw_cacheconfig
     ,&sysctl__hw_physmem
     ,&sysctl__hw_usermem
     ,&sysctl__hw_pagesize
@@ -573,6 +613,7 @@ struct sysctl_oid *newsysctl_list[] =
     ,&sysctl__hw_optional_floatingpoint
 
     ,&sysctl__hw_pagesize_compat
+    ,&sysctl__hw_pms    
     ,&sysctl__hw_busfrequency_compat
     ,&sysctl__hw_cpufrequency_compat
     ,&sysctl__hw_cachelinesize_compat
@@ -918,6 +959,9 @@ struct sysctl_oid *newsysctl_list[] =
    ,&sysctl__net_inet_ipsec_bypass
    ,&sysctl__net_inet_ipsec_esp_port
 #endif
+
+    ,&sysctl__vm_shared_region_trace_level
+
     ,(struct sysctl_oid *) 0
 };
 
diff --git a/bsd/kern/sysv_msg.c b/bsd/kern/sysv_msg.c
index 649eab266..798b73325 100644
--- a/bsd/kern/sysv_msg.c
+++ b/bsd/kern/sysv_msg.c
@@ -111,12 +111,10 @@ __private_extern__ void
 sysv_msg_lock_init( void )
 {
 	sysv_msg_subsys_lck_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(sysv_msg_subsys_lck_grp_attr);
 
 	sysv_msg_subsys_lck_grp = lck_grp_alloc_init("sysv_msg_subsys_lock", sysv_msg_subsys_lck_grp_attr);
 
 	sysv_msg_subsys_lck_attr = lck_attr_alloc_init();
-	/* lck_attr_setdebug(sysv_msg_subsys_lck_attr); */
 	lck_mtx_init(&sysv_msg_subsys_mutex, sysv_msg_subsys_lck_grp, sysv_msg_subsys_lck_attr);
 }
 
diff --git a/bsd/kern/sysv_sem.c b/bsd/kern/sysv_sem.c
index 5691a71a9..4237c3459 100644
--- a/bsd/kern/sysv_sem.c
+++ b/bsd/kern/sysv_sem.c
@@ -128,12 +128,10 @@ sysv_sem_lock_init( void )
 {
 
     sysv_sem_subsys_lck_grp_attr = lck_grp_attr_alloc_init();
-    lck_grp_attr_setstat(sysv_sem_subsys_lck_grp_attr);
 
-    sysv_sem_subsys_lck_grp = lck_grp_alloc_init("sysv_shm_subsys_lock", sysv_sem_subsys_lck_grp_attr);
+    sysv_sem_subsys_lck_grp = lck_grp_alloc_init("sysv_sem_subsys_lock", sysv_sem_subsys_lck_grp_attr);
 
     sysv_sem_subsys_lck_attr = lck_attr_alloc_init();
-    lck_attr_setdebug(sysv_sem_subsys_lck_attr); 
     lck_mtx_init(&sysv_sem_subsys_mutex, sysv_sem_subsys_lck_grp, sysv_sem_subsys_lck_attr);
 }
 
diff --git a/bsd/kern/sysv_shm.c b/bsd/kern/sysv_shm.c
index b8dfa9934..37822b44e 100644
--- a/bsd/kern/sysv_shm.c
+++ b/bsd/kern/sysv_shm.c
@@ -835,12 +835,10 @@ sysv_shm_lock_init( void )
 {
 
 	sysv_shm_subsys_lck_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(sysv_shm_subsys_lck_grp_attr);
 	
 	sysv_shm_subsys_lck_grp = lck_grp_alloc_init("sysv_shm_subsys_lock", sysv_shm_subsys_lck_grp_attr);
 	
 	sysv_shm_subsys_lck_attr = lck_attr_alloc_init();
-	/* lck_attr_setdebug(sysv_shm_subsys_lck_attr); */
 	lck_mtx_init(&sysv_shm_subsys_mutex, sysv_shm_subsys_lck_grp, sysv_shm_subsys_lck_attr);
 }
 
@@ -853,14 +851,14 @@ sysctl_shminfo(__unused struct sysctl_oid *oidp, void *arg1,
 	int error = 0;
 	int sysctl_shminfo_ret = 0;
 
-	error = SYSCTL_OUT(req, arg1, sizeof(user_ssize_t));
+	error = SYSCTL_OUT(req, arg1, sizeof(int64_t));
 	if (error || req->newptr == USER_ADDR_NULL)
 		return(error);
 
 	SYSV_SHM_SUBSYS_LOCK();
 	/* Set the values only if shared memory is not initialised */
 	if (!shm_inited) {
-		if ((error = SYSCTL_IN(req, arg1, sizeof(user_ssize_t))) 
+		if ((error = SYSCTL_IN(req, arg1, sizeof(int64_t))) 
 		    != 0) {
 			sysctl_shminfo_ret = error;
 			goto sysctl_shminfo_out;
@@ -868,18 +866,18 @@ sysctl_shminfo(__unused struct sysctl_oid *oidp, void *arg1,
 
 		if (arg1 == &shminfo.shmmax) {
 			if (shminfo.shmmax & PAGE_MASK_64) {
-				shminfo.shmmax = (user_ssize_t)-1;
+				shminfo.shmmax = (int64_t)-1;
 				sysctl_shminfo_ret = EINVAL;
 				goto sysctl_shminfo_out;
 			}
 		}
 
 		/* Initialize only when all values are set */
-		if ((shminfo.shmmax != (user_ssize_t)-1) &&
-			(shminfo.shmmin != (user_ssize_t)-1) &&	
-			(shminfo.shmmni != (user_ssize_t)-1) &&
-			(shminfo.shmseg != (user_ssize_t)-1) &&
-			(shminfo.shmall != (user_ssize_t)-1)) {
+		if ((shminfo.shmmax != (int64_t)-1) &&
+			(shminfo.shmmin != (int64_t)-1) &&	
+			(shminfo.shmmni != (int64_t)-1) &&
+			(shminfo.shmseg != (int64_t)-1) &&
+			(shminfo.shmall != (int64_t)-1)) {
 				shminit(NULL);
 		}
 	}
diff --git a/bsd/kern/tty_tty.c b/bsd/kern/tty_tty.c
index fac3c37cd..d6ad89213 100644
--- a/bsd/kern/tty_tty.c
+++ b/bsd/kern/tty_tty.c
@@ -119,8 +119,9 @@ cttyopen(__unused dev_t dev, int flag, __unused int mode, struct proc *p)
 		return (ENXIO);
 
 	context.vc_proc = p;
-	context.vc_ucred = p->p_ucred;
+	context.vc_ucred = kauth_cred_proc_ref(p);
 	error = VNOP_OPEN(ttyvp, flag, &context);
+	kauth_cred_unref(&context.vc_ucred);
 
 	return (error);
 }
diff --git a/bsd/kern/ubc_subr.c b/bsd/kern/ubc_subr.c
index cf2087a37..deaaa090a 100644
--- a/bsd/kern/ubc_subr.c
+++ b/bsd/kern/ubc_subr.c
@@ -76,9 +76,9 @@
 #endif /* DIAGNOSTIC */
 
 int ubc_info_init_internal(struct vnode *vp, int withfsize, off_t filesize);
-int ubc_umcallback(vnode_t, void *);
+static int ubc_umcallback(vnode_t, void *);
 int ubc_isinuse_locked(vnode_t, int, int);
-int ubc_msync_internal(vnode_t, off_t, off_t, off_t *, int, int *);
+static int ubc_msync_internal(vnode_t, off_t, off_t, off_t *, int, int *);
 
 struct zone	*ubc_info_zone;
 
@@ -199,12 +199,8 @@ ubc_info_init_internal(struct vnode *vp, int withfsize, off_t filesize)
 static void
 ubc_info_free(struct ubc_info *uip)
 {
-	kauth_cred_t credp;
-	
-	credp = uip->ui_ucred;
-	if (credp != NOCRED) {
-		uip->ui_ucred = NOCRED;
-		kauth_cred_rele(credp);
+	if (IS_VALID_CRED(uip->ui_ucred)) {
+		kauth_cred_unref(&uip->ui_ucred);
 	}
 
 	if (uip->ui_control != MEMORY_OBJECT_CONTROL_NULL)
@@ -369,7 +365,7 @@ ubc_setthreadcred(struct vnode *vp, struct proc *p, thread_t thread)
 	uip = vp->v_ubcinfo;
 	credp = uip->ui_ucred;
 
-	if (credp == NOCRED) {
+	if (!IS_VALID_CRED(credp)) {
 		/* use per-thread cred, if assumed identity, else proc cred */
 		if (uthread == NULL || (uthread->uu_flag & UT_SETUID) == 0) {
 			uip->ui_ucred = kauth_cred_proc_ref(p);
@@ -402,7 +398,7 @@ ubc_setcred(struct vnode *vp, struct proc *p)
 	uip = vp->v_ubcinfo;
 	credp = uip->ui_ucred;
 
-	if (credp == NOCRED) {
+	if (!IS_VALID_CRED(credp)) {
 		uip->ui_ucred = kauth_cred_proc_ref(p);
 	} 
 	vnode_unlock(vp);
diff --git a/bsd/kern/uipc_domain.c b/bsd/kern/uipc_domain.c
index e01929374..9bf1001f4 100644
--- a/bsd/kern/uipc_domain.c
+++ b/bsd/kern/uipc_domain.c
@@ -265,7 +265,6 @@ domaininit()
 	 * allocate lock group attribute and group for domain mutexes
 	 */
 	domain_proto_mtx_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setdefault(domain_proto_mtx_grp_attr);
 
 	domain_proto_mtx_grp = lck_grp_alloc_init("domain", domain_proto_mtx_grp_attr);
 		
@@ -273,7 +272,6 @@ domaininit()
 	 * allocate the lock attribute for per domain mutexes
 	 */
 	domain_proto_mtx_attr = lck_attr_alloc_init();
-	lck_attr_setdefault(domain_proto_mtx_attr);
 
 	if ((domain_proto_mtx = lck_mtx_alloc_init(domain_proto_mtx_grp, domain_proto_mtx_attr)) == NULL) {
 		printf("domaininit: can't init domain mtx for domain list\n");
diff --git a/bsd/kern/uipc_mbuf.c b/bsd/kern/uipc_mbuf.c
index 28a849ccd..81f27b090 100644
--- a/bsd/kern/uipc_mbuf.c
+++ b/bsd/kern/uipc_mbuf.c
@@ -151,7 +151,7 @@ static void mbuf_expand_thread(void);
 static int m_expand(int );
 static caddr_t m_bigalloc(int );
 static void m_bigfree(caddr_t ,  u_int ,  caddr_t );
-static struct mbuf * m_mbigget(struct mbuf *, int );
+__private_extern__ struct mbuf * m_mbigget(struct mbuf *, int );
 void mbinit(void);
 static void m_range_check(void *addr);
 
@@ -222,11 +222,9 @@ mbinit(void)
 	nclpp = round_page_32(MCLBYTES) / MCLBYTES;	/* see mbufgc() */
 	if (nclpp < 1) nclpp = 1;
 	mbuf_mlock_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setdefault(mbuf_mlock_grp_attr);
 
 	mbuf_mlock_grp = lck_grp_alloc_init("mbuf", mbuf_mlock_grp_attr);
 	mbuf_mlock_attr = lck_attr_alloc_init();
-	lck_attr_setdefault(mbuf_mlock_attr);
 
 	mbuf_mlock = lck_mtx_alloc_init(mbuf_mlock_grp, mbuf_mlock_attr);
 
@@ -824,7 +822,11 @@ m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
         to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
         if ((to->m_flags & M_EXT) == 0)
                 to->m_data = to->m_pktdat;
+	if (to->m_pkthdr.aux != NULL)
+		m_freem(to->m_pkthdr.aux);
         to->m_pkthdr = from->m_pkthdr;
+	to->m_pkthdr.aux = NULL;
+	(void) m_aux_copy(to, from);
         SLIST_INIT(&to->m_pkthdr.tags);
         return (m_tag_copy_chain(to, from, how));
 }
diff --git a/bsd/kern/uipc_mbuf2.c b/bsd/kern/uipc_mbuf2.c
index 649bc4db5..1a4340bb9 100644
--- a/bsd/kern/uipc_mbuf2.c
+++ b/bsd/kern/uipc_mbuf2.c
@@ -439,6 +439,46 @@ m_aux_delete(m, victim)
 	}
 }
 
+struct mbuf *
+m_aux_copy(struct mbuf *to, struct mbuf *from)
+{
+	struct mbuf *m;
+	struct mbuf *top = NULL, **np = &top;
+
+	if (!(to->m_flags & M_PKTHDR) || !(from->m_flags & M_PKTHDR))
+		return (NULL);
+
+	if ((m = from->m_pkthdr.aux) == NULL)
+		return (NULL);
+
+	while (m != NULL) {
+		struct mbuf *n;
+
+		MGET(n, M_DONTWAIT, m->m_type);
+		if (n == NULL) {
+			m_freem(top);
+			return (NULL);
+		}
+
+		/* Set length and data offset accordingly */
+		n->m_len = m->m_len;
+		n->m_data += (m->m_data - m->m_dat);
+
+		/* Copy databuf (mauxtag + possible aux data) */
+		bcopy(m->m_dat, n->m_dat, sizeof (m->m_dat));
+
+		*np = n;
+		np = &n->m_next;
+		m = m->m_next;
+	}
+
+	if (to->m_pkthdr.aux != NULL)
+		m_freem(to->m_pkthdr.aux);
+
+	to->m_pkthdr.aux = top;
+	return (top);
+}
+
 /* Get a packet tag structure along with specified data following. */
 struct m_tag *
 m_tag_alloc(u_int32_t id, u_int16_t type, int len, int wait)
diff --git a/bsd/kern/uipc_socket.c b/bsd/kern/uipc_socket.c
index 484b1c40d..9ba75e943 100644
--- a/bsd/kern/uipc_socket.c
+++ b/bsd/kern/uipc_socket.c
@@ -188,7 +188,6 @@ void socketinit()
 	 * allocate lock group attribute and group for socket cache mutex
 	 */
 	so_cache_mtx_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setdefault(so_cache_mtx_grp_attr);
 
 	so_cache_mtx_grp = lck_grp_alloc_init("so_cache", so_cache_mtx_grp_attr);
 		
@@ -196,7 +195,6 @@ void socketinit()
 	 * allocate the lock attribute for socket cache mutex
 	 */
 	so_cache_mtx_attr = lck_attr_alloc_init();
-	lck_attr_setdefault(so_cache_mtx_attr);
 
     so_cache_init_done = 1;
 
@@ -471,6 +469,9 @@ socreate(dom, aso, type, proto)
 	so->so_rcv.sb_flags |= SB_RECV;	/* XXX */
 	so->so_rcv.sb_so = so->so_snd.sb_so = so;
 #endif
+	so->next_lock_lr = 0;
+	so->next_unlock_lr = 0;
+	
 	
 //### Attachement will create the per pcb lock if necessary and increase refcount
 	so->so_usecount++;	/* for creation, make sure it's done before socket is inserted in lists */
@@ -571,7 +572,7 @@ sodealloc(so)
 		FREE(so->so_accf, M_ACCF);
 	}
 #endif /* INET */
-	kauth_cred_rele(so->so_cred);
+	kauth_cred_unref(&so->so_cred);
 	zfreei(so->so_zone, so);
 #else
 	if (so->cached_in_sock_layer == 1) 
@@ -973,13 +974,16 @@ soconnect2(so1, so2)
 	struct socket *so2;
 {
 	int error;
-//####### Assumes so1 is already locked /
 
-	socket_lock(so2, 1);
+	socket_lock(so1, 1);
+	if (so2->so_proto->pr_lock) 
+		socket_lock(so2, 1);
 
 	error = (*so1->so_proto->pr_usrreqs->pru_connect2)(so1, so2);
 	
-	socket_unlock(so2, 1);
+	socket_unlock(so1, 1);
+	if (so2->so_proto->pr_lock) 
+		socket_unlock(so2, 1);
 	return (error);
 }
 
@@ -2017,7 +2021,7 @@ static int sodelayed_copy(struct socket *so, struct uio *uio, struct mbuf **free
 int
 soshutdown(so, how)
 	register struct socket *so;
-	register int how;
+	int how;
 {
 	register struct protosw *pr = so->so_proto;
 	int ret;
@@ -2852,11 +2856,9 @@ socket_lock(so, refcount)
 	struct socket *so;
 	int refcount;
 {
-	int error = 0, lr, lr_saved;
-#ifdef __ppc__
-	__asm__ volatile("mflr %0" : "=r" (lr));
-        lr_saved = lr;
-#endif 
+	int error = 0, lr_saved;
+
+	lr_saved = (unsigned int) __builtin_return_address(0);
 
 	if (so->so_proto->pr_lock) {
 		error = (*so->so_proto->pr_lock)(so, refcount, lr_saved);
@@ -2868,7 +2870,8 @@ socket_lock(so, refcount)
 		lck_mtx_lock(so->so_proto->pr_domain->dom_mtx);
 		if (refcount)
 			so->so_usecount++;
-		so->reserved3 = (void*)lr_saved; /* save caller for refcount going to zero */
+		so->lock_lr[so->next_lock_lr] = (void *)lr_saved;
+		so->next_lock_lr = (so->next_lock_lr+1) % SO_LCKDBG_MAX;
 	}
 
 	return(error);
@@ -2880,15 +2883,10 @@ socket_unlock(so, refcount)
 	struct socket *so;
 	int refcount;
 {
-	int error = 0, lr, lr_saved;
+	int error = 0, lr_saved;
 	lck_mtx_t * mutex_held;
 
-#ifdef __ppc__
-__asm__ volatile("mflr %0" : "=r" (lr));
-        lr_saved = lr;
-#endif
-
-
+	lr_saved = (unsigned int) __builtin_return_address(0);
 
 	if (so->so_proto == NULL)
 		panic("socket_unlock null so_proto so=%x\n", so);
@@ -2900,6 +2898,9 @@ __asm__ volatile("mflr %0" : "=r" (lr));
 #ifdef MORE_LOCKING_DEBUG
 		lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
 #endif
+		so->unlock_lr[so->next_unlock_lr] = (void *)lr_saved;
+		so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX;
+
 		if (refcount) {
 			if (so->so_usecount <= 0)
 				panic("socket_unlock: bad refcount so=%x value=%d\n", so, so->so_usecount);
@@ -2907,8 +2908,6 @@ __asm__ volatile("mflr %0" : "=r" (lr));
 			if (so->so_usecount == 0) {
 				sofreelastref(so, 1);
 			}
-			else 
-				so->reserved4 = (void*)lr_saved; /* save caller */
 		}
 		lck_mtx_unlock(mutex_held);
 	}
@@ -2921,12 +2920,7 @@ sofree(so)
 	struct socket *so;
 {
 
-	int lr, lr_saved;
 	lck_mtx_t * mutex_held;
-#ifdef __ppc__
-	__asm__ volatile("mflr %0" : "=r" (lr));
-	lr_saved = lr;
-#endif
 	if (so->so_proto->pr_getlock != NULL)  
 		mutex_held = (*so->so_proto->pr_getlock)(so, 0);
 	else  
diff --git a/bsd/kern/uipc_socket2.c b/bsd/kern/uipc_socket2.c
index 1b2ced9a9..591ee95de 100644
--- a/bsd/kern/uipc_socket2.c
+++ b/bsd/kern/uipc_socket2.c
@@ -87,6 +87,7 @@
 #define DBG_FNC_SBDROP	NETDBG_CODE(DBG_NETSOCK, 4)
 #define DBG_FNC_SBAPPEND	NETDBG_CODE(DBG_NETSOCK, 5)
 
+static int sbcompress(struct sockbuf *, struct mbuf *, struct mbuf *);
 
 /*
  * Primitive routines for operating on sockets and socket buffers
@@ -293,6 +294,8 @@ sonewconn_internal(head, connstatus)
 	so->so_pgid  = head->so_pgid;
 	so->so_uid = head->so_uid;
 	so->so_usecount = 1;
+	so->next_lock_lr = 0;
+	so->next_unlock_lr = 0;
 
 #ifdef __APPLE__
 	so->so_rcv.sb_flags |= SB_RECV;	/* XXX */
@@ -335,8 +338,8 @@ sonewconn_internal(head, connstatus)
 	head->so_qlen++;
 
 #ifdef __APPLE__
-        /* Attach socket filters for this protocol */
-        sflt_initsock(so);
+	/* Attach socket filters for this protocol */
+	sflt_initsock(so);
 #endif
 	if (connstatus) {
 		so->so_state |= connstatus;
@@ -417,17 +420,13 @@ int
 sbwait(sb)
 	struct sockbuf *sb;
 {
-	int error = 0, lr, lr_saved;
+	int error = 0, lr_saved;
 	struct socket *so = sb->sb_so;
 	lck_mtx_t *mutex_held;
 	struct timespec ts;
 
-#ifdef __ppc__
-	__asm__ volatile("mflr %0" : "=r" (lr));
-	lr_saved = lr;
-#endif
+	lr_saved = (unsigned int) __builtin_return_address(0);
 	
-
 	if (so->so_proto->pr_getlock != NULL) 
 		mutex_held = (*so->so_proto->pr_getlock)(so, 0);
 	else 
@@ -465,12 +464,7 @@ sb_lock(sb)
 {
 	struct socket *so = sb->sb_so;
 	lck_mtx_t * mutex_held;
-	int error = 0, lr, lr_saved;
-
-#ifdef __ppc__
-	__asm__ volatile("mflr %0" : "=r" (lr));
-	lr_saved = lr;
-#endif
+	int error = 0;
 	
 	if (so == NULL)
 		panic("sb_lock: null so back pointer sb=%x\n", sb);
@@ -483,6 +477,7 @@ sb_lock(sb)
 			mutex_held = so->so_proto->pr_domain->dom_mtx;
 		if (so->so_usecount < 1)
 			panic("sb_lock: so=%x refcount=%d\n", so, so->so_usecount);
+
 		error = msleep((caddr_t)&sb->sb_flags, mutex_held,
 	    		(sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sblock", 0);
 		if (so->so_usecount < 1)
@@ -748,7 +743,7 @@ sbcheck(sb)
 int
 sbappendrecord(sb, m0)
 	register struct sockbuf *sb;
-	register struct mbuf *m0;
+	struct mbuf *m0;
 {
 	register struct mbuf *m;
 	int result = 0;
@@ -1472,13 +1467,12 @@ void
 sbunlock(struct sockbuf *sb, int keeplocked)
 {
 	struct socket *so = sb->sb_so;
-	int lr, lr_saved;
+	int lr_saved;
 	lck_mtx_t *mutex_held;
 
-#ifdef __ppc__
-	__asm__ volatile("mflr %0" : "=r" (lr));
-	lr_saved = lr;
-#endif
+
+	lr_saved = (unsigned int) __builtin_return_address(0);
+
 	sb->sb_flags &= ~SB_LOCK; 
 
 	if (so->so_proto->pr_getlock != NULL) 
@@ -1500,7 +1494,8 @@ sbunlock(struct sockbuf *sb, int keeplocked)
 		so->so_usecount--;
 		if (so->so_usecount < 0)
 			panic("sbunlock: unlock on exit so=%x lr=%x sb_flags=%x\n", so, so->so_usecount,lr_saved, sb->sb_flags);
-		so->reserved4= lr_saved;
+		so->unlock_lr[so->next_unlock_lr] = (void *)lr_saved;
+		so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX;
 		lck_mtx_unlock(mutex_held);
 	}
 }
diff --git a/bsd/kern/uipc_syscalls.c b/bsd/kern/uipc_syscalls.c
index cff67dd97..caed4ee61 100644
--- a/bsd/kern/uipc_syscalls.c
+++ b/bsd/kern/uipc_syscalls.c
@@ -1776,6 +1776,7 @@ sendfile(struct proc *p, struct sendfile_args *uap)
 	struct sf_hdtr hdtr;
 	off_t off, xfsize, sbytes = 0;
 	int error = 0, s;
+	kauth_cred_t safecred;
 
 	if (sf_bufs == NULL) {
 		/* Fail if initialization failed */
@@ -1930,8 +1931,10 @@ retry_lookup:
 			auio.uio_segflg = UIO_NOCOPY;
 			auio.uio_rw = UIO_READ;
 			uio_setresid(&auio, MAXBSIZE);
+			safecred = kauth_cred_proc_ref(p);
 			error = VOP_READ(vp, &auio, IO_VMIO | ((MAXBSIZE / bsize) << 16),
-			        p->p_ucred);
+			        safecred);
+			kauth_cred_unref(&safecred);
 			vm_page_flag_clear(pg, PG_ZERO);
 			vm_page_io_finish(pg);
 			if (error) {
diff --git a/bsd/kern/uipc_usrreq.c b/bsd/kern/uipc_usrreq.c
index 639ede855..262197e47 100644
--- a/bsd/kern/uipc_usrreq.c
+++ b/bsd/kern/uipc_usrreq.c
@@ -638,13 +638,17 @@ unp_bind(
 	char buf[SOCK_MAXADDRLEN];
 
 	context.vc_proc = p;
-	context.vc_ucred = p->p_ucred;	/* XXX kauth_cred_get() ??? proxy */
+	context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? proxy */
 
-	if (unp->unp_vnode != NULL)
+	if (unp->unp_vnode != NULL) {
+		kauth_cred_unref(&context.vc_ucred);
 		return (EINVAL);
+	}
 	namelen = soun->sun_len - offsetof(struct sockaddr_un, sun_path);
-	if (namelen <= 0)
+	if (namelen <= 0) {
+		kauth_cred_unref(&context.vc_ucred);
 		return EINVAL;
+	}
 	strncpy(buf, soun->sun_path, namelen);
 	buf[namelen] = 0;	/* null-terminate the string */
 	NDINIT(&nd, CREATE, FOLLOW | LOCKPARENT, UIO_SYSSPACE32,
@@ -652,6 +656,7 @@ unp_bind(
 /* SHOULD BE ABLE TO ADOPT EXISTING AND wakeup() ALA FIFO's */
 	error = namei(&nd);
 	if (error) {
+		kauth_cred_unref(&context.vc_ucred);
 		return (error);
 	}
 	dvp = nd.ni_dvp;
@@ -667,6 +672,7 @@ unp_bind(
 		vnode_put(dvp);
 		vnode_put(vp);
 
+		kauth_cred_unref(&context.vc_ucred);
 		return (EADDRINUSE);
 	}
 
@@ -686,6 +692,7 @@ unp_bind(
 	vnode_put(dvp);
 
 	if (error) {
+		kauth_cred_unref(&context.vc_ucred);
 		return (error);
 	}
 	vnode_ref(vp);	/* gain a longterm reference */
@@ -694,6 +701,7 @@ unp_bind(
 	unp->unp_addr = (struct sockaddr_un *)dup_sockaddr(nam, 1);
 	vnode_put(vp);		/* drop the iocount */
 
+	kauth_cred_unref(&context.vc_ucred);
 	return (0);
 }
 
@@ -713,18 +721,21 @@ unp_connect(
 	char buf[SOCK_MAXADDRLEN];
 
 	context.vc_proc = p;
-	context.vc_ucred = p->p_ucred;	/* XXX kauth_cred_get() ??? proxy */
+	context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? proxy */
 	so2 = so3 = NULL;
 
 	len = nam->sa_len - offsetof(struct sockaddr_un, sun_path);
-	if (len <= 0)
+	if (len <= 0) {
+		kauth_cred_unref(&context.vc_ucred);
 		return EINVAL;
+	}
 	strncpy(buf, soun->sun_path, len);
 	buf[len] = 0;
 
 	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE32, CAST_USER_ADDR_T(buf), &context);
 	error = namei(&nd);
 	if (error) {
+		kauth_cred_unref(&context.vc_ucred);
 		return (error);
 	}
 	nameidone(&nd);
@@ -782,7 +793,7 @@ unp_connect(
 		 * from its process structure at the time of connect()
 		 * (which is now).
 		 */
-		cru2x(p->p_ucred, &unp3->unp_peercred);
+		cru2x(context.vc_ucred, &unp3->unp_peercred);
 		unp3->unp_flags |= UNP_HAVEPC;
 		/*
 		 * The receiver's (server's) credentials are copied
@@ -808,6 +819,7 @@ bad:
 			so2->so_usecount--;	/* release count on socket */
 	
 	vnode_put(vp);
+	kauth_cred_unref(&context.vc_ucred);
 	return (error);
 }
 
@@ -1368,8 +1380,9 @@ unp_listen(
 	struct unpcb *unp,
 	struct proc *p)
 {
-
-	cru2x(p->p_ucred, &unp->unp_peercred);
+	kauth_cred_t safecred = kauth_cred_proc_ref(p);
+	cru2x(safecred, &unp->unp_peercred);
+	kauth_cred_unref(&safecred);
 	unp->unp_flags |= UNP_HAVEPCCACHED;
 	return (0);
 }
diff --git a/bsd/machine/_limits.h b/bsd/machine/_limits.h
index 9f8db80c6..a34289f11 100644
--- a/bsd/machine/_limits.h
+++ b/bsd/machine/_limits.h
@@ -30,7 +30,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/_limits.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/_limits.h"
 #else
 #error architecture not supported
diff --git a/bsd/machine/_types.h b/bsd/machine/_types.h
index 33c612f56..a84871288 100644
--- a/bsd/machine/_types.h
+++ b/bsd/machine/_types.h
@@ -30,7 +30,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/_types.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/_types.h"
 #else
 #error architecture not supported
diff --git a/bsd/machine/cons.h b/bsd/machine/cons.h
index 246984fbd..a5c4e92b4 100644
--- a/bsd/machine/cons.h
+++ b/bsd/machine/cons.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include <dev/ppc/cons.h>
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include <dev/i386/cons.h>
 #else
 #error architecture not supported
diff --git a/bsd/machine/disklabel.h b/bsd/machine/disklabel.h
index 323adb14c..fe5418f70 100644
--- a/bsd/machine/disklabel.h
+++ b/bsd/machine/disklabel.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/disklabel.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/disklabel.h"
 #else
 #error architecture not supported
diff --git a/bsd/machine/endian.h b/bsd/machine/endian.h
index 2f302313f..fd9555a4c 100644
--- a/bsd/machine/endian.h
+++ b/bsd/machine/endian.h
@@ -34,7 +34,7 @@
 
 #if defined (__ppc__) || defined(__ppc64__)
 #include "ppc/endian.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/endian.h"
 #else
 #error architecture not supported
diff --git a/bsd/machine/exec.h b/bsd/machine/exec.h
index dbdaf3053..6dd16564a 100644
--- a/bsd/machine/exec.h
+++ b/bsd/machine/exec.h
@@ -31,10 +31,21 @@
 #ifndef _BSD_MACHINE_EXEC_H_
 #define _BSD_MACHINE_EXEC_H_
 
+#include <sys/param.h>
+
+struct exec_archhandler {
+	char path[MAXPATHLEN];
+	uint32_t fsid;
+	long fileid;
+};
+
+extern struct exec_archhandler exec_archhandler_ppc;
+extern int set_archhandler(struct proc *p, int arch);
+extern int grade_binary(cpu_type_t exectype, cpu_subtype_t execsubtype);
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/exec.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/exec.h"
 #else
 #error architecture not supported
diff --git a/bsd/machine/param.h b/bsd/machine/param.h
index 735edd28b..f8a8a8781 100644
--- a/bsd/machine/param.h
+++ b/bsd/machine/param.h
@@ -34,7 +34,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/param.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/param.h"
 #else
 #error architecture not supported
diff --git a/bsd/machine/profile.h b/bsd/machine/profile.h
index eabd5200f..b7fe893b7 100644
--- a/bsd/machine/profile.h
+++ b/bsd/machine/profile.h
@@ -38,7 +38,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/profile.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/profile.h"
 #else
 #error architecture not supported
diff --git a/bsd/machine/psl.h b/bsd/machine/psl.h
index 20903d575..2157a0018 100644
--- a/bsd/machine/psl.h
+++ b/bsd/machine/psl.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/psl.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/psl.h"
 #else
 #error architecture not supported
diff --git a/bsd/machine/ptrace.h b/bsd/machine/ptrace.h
index 52317e3d3..f38087a08 100644
--- a/bsd/machine/ptrace.h
+++ b/bsd/machine/ptrace.h
@@ -34,7 +34,7 @@
 
 #if defined (__ppc__) || defined(__ppc64__)
 #include "ppc/ptrace.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/ptrace.h"
 #else
 #error architecture not supported
diff --git a/bsd/machine/reboot.h b/bsd/machine/reboot.h
index 0bbfc4140..db65fd37b 100644
--- a/bsd/machine/reboot.h
+++ b/bsd/machine/reboot.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/reboot.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/reboot.h"
 #else
 #error architecture not supported
diff --git a/bsd/machine/reg.h b/bsd/machine/reg.h
index d529b2336..0ec806cf1 100644
--- a/bsd/machine/reg.h
+++ b/bsd/machine/reg.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/reg.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/reg.h"
 #else
 #error architecture not supported
diff --git a/bsd/machine/setjmp.h b/bsd/machine/setjmp.h
index 32e20857f..52d826b7f 100644
--- a/bsd/machine/setjmp.h
+++ b/bsd/machine/setjmp.h
@@ -33,7 +33,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/setjmp.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/setjmp.h"
 #else
 #error architecture not supported
diff --git a/bsd/machine/signal.h b/bsd/machine/signal.h
index ab0e99bb5..9cf3719ff 100644
--- a/bsd/machine/signal.h
+++ b/bsd/machine/signal.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/signal.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/signal.h"
 #else
 #error architecture not supported
diff --git a/bsd/machine/types.h b/bsd/machine/types.h
index e1b09be15..21715f516 100644
--- a/bsd/machine/types.h
+++ b/bsd/machine/types.h
@@ -34,7 +34,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/types.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/types.h"
 #else
 #error architecture not supported
diff --git a/bsd/machine/ucontext.h b/bsd/machine/ucontext.h
index 24aefb80c..75a74437e 100644
--- a/bsd/machine/ucontext.h
+++ b/bsd/machine/ucontext.h
@@ -30,7 +30,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/ucontext.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/ucontext.h"
 #else
 #error architecture not supported
diff --git a/bsd/machine/vmparam.h b/bsd/machine/vmparam.h
index e73bd96de..3e7c455ee 100644
--- a/bsd/machine/vmparam.h
+++ b/bsd/machine/vmparam.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/vmparam.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "i386/vmparam.h"
 #else
 #error architecture not supported
diff --git a/bsd/man/man2/Makefile b/bsd/man/man2/Makefile
index 3473b0190..7312eba32 100644
--- a/bsd/man/man2/Makefile
+++ b/bsd/man/man2/Makefile
@@ -69,6 +69,7 @@ DATAFILES = \
 	gettimeofday.2	\
 	getuid.2	\
 	getxattr.2	\
+	i386_get_ldt.2	\
 	intro.2	\
 	ioctl.2	\
 	issetugid.2	\
@@ -174,7 +175,7 @@ DATAFILES = \
 
 # List of source/target hard link pairs for installed manual pages; source
 # names may be repeated
-MLINKS= kqueue.2 kevent.2
+MLINKS= kqueue.2 kevent.2 i386_get_ldt.2 i386_set_ldt.2
 
 INSTALL_MAN_LIST = ${DATAFILES}
 INSTALL_MAN_LINKS = ${MLINKS}
diff --git a/bsd/man/man2/i386_get_ldt.2 b/bsd/man/man2/i386_get_ldt.2
new file mode 100644
index 000000000..40ae07566
--- /dev/null
+++ b/bsd/man/man2/i386_get_ldt.2
@@ -0,0 +1,143 @@
+.\" Copyright (c) 1980, 1991 Regents of the University of California.
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\" 3. All advertising materials mentioning features or use of this software
+.\"    must display the following acknowledgement:
+.\"	This product includes software developed by the University of
+.\"	California, Berkeley and its contributors.
+.\" 4. Neither the name of the University nor the names of its contributors
+.\"    may be used to endorse or promote products derived from this software
+.\"    without specific prior written permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\"     from: @(#)fork.2	6.5 (Berkeley) 3/10/91
+.\" $FreeBSD: /repoman/r/ncvs/src/lib/libc/i386/sys/i386_get_ldt.2,v 1.21 2004/07/02 19:07:30 ru Exp $
+.\"
+.Dd September 20, 1993
+.Dt I386_GET_LDT 2
+.Os
+.Sh NAME
+.Nm i386_get_ldt ,
+.Nm i386_set_ldt
+.Nd manage i386 per-process Local Descriptor Table entries
+.Sh LIBRARY
+.Lb libc
+.Sh SYNOPSIS
+.In architecture/i386/table.h
+.In i386/user_ldt.h
+.Ft int
+.Fn i386_get_ldt "int start_sel" "union ldt_entry *descs" "int num_sels"
+.Ft int
+.Fn i386_set_ldt "int start_sel" "union ldt_entry *descs" "int num_sels"
+.Sh DESCRIPTION
+The
+.Fn i386_get_ldt
+system call
+will return the list of i386 descriptors that the process has in its
+LDT.
+The
+.Fn i386_set_ldt
+system call
+will set a list of i386 descriptors for the current process in its
+LDT.
+Both routines accept a starting selector number
+.Fa start_sel ,
+an array of memory that
+will contain the descriptors to be set or returned
+.Fa descs ,
+and the number of entries to set or return
+.Fa num_sels .
+.Pp
+The argument
+.Fa descs
+can be either code_desc_t, data_desc_t or call_gate_t and are defined in
+.In architecture/i386/desc.h .
+These structures are defined by the architecture
+as disjoint bit-fields, so care must be taken in constructing them.
+.Pp
+If
+.Fa start_sel
+is
+.Em LDT_AUTO_ALLOC ,
+.Fa num_sels
+is 1 and the descriptor pointed to by
+.Fa descs
+is legal, then
+.Fn i386_set_ldt
+will allocate a descriptor and return its
+selector number.
+.Pp
+If
+.Fa num_descs
+is 1,
+.Fa start_sels
+is valid, and
+.Fa descs
+is NULL, then
+.Fn i386_set_ldt
+will free that descriptor
+(making it available to be reallocated again later).
+.Pp
+If
+.Fa num_descs
+is 0,
+.Fa start_sels
+is 0 and
+.Fa descs
+is NULL then, as a special case,
+.Fn i386_set_ldt
+will free all descriptors.
+.Sh RETURN VALUES
+Upon successful completion,
+.Fn i386_get_ldt
+returns the number of descriptors currently in the LDT.
+The
+.Fn i386_set_ldt
+system call
+returns the first selector set.
+In the case when a descriptor is allocated by the kernel, its number will
+be returned.
+Otherwise, a value of -1 is returned and the global
+variable
+.Va errno
+is set to indicate the error.
+.Sh ERRORS
+The
+.Fn i386_get_ldt
+and
+.Fn i386_set_ldt
+system calls
+will fail if:
+.Bl -tag -width Er
+.It Bq Er EINVAL
+An inappropriate value was used for
+.Fa start_sel
+or
+.Fa num_sels .
+.It Bq Er EACCES
+The caller attempted to use a descriptor that would
+circumvent protection or cause a failure.
+.El
+.Sh SEE ALSO
+i386 Microprocessor Programmer's Reference Manual, Intel
+.Sh WARNING
+You can really hose your process using this.
diff --git a/bsd/man/man2/minherit.2 b/bsd/man/man2/minherit.2
index 5943caf37..01e4ffbbd 100644
--- a/bsd/man/man2/minherit.2
+++ b/bsd/man/man2/minherit.2
@@ -1,4 +1,4 @@
-.\"	$Id: minherit.2,v 1.2 2002/03/13 00:18:19 lindak Exp $
+.\"	$Id: minherit.2,v 1.2.2684.1 2005/06/24 01:47:10 lindak Exp $
 .\"
 .\" Copyright (c) 1991, 1993
 .\"	The Regents of the University of California.  All rights reserved.
diff --git a/bsd/miscfs/devfs/devfs_tree.c b/bsd/miscfs/devfs/devfs_tree.c
index 89875c92a..5f88c7662 100644
--- a/bsd/miscfs/devfs/devfs_tree.c
+++ b/bsd/miscfs/devfs/devfs_tree.c
@@ -133,14 +133,12 @@ static int devfs_ready = 0;
 int
 devfs_sinit(void)
 {
-        int error;
+    int error;
 
-        devfs_lck_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(devfs_lck_grp_attr);
+    devfs_lck_grp_attr = lck_grp_attr_alloc_init();
 	devfs_lck_grp = lck_grp_alloc_init("devfs_lock", devfs_lck_grp_attr);
 
 	devfs_lck_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(devfs_lck_attr);
 
 	lck_mtx_init(&devfs_mutex, devfs_lck_grp, devfs_lck_attr);
 
diff --git a/bsd/miscfs/devfs/devfs_vfsops.c b/bsd/miscfs/devfs/devfs_vfsops.c
index c97601787..d7db72496 100644
--- a/bsd/miscfs/devfs/devfs_vfsops.c
+++ b/bsd/miscfs/devfs/devfs_vfsops.c
@@ -77,6 +77,7 @@ static int devfs_statfs( struct mount *mp, struct vfsstatfs *sbp, vfs_context_t
 static int devfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context);
 
 static struct vfstable * devfs_vfsp = 0;
+extern int setup_kmem;
 
 
 /*-
@@ -98,10 +99,9 @@ devfs_init(struct vfsconf *vfsp)
 		    UID_ROOT, GID_WHEEL, 0622, "console");
     devfs_make_node(makedev(2, 0), DEVFS_CHAR, 
 		    UID_ROOT, GID_WHEEL, 0666, "tty");
-    devfs_make_node(makedev(3, 0), DEVFS_CHAR, 
-		    UID_ROOT, GID_KMEM, 0640, "mem");
-    devfs_make_node(makedev(3, 1), DEVFS_CHAR, 
-		    UID_ROOT, GID_KMEM, 0640, "kmem");
+    if (setup_kmem) {
+    	devfs_setup_kmem();
+    }
     devfs_make_node(makedev(3, 2), DEVFS_CHAR, 
 		    UID_ROOT, GID_WHEEL, 0666, "null");
     devfs_make_node(makedev(3, 3), DEVFS_CHAR, 
@@ -111,6 +111,16 @@ devfs_init(struct vfsconf *vfsp)
     return 0;
 }
 
+__private_extern__ void
+devfs_setup_kmem(void)
+{
+    	devfs_make_node(makedev(3, 0), DEVFS_CHAR, 
+		    UID_ROOT, GID_KMEM, 0640, "mem");
+    	devfs_make_node(makedev(3, 1), DEVFS_CHAR, 
+		    UID_ROOT, GID_KMEM, 0640, "kmem");
+}
+
+
 /*-
  *  mp	 - pointer to 'mount' structure
  *  path - addr in user space of mount point (ie /usr or whatever)
diff --git a/bsd/miscfs/devfs/devfs_vnops.c b/bsd/miscfs/devfs/devfs_vnops.c
index b58a1d1d6..adacffa55 100644
--- a/bsd/miscfs/devfs/devfs_vnops.c
+++ b/bsd/miscfs/devfs/devfs_vnops.c
@@ -1342,7 +1342,6 @@ static struct vnodeopv_entry_desc devfs_spec_vnodeop_entries[] = {
 	{ &vnop_pathconf_desc, (VOPFUNC)spec_pathconf },		/* pathconf */
 	{ &vnop_advlock_desc, (VOPFUNC)spec_advlock },		/* advlock */
 	{ &vnop_bwrite_desc, (VOPFUNC)vn_bwrite },
-	{ &vnop_devblocksize_desc, (VOPFUNC)spec_devblocksize },	/* devblocksize */
 	{ &vnop_pagein_desc, (VOPFUNC)err_pagein },		/* Pagein */
 	{ &vnop_pageout_desc, (VOPFUNC)err_pageout },		/* Pageout */
 	{ &vnop_copyfile_desc, (VOPFUNC)err_copyfile },		/* Copyfile */
diff --git a/bsd/miscfs/specfs/spec_vnops.c b/bsd/miscfs/specfs/spec_vnops.c
index ee2ea1d8f..1c88c6a22 100644
--- a/bsd/miscfs/specfs/spec_vnops.c
+++ b/bsd/miscfs/specfs/spec_vnops.c
@@ -129,7 +129,6 @@ struct vnodeopv_entry_desc spec_vnodeop_entries[] = {
 	{ &vnop_pathconf_desc, (VOPFUNC)spec_pathconf },		/* pathconf */
 	{ &vnop_advlock_desc, (VOPFUNC)err_advlock },		/* advlock */
 	{ &vnop_bwrite_desc, (VOPFUNC)spec_bwrite },		/* bwrite */
-	{ &vnop_devblocksize_desc, (VOPFUNC)spec_devblocksize }, /* devblocksize */
 	{ &vnop_pagein_desc, (VOPFUNC)err_pagein },		/* Pagein */
 	{ &vnop_pageout_desc, (VOPFUNC)err_pageout },		/* Pageout */
         { &vnop_copyfile_desc, (VOPFUNC)err_copyfile },		/* Copyfile */
@@ -860,17 +859,6 @@ spec_pathconf(ap)
 	/* NOTREACHED */
 }
 
-int
-spec_devblocksize(ap)
-        struct vnop_devblocksize_args /* {
-	        struct vnode *a_vp;
-	        int *a_retval;
-        } */ *ap;
-{
-        *ap->a_retval = (ap->a_vp->v_specsize);
-        return (0);
-}
-
 /*
  * Special device failed operation
  */
diff --git a/bsd/miscfs/specfs/specdev.h b/bsd/miscfs/specfs/specdev.h
index 9cab49b47..788969708 100644
--- a/bsd/miscfs/specfs/specdev.h
+++ b/bsd/miscfs/specfs/specdev.h
@@ -160,7 +160,6 @@ int	spec_pathconf (struct vnop_pathconf_args *);
 #define spec_valloc (int (*) (struct  vnop_access_args *))err_valloc
 #define spec_vfree (int (*) (struct  vnop_access_args *))err_vfree
 #define spec_bwrite (int (*) (struct  vnop_bwrite_args *))nop_bwrite
-int     spec_devblocksize (struct vnop_devblocksize_args *);
 int spec_blktooff (struct  vnop_blktooff_args *);
 int spec_offtoblk (struct  vnop_offtoblk_args *);
 int spec_blockmap (struct  vnop_blockmap_args *);
diff --git a/bsd/miscfs/synthfs/synthfs_vfsops.c b/bsd/miscfs/synthfs/synthfs_vfsops.c
index 48ae43a7d..4a2865231 100644
--- a/bsd/miscfs/synthfs/synthfs_vfsops.c
+++ b/bsd/miscfs/synthfs/synthfs_vfsops.c
@@ -406,12 +406,13 @@ vn_mkdir(struct proc *p, char *path, int mode)
 	int error;
 
 	context.vc_proc = p;
-	context.vc_ucred = proc_ucred(p);	/* XXX kauth_cred_get() ??? proxy */
+	context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? proxy */
 
 	NDINIT(&nd, CREATE, LOCKPARENT, UIO_SYSSPACE32, CAST_USER_ADDR_T(path), &context);
 	error = namei(&nd);
 	if (error) {
 		DBG_VOP(("vn_mkdir: error from namei, error = %d.\n", error));
+		kauth_cred_unref(&context.vc_ucred);
 		return (error);
 	};
 	vp = nd.ni_vp;
@@ -433,6 +434,7 @@ vn_mkdir(struct proc *p, char *path, int mode)
 	        vnode_put(nd.ni_vp);
 	nameidone(&nd);
 
+	kauth_cred_unref(&context.vc_ucred);
 	return (error);
 }
 
@@ -446,10 +448,13 @@ vn_symlink(struct proc *p, char *path, char *link) {
 	int error;
 
 	context.vc_proc = p;
-	context.vc_ucred = proc_ucred(p);	/* XXX kauth_cred_get() ??? proxy */
+	context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? proxy */
 
 	NDINIT(&nd, CREATE, LOCKPARENT, UIO_SYSSPACE32, CAST_USER_ADDR_T(link), &context);
-	if ((error = namei(&nd))) return error;
+	if ((error = namei(&nd))) {
+		kauth_cred_unref(&context.vc_ucred);
+		return error;
+	}
 
 	if (nd.ni_vp == NULL) {
 		VATTR_INIT(&va);
@@ -465,6 +470,7 @@ vn_symlink(struct proc *p, char *path, char *link) {
 		vnode_put(nd.ni_vp);
 	nameidone(&nd);
 
+	kauth_cred_unref(&context.vc_ucred);
 	return (error);
 }
 
diff --git a/bsd/miscfs/synthfs/synthfs_vnops.c b/bsd/miscfs/synthfs/synthfs_vnops.c
index 56f4d6dd3..c52310b4e 100644
--- a/bsd/miscfs/synthfs/synthfs_vnops.c
+++ b/bsd/miscfs/synthfs/synthfs_vnops.c
@@ -111,7 +111,6 @@ struct vnodeopv_entry_desc synthfs_vnodeop_entries[] = {
     {&vnop_allocate_desc, (VOPFUNC)err_allocate},		/* allocate		- not supported  */
 	{&vnop_pagein_desc, (VOPFUNC)err_pagein},		/* pagein		- not supported  */
 	{&vnop_pageout_desc, (VOPFUNC)err_pageout},		/* pageout		- not supported  */
-	{&vnop_devblocksize_desc, (VOPFUNC)err_devblocksize},	/* devblocksize - not supported  */
 	{&vnop_searchfs_desc, (VOPFUNC)err_searchfs},		/* searchfs		- not supported */
 	{&vnop_copyfile_desc, (VOPFUNC)err_copyfile},		/* copyfile - not supported */
  	{ &vnop_blktooff_desc, (VOPFUNC)err_blktooff },		/* blktooff not supported */
diff --git a/bsd/miscfs/union/union.h b/bsd/miscfs/union/union.h
index c2ad169ce..e29c66876 100644
--- a/bsd/miscfs/union/union.h
+++ b/bsd/miscfs/union/union.h
@@ -96,19 +96,12 @@ struct union_mount {
  */
 /* LP64todo - should this move? */
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_union_args {
 	user_addr_t	target;		/* Target of loopback  */
 	int			mntflags;	/* Options on the mount */
+	char		_pad[4];
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 /*
  * DEFDIRMODE is the mode bits used to create a shadow directory.
  */
diff --git a/bsd/miscfs/union/union_subr.c b/bsd/miscfs/union/union_subr.c
index 80b3a0386..7eff77ebc 100644
--- a/bsd/miscfs/union/union_subr.c
+++ b/bsd/miscfs/union/union_subr.c
@@ -882,7 +882,6 @@ union_vn_create(vpp, un, p)
 	struct proc *p;
 {
 	struct vnode *vp;
-	kauth_cred_t cred = p->p_ucred;
 	struct vnode_attr vat;
 	struct vnode_attr *vap = &vat;
 	struct vfs_context context;
@@ -895,7 +894,7 @@ union_vn_create(vpp, un, p)
 	*vpp = NULLVP;
 
 	context.vc_proc = p;
-	context.vc_ucred = p->p_ucred;
+	context.vc_ucred = kauth_cred_proc_ref(p);
 
 	/*
 	 * Build a new componentname structure (for the same
@@ -919,12 +918,15 @@ union_vn_create(vpp, un, p)
 	cn.cn_consume = 0;
 
 	vnode_get(un->un_dirvp);
-	if (error = relookup(un->un_dirvp, &vp, &cn))
+	if (error = relookup(un->un_dirvp, &vp, &cn)) {
+		kauth_cred_unref(&context.vc_ucred);
 		return (error);
+	}
 	vnode_put(un->un_dirvp);
 
 	if (vp) {
 	        vnode_put(un->un_dirvp);
+		kauth_cred_unref(&context.vc_ucred);
 		vnode_put(vp);
 		return (EEXIST);
 	}
@@ -946,11 +948,14 @@ union_vn_create(vpp, un, p)
 	VATTR_SET(vap, va_type, VREG);
 	VATTR_SET(vap, va_mode, cmode);
 
-	if (error = vn_create(un->un_dirvp, &vp, &cn, vap, 0, &context))
+	if (error = vn_create(un->un_dirvp, &vp, &cn, vap, 0, &context)) {
+		kauth_cred_unref(&context.vc_ucred);
 		return (error);
+	}
 
 	if (error = VNOP_OPEN(vp, fmode, &context)) {
 		vnode_put(vp);
+		kauth_cred_unref(&context.vc_ucred);
 		return (error);
 	}
 
@@ -959,6 +964,7 @@ union_vn_create(vpp, un, p)
 		panic("union: v_writecount");
 	vnode_unlock(vp);
 	*vpp = vp;
+	kauth_cred_unref(&context.vc_ucred);
 	return (0);
 }
 
diff --git a/bsd/miscfs/union/union_vfsops.c b/bsd/miscfs/union/union_vfsops.c
index 9846838f1..6d93bbba8 100644
--- a/bsd/miscfs/union/union_vfsops.c
+++ b/bsd/miscfs/union/union_vfsops.c
@@ -268,8 +268,8 @@ union_mount(mount_t mp, __unused vnode_t devvp, user_addr_t data, vfs_context_t
 bad:
 	if (um)
 		_FREE(um, M_UFSMNT);
-	if (cred != NOCRED)
-		kauth_cred_rele(cred);
+	if (IS_VALID_CRED(cred))
+		kauth_cred_unref(&cred);
 	if (upperrootvp)
 		vnode_put(upperrootvp);
 	if (lowerrootvp)
@@ -360,10 +360,8 @@ union_unmount(mount_t mp, int mntflags, __unused vfs_context_t context)
 	if (um->um_lowervp)
 		vnode_put(um->um_lowervp);
 	vnode_put(um->um_uppervp);
-	cred = um->um_cred;
-	if (cred != NOCRED) {
-		um->um_cred = NOCRED;
-		kauth_cred_rele(cred);
+	if (IS_VALID_CRED(um->um_cred)) {
+		kauth_cred_unref(&um->um_cred);
 	}
 	/*
 	 * Release reference on underlying root vnode
diff --git a/bsd/miscfs/volfs/volfs_vnops.c b/bsd/miscfs/volfs/volfs_vnops.c
index 2a8f29ebd..d6988d0b9 100644
--- a/bsd/miscfs/volfs/volfs_vnops.c
+++ b/bsd/miscfs/volfs/volfs_vnops.c
@@ -147,7 +147,6 @@ struct vnodeopv_entry_desc volfs_vnodeop_entries[] = {
     {&vnop_allocate_desc, (VOPFUNC)err_allocate},	/* allocate */
 	{&vnop_pagein_desc, (VOPFUNC)err_pagein},	/* pagein */
 	{&vnop_pageout_desc, (VOPFUNC)err_pageout},	/* pageout */
-	{&vnop_devblocksize_desc, (VOPFUNC)err_devblocksize},	/* devblocksize */
 	{&vnop_searchfs_desc, (VOPFUNC)err_searchfs},	/* searchfs */
 	{&vnop_copyfile_desc, (VOPFUNC)err_copyfile },	/* Copyfile */
 	{&vnop_blktooff_desc, (VOPFUNC)err_blktooff},	/* blktooff */
diff --git a/bsd/net/bpf.c b/bsd/net/bpf.c
index f9f729bd2..9a1af2a9b 100644
--- a/bsd/net/bpf.c
+++ b/bsd/net/bpf.c
@@ -1617,12 +1617,10 @@ bpf_init(__unused void *unused)
 		bpf_devsw_installed = 1;
 
         bpf_mlock_grp_attr = lck_grp_attr_alloc_init();
-        lck_grp_attr_setdefault(bpf_mlock_grp_attr);
 
         bpf_mlock_grp = lck_grp_alloc_init("bpf", bpf_mlock_grp_attr);
 
         bpf_mlock_attr = lck_attr_alloc_init();
-        lck_attr_setdefault(bpf_mlock_attr);
 
         bpf_mlock = lck_mtx_alloc_init(bpf_mlock_grp, bpf_mlock_attr);
 
diff --git a/bsd/net/bpf.h b/bsd/net/bpf.h
index 6bf91d84d..fc68f2f56 100644
--- a/bsd/net/bpf.h
+++ b/bsd/net/bpf.h
@@ -109,19 +109,11 @@ struct bpf_program {
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with bpf_program
  */
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_bpf_program {
 	u_int		bf_len;
-	user_addr_t	bf_insns;
+	user_addr_t	bf_insns __attribute__((aligned(8)));
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif // KERNEL
 
 /*
diff --git a/bsd/net/dlil.c b/bsd/net/dlil.c
index 4a27598f6..14d1ca03c 100644
--- a/bsd/net/dlil.c
+++ b/bsd/net/dlil.c
@@ -557,7 +557,6 @@ dlil_init(void)
 	
 	/* Setup the lock groups we will use */
 	grp_attributes = lck_grp_attr_alloc_init();
-	lck_grp_attr_setdefault(grp_attributes);
 
 	dlil_lock_group = lck_grp_alloc_init("dlil internal locks", grp_attributes);
 #if IFNET_RW_LOCK
@@ -572,10 +571,8 @@ dlil_init(void)
 	
 	/* Setup the lock attributes we will use */
 	lck_attributes = lck_attr_alloc_init();
-	lck_attr_setdefault(lck_attributes);
 	
 	ifnet_lock_attr = lck_attr_alloc_init();
-	lck_attr_setdefault(ifnet_lock_attr);
 	
 	dlil_input_lock = lck_spin_alloc_init(input_lock_grp, lck_attributes);
 	input_lock_grp = 0;
@@ -2378,9 +2375,8 @@ dlil_if_attach_with_address(
 			DLIL_PRINTF("dlil_if_attach -- init_if failed with %d\n", stat);
 		}
 	}
-
-	ifnet_lock_done(ifp);
     
+	ifnet_lock_done(ifp);
     dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_IF_ATTACHED, 0, 0);
 
     return 0;
diff --git a/bsd/net/ether_at_pr_module.c b/bsd/net/ether_at_pr_module.c
index 8b1d3d31e..911e0d4b5 100644
--- a/bsd/net/ether_at_pr_module.c
+++ b/bsd/net/ether_at_pr_module.c
@@ -191,7 +191,7 @@ ether_pre_output(
 	eh = (struct ether_header *)dst_netaddr->sa_data;
 	bcopy((caddr_t)eh->ether_dhost, (caddr_t)edst, 6);
 		
-	*(u_short *)type = m->m_pkthdr.len;
+	*(u_short *)type = htons(m->m_pkthdr.len);
     }
     break;
 
diff --git a/bsd/net/if_bond.c b/bsd/net/if_bond.c
index d5b5e3abd..a4b4531cf 100644
--- a/bsd/net/if_bond.c
+++ b/bsd/net/if_bond.c
@@ -104,8 +104,6 @@ my_lck_grp_alloc_init(const char * grp_name)
     lck_grp_attr_t *	grp_attrs;
     
     grp_attrs = lck_grp_attr_alloc_init();
-    lck_grp_attr_setdefault(grp_attrs);
-    lck_grp_attr_setdefault(grp_attrs);
     grp = lck_grp_alloc_init(grp_name, grp_attrs);
     lck_grp_attr_free(grp_attrs);
     return (grp);
@@ -118,7 +116,6 @@ my_lck_mtx_alloc_init(lck_grp_t * lck_grp)
     lck_mtx_t *		lck_mtx;
 
     lck_attrs = lck_attr_alloc_init();
-    lck_attr_setdefault(lck_attrs);
     lck_mtx = lck_mtx_alloc_init(lck_grp, lck_attrs);
     lck_attr_free(lck_attrs);
     return (lck_mtx);
diff --git a/bsd/net/if_var.h b/bsd/net/if_var.h
index c58305470..f8b5a0396 100644
--- a/bsd/net/if_var.h
+++ b/bsd/net/if_var.h
@@ -91,7 +91,7 @@
 #define APPLE_IF_FAM_STF       12
 #define APPLE_IF_FAM_FIREWIRE  13
 #define APPLE_IF_FAM_BOND      14
-#endif __APPLE__
+#endif /* __APPLE__ */
 
 /*
  * 72 was chosen below because it is the size of a TCP/IP
diff --git a/bsd/net/if_vlan.c b/bsd/net/if_vlan.c
index 622df1c0c..957865e65 100644
--- a/bsd/net/if_vlan.c
+++ b/bsd/net/if_vlan.c
@@ -119,7 +119,6 @@ my_lck_grp_alloc_init(const char * grp_name)
     lck_grp_attr_t *	grp_attrs;
     
     grp_attrs = lck_grp_attr_alloc_init();
-    lck_grp_attr_setdefault(grp_attrs);
     grp = lck_grp_alloc_init(grp_name, grp_attrs);
     lck_grp_attr_free(grp_attrs);
     return (grp);
@@ -132,7 +131,6 @@ my_lck_mtx_alloc_init(lck_grp_t * lck_grp)
     lck_mtx_t *		lck_mtx;
 
     lck_attrs = lck_attr_alloc_init();
-    lck_attr_setdefault(lck_attrs);
     lck_mtx = lck_mtx_alloc_init(lck_grp, lck_attrs);
     lck_attr_free(lck_attrs);
     return (lck_mtx);
diff --git a/bsd/net/kext_net.h b/bsd/net/kext_net.h
index a60e400a9..7df166cda 100644
--- a/bsd/net/kext_net.h
+++ b/bsd/net/kext_net.h
@@ -104,9 +104,7 @@ int		sflt_attach_private(struct socket *so, struct socket_filter *filter, sflt_h
  *  the 'where' NKE.  If the latter is NULL, the flags indicate "first"
  *  or "last"
  */
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 struct so_nke
 {	unsigned int nke_handle;
@@ -115,9 +113,7 @@ struct so_nke
 	unsigned long reserved[4];	/* for future use */
 };
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 #endif /* NET_KEXT_NET_H */
 
diff --git a/bsd/net/kpi_interface.c b/bsd/net/kpi_interface.c
index bd5323801..fd4c6e22b 100644
--- a/bsd/net/kpi_interface.c
+++ b/bsd/net/kpi_interface.c
@@ -1030,7 +1030,10 @@ errno_t ifnet_get_multicast_list(ifnet_t interface, ifmultiaddr_t **addresses)
 	}
 	
 	MALLOC(*addresses, ifmultiaddr_t*, sizeof(ifmultiaddr_t) * (cmax + 1), M_TEMP, M_NOWAIT);
-	if (*addresses == NULL) return ENOMEM;
+	if (*addresses == NULL) {
+		if (lock) ifnet_lock_done(interface);
+		return ENOMEM;
+	}
 	
 	LIST_FOREACH(addr, &interface->if_multiaddrs, ifma_link)
 	{
diff --git a/bsd/net/kpi_protocol.c b/bsd/net/kpi_protocol.c
index 5dda02574..ff81434be 100644
--- a/bsd/net/kpi_protocol.c
+++ b/bsd/net/kpi_protocol.c
@@ -95,11 +95,9 @@ proto_kpi_init(void)
 	
 	/* Allocate a mtx lock */
 	grp_attrib = lck_grp_attr_alloc_init();
-	lck_grp_attr_setdefault(grp_attrib);
 	lck_group = lck_grp_alloc_init("protocol kpi", grp_attrib);
 	lck_grp_attr_free(grp_attrib);
 	lck_attrib = lck_attr_alloc_init();
-	lck_attr_setdefault(lck_attrib);
 	proto_input_lock = lck_mtx_alloc_init(lck_group, lck_attrib);
 	lck_grp_free(lck_group);
 	lck_attr_free(lck_attrib);
diff --git a/bsd/net/ndrv.c b/bsd/net/ndrv.c
index f2f42839f..60e9be31c 100644
--- a/bsd/net/ndrv.c
+++ b/bsd/net/ndrv.c
@@ -295,7 +295,10 @@ ndrv_event(struct ifnet *ifp, struct kev_msg *event)
 		event->kev_class == KEV_NETWORK_CLASS &&
 		event->kev_subclass == KEV_DL_SUBCLASS &&
 		event->event_code == KEV_DL_IF_DETACHING) {
+		lck_mtx_assert(ndrvdomain.dom_mtx, LCK_MTX_ASSERT_NOTOWNED);
+		lck_mtx_lock(ndrvdomain.dom_mtx);
 		ndrv_handle_ifp_detach(ifp->if_family, ifp->if_unit);
+		lck_mtx_unlock(ndrvdomain.dom_mtx);
 	}
 }
 
@@ -364,7 +367,9 @@ ndrv_bind(struct socket *so, struct sockaddr *nam, __unused struct proc *p)
 		ndrv_proto.event = ndrv_event;
 		
 		/* We aren't worried about double attaching, that should just return an error */
+		socket_unlock(so, 0);
 		result = dlil_attach_protocol(&ndrv_proto);
+		socket_lock(so, 0);
 		if (result && result != EEXIST) {
 			return result;
 		}
@@ -530,21 +535,24 @@ ndrv_do_detach(struct ndrv_cb *np)
     struct ndrv_cb*	cur_np = NULL;
     struct socket *so = np->nd_socket;
     int error = 0;
+    struct ifnet * ifp;
 
 #if NDRV_DEBUG
 	kprintf("NDRV detach: %x, %x\n", so, np);
 #endif
     ndrv_remove_all_multicast(np);
-    
-    if (np->nd_if) {
-		if (np->nd_proto_family != PF_NDRV &&
-			np->nd_proto_family != 0) {
-			dlil_detach_protocol(np->nd_if, np->nd_proto_family);
+    ifp = np->nd_if;
+    /* Remove from the linked list of control blocks */
+    TAILQ_REMOVE(&ndrvl, np, nd_next);
+    if (ifp != NULL) {
+		u_long proto_family = np->nd_proto_family;
+
+		if (proto_family != PF_NDRV && proto_family != 0) {
+			socket_unlock(so, 0);
+			dlil_detach_protocol(ifp, proto_family);
+			socket_lock(so, 0);
 		}
 		
-		/* Remove from the linked list of control blocks */
-		TAILQ_REMOVE(&ndrvl, np, nd_next);
-		
 		/* Check if this is the last socket attached to this interface */
 		TAILQ_FOREACH(cur_np, &ndrvl, nd_next) {
 			if (cur_np->nd_family == np->nd_family &&
@@ -555,13 +563,16 @@ ndrv_do_detach(struct ndrv_cb *np)
 		
 		/* If there are no other interfaces, detach PF_NDRV from the interface */
 		if (cur_np == NULL) {
-			dlil_detach_protocol(np->nd_if, PF_NDRV);
+			socket_unlock(so, 0);
+			dlil_detach_protocol(ifp, PF_NDRV);
+			socket_lock(so, 0);
 		}
-	} else {
-		/* Remove from the linked list of control blocks */
-		TAILQ_REMOVE(&ndrvl, np, nd_next);
 	}
     
+    	if (np->nd_laddr != NULL) {
+		FREE((caddr_t)np->nd_laddr, M_IFADDR);
+		np->nd_laddr = NULL;
+	}
 	FREE((caddr_t)np, M_PCB);
 	so->so_pcb = 0;
 	so->so_flags |= SOF_PCBCLEARING;
@@ -572,6 +583,7 @@ ndrv_do_detach(struct ndrv_cb *np)
 static int
 ndrv_do_disconnect(struct ndrv_cb *np)
 {
+	struct socket * so = np->nd_socket;
 #if NDRV_DEBUG
 	kprintf("NDRV disconnect: %x\n", np);
 #endif
@@ -580,9 +592,9 @@ ndrv_do_disconnect(struct ndrv_cb *np)
         FREE(np->nd_faddr, M_IFADDR);
 		np->nd_faddr = 0;
 	}
-	if (np->nd_socket->so_state & SS_NOFDREF)
+	if (so->so_state & SS_NOFDREF)
 		ndrv_do_detach(np);
-	soisdisconnected(np->nd_socket);
+	soisdisconnected(so);
 	return(0);
 }
 
@@ -651,6 +663,7 @@ ndrv_setspec(struct ndrv_cb *np, struct sockopt *sopt)
     struct dlil_demux_desc*		dlilDemux = NULL;
     struct ndrv_demux_desc*		ndrvDemux = NULL;
     int							error = 0;
+    struct socket *so = np->nd_socket; 
     
     /* Sanity checking */
     if (np->nd_proto_family != PF_NDRV)
@@ -720,7 +733,9 @@ ndrv_setspec(struct ndrv_cb *np, struct sockopt *sopt)
     if (error == 0)
     {
         /* We've got all our ducks lined up...lets attach! */
+	socket_unlock(so, 0);
         error = dlil_attach_protocol(&dlilSpec);
+	socket_lock(so, 0);
         if (error == 0)
         	np->nd_proto_family = dlilSpec.protocol_family;
     }
diff --git a/bsd/net/raw_usrreq.c b/bsd/net/raw_usrreq.c
index d10b1ff94..236033d47 100644
--- a/bsd/net/raw_usrreq.c
+++ b/bsd/net/raw_usrreq.c
@@ -85,14 +85,10 @@ raw_init()
 {
 	raw_mtx_grp_attr = lck_grp_attr_alloc_init();
 
-	lck_grp_attr_setdefault(raw_mtx_grp_attr);
-
 	raw_mtx_grp = lck_grp_alloc_init("rawcb", raw_mtx_grp_attr);
 
 	raw_mtx_attr = lck_attr_alloc_init();
 
-	lck_attr_setdefault(raw_mtx_attr);
-
 	if ((raw_mtx = lck_mtx_alloc_init(raw_mtx_grp, raw_mtx_attr)) == NULL) {
 		printf("raw_init: can't alloc raw_mtx\n");
 		return;
diff --git a/bsd/net/route.c b/bsd/net/route.c
index c0a136aaf..7bb3d89be 100644
--- a/bsd/net/route.c
+++ b/bsd/net/route.c
@@ -117,14 +117,10 @@ route_init()
 {
 	rt_mtx_grp_attr = lck_grp_attr_alloc_init();
 
-	lck_grp_attr_setdefault(rt_mtx_grp_attr);
-
 	rt_mtx_grp = lck_grp_alloc_init("route", rt_mtx_grp_attr);
 
 	rt_mtx_attr = lck_attr_alloc_init();
 
-	lck_attr_setdefault(rt_mtx_attr);
-
 	if ((rt_mtx = lck_mtx_alloc_init(rt_mtx_grp, rt_mtx_attr)) == NULL) {
 		printf("route_init: can't alloc rt_mtx\n");
 		return;
diff --git a/bsd/net/zlib.c b/bsd/net/zlib.c
index 899fdf1a0..8c2f6dd9e 100644
--- a/bsd/net/zlib.c
+++ b/bsd/net/zlib.c
@@ -55,7 +55,7 @@
    subject to change. Applications should only use zlib.h.
  */
 
-/* @(#) $Id: zlib.c,v 1.10 2004/07/29 19:17:20 lindak Exp $ */
+/* @(#) $Id: zlib.c,v 1.10.874.1 2005/06/24 01:47:11 lindak Exp $ */
 
 #ifndef _Z_UTIL_H
 #define _Z_UTIL_H
@@ -301,7 +301,7 @@ void   zcfree  OF((voidpf opaque, voidpf ptr));
    subject to change. Applications should only use zlib.h.
  */
 
-/* @(#) $Id: zlib.c,v 1.10 2004/07/29 19:17:20 lindak Exp $ */
+/* @(#) $Id: zlib.c,v 1.10.874.1 2005/06/24 01:47:11 lindak Exp $ */
 
 #ifndef _DEFLATE_H
 #define _DEFLATE_H
@@ -661,7 +661,7 @@ void _tr_stored_block OF((deflate_state *s, charf *buf, ulg stored_len,
  *
  */
 
-/* @(#) $Id: zlib.c,v 1.10 2004/07/29 19:17:20 lindak Exp $ */
+/* @(#) $Id: zlib.c,v 1.10.874.1 2005/06/24 01:47:11 lindak Exp $ */
 
 /* #include "deflate.h" */
 
@@ -2003,7 +2003,7 @@ local block_state deflate_slow(s, flush)
  *          Addison-Wesley, 1983. ISBN 0-201-06672-6.
  */
 
-/* @(#) $Id: zlib.c,v 1.10 2004/07/29 19:17:20 lindak Exp $ */
+/* @(#) $Id: zlib.c,v 1.10.874.1 2005/06/24 01:47:11 lindak Exp $ */
 
 /* #define GEN_TREES_H */
 
@@ -5548,7 +5548,7 @@ z_streamp z;
  * For conditions of distribution and use, see copyright notice in zlib.h 
  */
 
-/* @(#) $Id: zlib.c,v 1.10 2004/07/29 19:17:20 lindak Exp $ */
+/* @(#) $Id: zlib.c,v 1.10.874.1 2005/06/24 01:47:11 lindak Exp $ */
 
 /* #include "zutil.h" */
 
@@ -5778,7 +5778,7 @@ void  zcfree (opaque, ptr)
  * For conditions of distribution and use, see copyright notice in zlib.h 
  */
 
-/* @(#) $Id: zlib.c,v 1.10 2004/07/29 19:17:20 lindak Exp $ */
+/* @(#) $Id: zlib.c,v 1.10.874.1 2005/06/24 01:47:11 lindak Exp $ */
 
 /* #include "zlib.h" */
 
diff --git a/bsd/netat/adsp.c b/bsd/netat/adsp.c
index 6d2834368..c8fa72e60 100644
--- a/bsd/netat/adsp.c
+++ b/bsd/netat/adsp.c
@@ -56,7 +56,6 @@ struct adsp_debug adsp_dtable[1025];
 int ad_entry = 0;
 #endif
 
-extern atlock_t adspgen_lock;
 
 adspAllocateCCB(gref)
     register gref_t *gref;	/* READ queue */
@@ -75,9 +74,6 @@ adspAllocateCCB(gref)
     sp->pid = gref->pid; /* save the caller process pointer */
     sp->gref = gref;		/* save a back pointer to the WRITE queue */
     sp->sp_mp = ccb_mp;		/* and its message block */
-    ATLOCKINIT(sp->lock);
-    ATLOCKINIT(sp->lockClose);
-    ATLOCKINIT(sp->lockRemove);
     return 1;
 }
 
@@ -85,19 +81,14 @@ adspRelease(gref)
     register gref_t *gref;	/* READ queue */
 {
     register CCBPtr sp;
-    int s, l;
 
-    ATDISABLE(l, adspgen_lock);
     if (gref->info) {
 	sp = (CCBPtr)gbuf_rptr(((gbuf_t *)gref->info));
-	ATDISABLE(s, sp->lock);
-	ATENABLE(s, adspgen_lock);
 				/* Tells completion routine of close */
 				/* packet to remove us. */
 
 	if (sp->state == sPassive || sp->state == sClosed || 
 	    sp->state == sOpening || sp->state == sListening) {
-	    ATENABLE(l, sp->lock);
 	    if (sp->state == sListening)
 		CompleteQueue(&sp->opb, errAborted);
 	    sp->removing = 1;	/* Prevent allowing another dspClose. */
@@ -106,7 +97,6 @@ adspRelease(gref)
 	} else {			/* sClosing & sOpen */
 	    sp->state = sClosing;
 	}
-	ATENABLE(l, sp->lock);
 
 	if (CheckOkToClose(sp)) { /* going to close */
 	    sp->sendCtl = B_CTL_CLOSE; /* Send close advice */
@@ -116,13 +106,10 @@ adspRelease(gref)
 		    sp->sendCtl = B_CTL_CLOSE; /* Setup to send close advice */
 	}
 	CheckSend(sp);		/* and force out the close */
-	ATDISABLE(s, sp->lock);
 	    sp->removing = 1;	/* Prevent allowing another dspClose. */
 	    sp->state = sClosed;
-	ATENABLE(s, sp->lock);
 	    DoClose(sp, errAborted, 0);  /* to closed and remove CCB */
-    } else
-	ATENABLE(l, adspgen_lock);
+    } 
 }
 
 
@@ -365,11 +352,11 @@ adsp_sendddp(sp, mp, length, dstnetaddr, ddptype)
    /* Set up the DDP header */
 
    ddp = (DDPX_FRAME *) gbuf_rptr(mp);
-   UAS_ASSIGN(ddp->ddpx_length, (length + DDPL_FRAME_LEN));
+   UAS_ASSIGN_HTON(ddp->ddpx_length, (length + DDPL_FRAME_LEN));
    UAS_ASSIGN(ddp->ddpx_cksm, 0);
    if (sp) {
 	if (sp->useCheckSum)
-	   UAS_ASSIGN(ddp->ddpx_cksm, 1);
+	   UAS_ASSIGN_HTON(ddp->ddpx_cksm, 1);
    }
 
    NET_ASSIGN(ddp->ddpx_dnet, dstnetaddr->a.net);
diff --git a/bsd/netat/adsp_CLDeny.c b/bsd/netat/adsp_CLDeny.c
index f7c045ef4..1de474dc8 100644
--- a/bsd/netat/adsp_CLDeny.c
+++ b/bsd/netat/adsp_CLDeny.c
@@ -88,8 +88,8 @@ int adspCLDeny(struct adspcmd *pb, CCBPtr sp)
     adspp->descriptor = ADSP_CONTROL_BIT | ADSP_CTL_ODENY;
     adspop = (ADSP_OPEN_DATAPtr)gbuf_wptr(mp);
     gbuf_winc(mp,ADSP_OPEN_FRAME_LEN);
-    UAS_ASSIGN(adspop->dstCID, pb->u.openParams.remoteCID);
-    UAS_ASSIGN(adspop->version, 0x100);
+    UAS_ASSIGN_HTON(adspop->dstCID, pb->u.openParams.remoteCID);
+    UAS_ASSIGN_HTON(adspop->version, 0x100);
     adsp_sendddp(sp, mp, 
 		 DDPL_FRAME_LEN + ADSP_FRAME_LEN + ADSP_OPEN_FRAME_LEN, 
 		 &pb->u.openParams.remoteAddress, DDP_ADSP);
diff --git a/bsd/netat/adsp_CLListen.c b/bsd/netat/adsp_CLListen.c
index 69dbe7229..949b70e81 100644
--- a/bsd/netat/adsp_CLListen.c
+++ b/bsd/netat/adsp_CLListen.c
@@ -75,7 +75,6 @@ int adspCLListen(sp, pb)	/* (DSPPBPtr pb) */
 {
     register struct adspcmd *clpb;
     gbuf_t *mp;
-    int s;
 
     if (sp == 0) {
 	pb->ioResult = errRefNum;
@@ -93,12 +92,8 @@ int adspCLListen(sp, pb)	/* (DSPPBPtr pb) */
 	    clpb = (struct adspcmd *)gbuf_rptr(mp);
 	    clpb->ioc = 0;
 	    clpb->mp = mp;
-	    ATDISABLE(s, sp->lock);
-	    if (qAddToEnd(&sp->opb, clpb)){	/* Add to list of listeners */
-	     ATENABLE(s, sp->lock);
-		return EFAULT; /* bogus, but discriminate from other errors */
-	    }
-	    ATENABLE(s, sp->lock);
+	    if (qAddToEnd(&sp->opb, clpb))	/* Add to list of listeners */
+			return EFAULT; /* bogus, but discriminate from other errors */
     } else {
 	    pb->ioResult = errDSPQueueSize;
 	    return ENOBUFS;
diff --git a/bsd/netat/adsp_Close.c b/bsd/netat/adsp_Close.c
index 9bdeb69d6..67541a430 100644
--- a/bsd/netat/adsp_Close.c
+++ b/bsd/netat/adsp_Close.c
@@ -62,7 +62,6 @@
 #include <netat/adsp.h>
 #include <netat/adsp_internal.h>
 
-extern atlock_t adspall_lock;
 
 static void qRemove(CCBPtr, CCBPtr);
 
@@ -128,7 +127,6 @@ int  CompleteQueue(qhead, code)	/* (DSPPBPtr FPTR qhead, OSErr code) */
     register gref_t *gref;
     register int    total = 0;
     CCBPtr sp = 0;
-    int s;
 
     n = *qhead;			/* Get first item */
     *qhead = 0;			/* Zero out the queue */
@@ -137,7 +135,6 @@ int  CompleteQueue(qhead, code)	/* (DSPPBPtr FPTR qhead, OSErr code) */
 	if (gref->info) {
 	    sp = (CCBPtr)gbuf_rptr(((gbuf_t *)gref->info));
 	    atalk_flush(sp->gref);
-	    ATDISABLE(s, sp->lock);
 	    }
     }
 
@@ -150,8 +147,6 @@ int  CompleteQueue(qhead, code)	/* (DSPPBPtr FPTR qhead, OSErr code) */
 	} else
 	    gbuf_freem(p->mp);
     }				/* while */
-    if (sp)
-	ATENABLE(s, sp->lock);
     return(total);
 }
 
@@ -358,7 +353,6 @@ int adspClose(sp, pb)		/* (DSPPBPtr pb) */
     register CCBPtr sp;
     register struct adspcmd *pb;
 {
-    int	s;
     register gbuf_t *mp;
 	
     /* Must execute nearly all of this with ints off because user could 
@@ -410,10 +404,8 @@ int adspClose(sp, pb)		/* (DSPPBPtr pb) */
      * is still pending.
      */
     if (pb->csCode == (short)dspClose) {
-	ATDISABLE(s, sp->lock);
 	if ((sp->state == (short)sPassive) || (sp->state == (short)sOpening)) {
 	    sp->state = sClosed;
-	    ATENABLE(s, sp->lock);
 	    DoClose(sp, errAborted, 0);
 	    pb->ioResult = 0;
 	    adspioc_ack(0, pb->ioc, pb->gref);
@@ -421,23 +413,19 @@ int adspClose(sp, pb)		/* (DSPPBPtr pb) */
 	}
 		
 	if (sp->state == (word)sClosed)	{ /* Ok to close a closed connection */
-	    ATENABLE(s, sp->lock);
 	    pb->ioResult = 0;
 	    adspioc_ack(0, pb->ioc, pb->gref);
 	    return 0;
 	}
 	if ((sp->state != (word)sOpen) && (sp->state != (word)sClosing)) {
-	    ATENABLE(s, sp->lock);
 	    pb->ioResult = errState;
 	    return EINVAL;
 	}
 		
 	sp->state = sClosing;	/* No matter what, we're closing */
-	ATENABLE(s, sp->lock);
     } 				/* dspClose */
     
     else {			/* dspRemove */
-	ATDISABLE(s, sp->lock);
 	sp->removing = 1;	/* Prevent allowing another dspClose. */
 				/* Tells completion routine of close */
 				/* packet to remove us. */
@@ -445,13 +433,10 @@ int adspClose(sp, pb)		/* (DSPPBPtr pb) */
 	if (sp->state == sPassive || sp->state == sClosed || 
 	    sp->state == sOpening) {
 	    sp->state = sClosed;
-	    ATENABLE(s, sp->lock);
 	    DoClose(sp, errAborted, 0); /* Will remove CCB! */
 	    return 0;
-	} else {			/* sClosing & sOpen */
+	} else			/* sClosing & sOpen */
 	    sp->state = sClosing;
-	    ATENABLE(s, sp->lock);
-	}
 	
     }				/* dspRemove */
 
@@ -467,9 +452,7 @@ int adspClose(sp, pb)		/* (DSPPBPtr pb) */
 	    pb = (struct adspcmd *)gbuf_rptr(mp); /* get new parameter block */
 	    pb->ioc = 0;
 	    pb->mp = mp;
-	    ATDISABLE(s, sp->lock);
 	    qAddToEnd(&sp->opb, pb);	/* and save it */
-	    ATENABLE(s, sp->lock);
     } else {
 	    pb->ioResult = 0;
 	    adspioc_ack(0, pb->ioc, pb->gref); /* release user, and keep no copy
@@ -485,19 +468,15 @@ static void qRemove(qptr, elem)
     register CCBPtr qptr;
     register CCBPtr elem;
 {
-	int s;
 
-	ATDISABLE(s, adspall_lock);
     while(qptr->ccbLink) {
 	if ((DSPPBPtr)(qptr->ccbLink) == (DSPPBPtr)elem) {
 	    qptr->ccbLink = elem->ccbLink;
 	    elem->ccbLink = 0;
-	    ATENABLE(s, adspall_lock);
 	    return;
 	}
 	qptr = qptr->ccbLink;
     }
-	ATENABLE(s, adspall_lock);
 }
 
 int RxClose(sp)
@@ -505,17 +484,11 @@ int RxClose(sp)
 {
     register gbuf_t *mp;
     register struct adspcmd *pb;
-	int s, l;
 
-	ATDISABLE(l, sp->lockClose);
-	ATDISABLE(s, sp->lock);
-	if ((sp->state == sClosing) || (sp->state == sClosed)) {
-		ATENABLE(s, sp->lock);
-		ATENABLE(l, sp->lockClose);
+	if ((sp->state == sClosing) || (sp->state == sClosed))
 		return 0;
-	}
+	
     sp->state = sClosed;
-	ATENABLE(s, sp->lock);
     CheckReadQueue(sp);		/* try to deliver all remaining data */
 
     if ( (mp = gbuf_alloc(sizeof(struct adspcmd), PRI_HI)) ) {
@@ -532,6 +505,5 @@ int RxClose(sp)
 if ((sp->userFlags & eClosed) == 0)
     DoClose(sp, errAborted, -1);	/* abort send requests and timers */
 
-	ATENABLE(l, sp->lockClose);
     return 0;
 }
diff --git a/bsd/netat/adsp_Control.c b/bsd/netat/adsp_Control.c
index c802c837d..3bc19f846 100644
--- a/bsd/netat/adsp_Control.c
+++ b/bsd/netat/adsp_Control.c
@@ -135,7 +135,6 @@ void CheckSend(sp)		/* (CCBPtr sp) */
 {
     int i;
     int	attnMsg;		/* True if attention message */
-    int	s;
     register gbuf_t *mp;	/* send message block */
 #ifdef notdef
     register gbuf_t *tmp;
@@ -161,7 +160,6 @@ top:
 		gbuf_freel(mlist);
 	return;		/* can't get buffers... do nothing! */
     }
-    ATDISABLE(s, sp->lock);
     sp->callSend = 0;		/* Clear flag */
     use_attention_code = 0;
     len = 0;
@@ -182,24 +180,24 @@ top:
 
 	    /* point past ADSP header (no attention) */
 	    dp = ((char *) gbuf_wptr(mp)) + ADSP_FRAME_LEN; 
-	    UAL_ASSIGN(sp->f.pktFirstByteSeq, netdw(sp->firstRtmtSeq));
+	    UAL_ASSIGN_HTON(sp->f.pktFirstByteSeq, sp->firstRtmtSeq);
 	    
-	    UAS_ASSIGN(sp->of.version, netw(0x0100)); /* Fill in open connection parms */
-	    UAS_ASSIGN(sp->of.dstCID, sp->remCID);	/* Destination CID */
-	    UAL_ASSIGN(sp->of.pktAttnRecvSeq, netdw(sp->attnRecvSeq));
+	    UAS_ASSIGN_HTON(sp->of.version, netw(0x0100)); /* Fill in open connection parms */
+	    UAS_ASSIGN_HTON(sp->of.dstCID, sp->remCID);	/* Destination CID */
+	    UAL_ASSIGN_HTON(sp->of.pktAttnRecvSeq, sp->attnRecvSeq);
 	    bcopy((caddr_t) &sp->of, (caddr_t) dp, ADSP_OPEN_FRAME_LEN);
 	    len += ADSP_OPEN_FRAME_LEN;
 
 	    if (i & B_CTL_OREQ) {
-		UAS_ASSIGN(sp->f.CID, sp->locCID);
+		UAS_ASSIGN_HTON(sp->f.CID, sp->locCID);
 		mask = B_CTL_OREQ;
 		sp->f.descriptor = ADSP_CONTROL_BIT | ADSP_CTL_OREQ;
 	    } else if (i & B_CTL_OACK) {
-		UAS_ASSIGN(sp->f.CID, sp->locCID);
+		UAS_ASSIGN_HTON(sp->f.CID, sp->locCID);
 		mask = B_CTL_OACK;
 		sp->f.descriptor = ADSP_CONTROL_BIT | ADSP_CTL_OACK;
 	    } else if (i & B_CTL_OREQACK) {
-		UAS_ASSIGN(sp->f.CID, sp->locCID);
+		UAS_ASSIGN_HTON(sp->f.CID, sp->locCID);
 		mask = B_CTL_OREQACK;
 		sp->f.descriptor = ADSP_CONTROL_BIT | ADSP_CTL_OREQACK;
 	    } else 		/* Deny */
@@ -225,7 +223,7 @@ top:
 	    }
 	} else {
 	    /* seq # of next byte to send */
-	    UAL_ASSIGN(sp->f.pktFirstByteSeq, netdw(sp->sendSeq));	
+	    UAL_ASSIGN_HTON(sp->f.pktFirstByteSeq, sp->sendSeq);	
 			
 	    if (i & B_CTL_CLOSE) {
 		sp->state = sClosed; /* Now we're closed */
@@ -304,7 +302,7 @@ top:
     }
 
     if (sp->sendDataAck) {
-	UAL_ASSIGN(sp->f.pktFirstByteSeq, netdw(sp->sendSeq)); /* seq # of next byte */
+	UAL_ASSIGN_HTON(sp->f.pktFirstByteSeq, sp->sendSeq); /* seq # of next byte */
 	attnMsg = 0;
 	sp->f.descriptor = ADSP_CONTROL_BIT;
 	goto sendit;
@@ -315,7 +313,6 @@ top:
      */
     if (mp)
 	gbuf_freem(mp);
-    ATENABLE(s, sp->lock);
     if (mlist)
 	adsp_sendddp(sp, mlist, 0, &sp->remoteAddress, DDP_ADSP);
     return;
@@ -323,13 +320,13 @@ top:
 sendit:
 
     if (attnMsg) {
-	UAL_ASSIGN(sp->f.pktFirstByteSeq, netdw(sp->attnSendSeq));
-	UAL_ASSIGN(sp->f.pktNextRecvSeq, netdw(sp->attnRecvSeq));
+	UAL_ASSIGN_HTON(sp->f.pktFirstByteSeq, sp->attnSendSeq);
+	UAL_ASSIGN_HTON(sp->f.pktNextRecvSeq, sp->attnRecvSeq);
 	UAS_ASSIGN(sp->f.pktRecvWdw, 0);	/* Always zero in attn pkt */
     } else {
 	sp->sendDataAck = 0;
-	UAL_ASSIGN(sp->f.pktNextRecvSeq, netdw(sp->recvSeq));
-	UAS_ASSIGN(sp->f.pktRecvWdw, netw(CalcRecvWdw(sp)));
+	UAL_ASSIGN_HTON(sp->f.pktNextRecvSeq, sp->recvSeq);
+	UAS_ASSIGN_HTON(sp->f.pktRecvWdw, CalcRecvWdw(sp));
     }
     if (use_attention_code) {
 	bcopy((caddr_t) &sp->f, (caddr_t) gbuf_wptr(mp), ADSP_FRAME_LEN + 2);
@@ -347,12 +344,10 @@ sendit:
 
     if (sp->state == sClosed) {	/* must have sent a close advice */
 				/* send header + data */
-	ATENABLE(s, sp->lock);
 	adsp_sendddp(sp, mlist, 0, &sp->remoteAddress, DDP_ADSP);
 	DoClose(sp, 0, -1);	/* complete close! */
 	return;
     }
-    ATENABLE(s, sp->lock);
     if (sp->state == sClosing) /* See if we were waiting on this write */
 	CheckOkToClose(sp);
     goto top;
@@ -407,7 +402,7 @@ attachData(sp, mp)
      * The easiest fix to this timing dilemma seems to be to reset 
      * sendSeq to first Rtmt Seq if we're sending the first packet.
      */
-    UAL_ASSIGN(sp->f.pktFirstByteSeq, netdw(sp->sendSeq));
+    UAL_ASSIGN_HTON(sp->f.pktFirstByteSeq, sp->sendSeq);
 		
     if (smp = sp->sbuf_mb) /* Get oldest header */
 	eom = 1;
diff --git a/bsd/netat/adsp_Init.c b/bsd/netat/adsp_Init.c
index 7273dd963..6c7895cb2 100644
--- a/bsd/netat/adsp_Init.c
+++ b/bsd/netat/adsp_Init.c
@@ -56,7 +56,6 @@
 #include <netat/adsp.h>
 #include <netat/adsp_internal.h>
 
-extern atlock_t adspgen_lock;
 
 /*
  * InitContinue
@@ -76,7 +75,6 @@ static void InitContinue(sp, pb) /* (CCBPtr sp, DSPPBPtr pb, int soc) */
     CCBPtr sp;
     struct adspcmd *pb;
 {
-    int s;
 
     /* Save connection's socket # in CCB */
     sp->localSocket = pb->socket; 
@@ -84,9 +82,7 @@ static void InitContinue(sp, pb) /* (CCBPtr sp, DSPPBPtr pb, int soc) */
     /*
      * Link the new ccb onto queue.  Must be done with interrupts off.
      */
-    ATDISABLE(s, adspgen_lock);
     qAddToEnd(AT_ADSP_STREAMS, sp); /* Put on linked list of connections */
-    ATENABLE(s, adspgen_lock);
     return;
 }
 
diff --git a/bsd/netat/adsp_Open.c b/bsd/netat/adsp_Open.c
index 5507afe8f..132f9b405 100644
--- a/bsd/netat/adsp_Open.c
+++ b/bsd/netat/adsp_Open.c
@@ -53,7 +53,6 @@
 #include <netat/adsp.h>
 #include <netat/adsp_internal.h>
 
-extern atlock_t adspgen_lock;
 
 /*
  * NextCID
@@ -67,12 +66,10 @@ extern atlock_t adspgen_lock;
  */
 unsigned short NextCID()
 {
-	int s;
 	unsigned short num;
 	register CCB *queue;
 
 	while (1) {
-	    ATDISABLE(s, adspgen_lock);		/* Disable interrupts */
 	    num = ++adspGlobal.lastCID;
 	    /* qfind_w below is in 68K assembly */
 	    /* point to the first element */
@@ -83,7 +80,6 @@ unsigned short NextCID()
 			break;
 		    queue = queue->ccbLink;
 	    }
-	    ATENABLE(s, adspgen_lock);
 	    if (queue == (CCBPtr)NULL)
 		break;	
 	}
@@ -247,7 +243,7 @@ int adspOpen(sp, pb)		/* (DSPPBPtr pb) */
     if (ocMode == ocEstablish) { /* Only set these if establish mode */
 	sp->recvSeq = pb->u.openParams.recvSeq;
 	sp->attnRecvSeq = pb->u.openParams.attnRecvSeq;
-	UAS_ASSIGN(sp->f.CID, sp->locCID); /* Preset the CID in the ADSP header */
+	UAS_ASSIGN_HTON(sp->f.CID, sp->locCID); /* Preset the CID in the ADSP header */
 	/* This is done elsewhere for all other modes */
 	InsertTimerElem(&adspGlobal.slowTimers, &sp->ProbeTimer, 
 			sp->probeInterval);
diff --git a/bsd/netat/adsp_Packet.c b/bsd/netat/adsp_Packet.c
index 6a4f4fb0c..41f13cd65 100644
--- a/bsd/netat/adsp_Packet.c
+++ b/bsd/netat/adsp_Packet.c
@@ -547,7 +547,6 @@ static int RXConnection(gref, spPtr, f, len, addr, dsoc)
     gbuf_t *mp;
     ADSP_FRAMEPtr adspp;
     ADSP_OPEN_DATAPtr adspop;
-    int s;
 
     op = (ADSP_OPEN_DATAPtr)&f->data[0]; /* Point to Open-Connection parms */
     len -= ADSP_FRAME_LEN;
@@ -573,7 +572,7 @@ static int RXConnection(gref, spPtr, f, len, addr, dsoc)
 	adspop = (ADSP_OPEN_DATAPtr)gbuf_wptr(mp);
 	gbuf_winc(mp,ADSP_OPEN_FRAME_LEN);
 	UAS_UAS(adspop->dstCID, f->CID);
-	UAS_ASSIGN(adspop->version, 0x100);
+	UAS_ASSIGN_HTON(adspop->version, 0x100);
 	adsp_sendddp(0, mp, DDPL_FRAME_LEN + ADSP_FRAME_LEN + 
 		   ADSP_OPEN_FRAME_LEN, &addr, DDP_ADSP);
 
@@ -582,8 +581,8 @@ static int RXConnection(gref, spPtr, f, len, addr, dsoc)
     m.addr = addr;
     m.socket = dsoc;
     m.descriptor = f->descriptor;
-    m.srcCID = UAS_VALUE(f->CID);
-    m.dstCID = UAS_VALUE(op->dstCID);	/* On even-byte boundry */
+    m.srcCID = UAS_VALUE_NTOH(f->CID);
+    m.dstCID = UAS_VALUE_NTOH(op->dstCID);	/* On even-byte boundry */
     m.idx = ((f->descriptor & ADSP_CONTROL_MASK) - 1) * 4;
                                           
     /*
@@ -604,7 +603,6 @@ static int RXConnection(gref, spPtr, f, len, addr, dsoc)
 				  (ProcPtr)MatchListener)) == 0)
 	    return 1;
 
-	ATDISABLE(s, sp->lock);
 	p = (struct adspcmd *)&sp->opb;
 	while (n = (struct adspcmd *)p->qLink) /* Hunt down list of listens */
 	{
@@ -621,11 +619,10 @@ static int RXConnection(gref, spPtr, f, len, addr, dsoc)
 		p->qLink = n->qLink; /* Unlink this param block */
 		n->u.openParams.remoteCID = m.srcCID;
 		*((AddrUnionPtr)&n->u.openParams.remoteAddress)	= addr;
-		n->u.openParams.sendSeq	= netdw(UAL_VALUE(f->pktNextRecvSeq));
-		n->u.openParams.sendWindow = netw(UAS_VALUE(f->pktRecvWdw));
-		n->u.openParams.attnSendSeq = netdw(UAL_VALUE(op->pktAttnRecvSeq));
+		n->u.openParams.sendSeq	= UAL_VALUE_NTOH(f->pktNextRecvSeq);
+		n->u.openParams.sendWindow = UAS_VALUE_NTOH(f->pktRecvWdw);
+		n->u.openParams.attnSendSeq = UAL_VALUE_NTOH(op->pktAttnRecvSeq);
 		n->ioResult = 0;
-		ATENABLE(s, sp->lock);
 		completepb(sp, n); /* complete copy of request */
 				/* complete(n, 0); */
 		return 0;
@@ -635,29 +632,26 @@ static int RXConnection(gref, spPtr, f, len, addr, dsoc)
 			
 	}			/* while */
 		
-	ATENABLE(s, sp->lock);
 	return 1;
     }
 	
     *spPtr = sp;		/* Save ptr to stream we just found */
 	
-	ATDISABLE(s, sp->lock);
     sp->openState = m.t->openState; /* Move to next state (may be same) */
     sp->state = m.t->state;	/* Move to next state (may be same) */
 
     if (m.t->action & A_SAVEPARMS) { /* Need to Save open-conn parms */
-	sp->firstRtmtSeq = sp->sendSeq = netdw(UAL_VALUE(f->pktNextRecvSeq));
-	sp->sendWdwSeq = netdw(UAL_VALUE(f->pktNextRecvSeq)) + netw(UAS_VALUE(f->pktRecvWdw)) - 1;
-	sp->attnSendSeq = netdw(UAL_VALUE(op->pktAttnRecvSeq)); /* on even boundry */
+	sp->firstRtmtSeq = sp->sendSeq = UAL_VALUE_NTOH(f->pktNextRecvSeq);
+	sp->sendWdwSeq = UAL_VALUE_NTOH(f->pktNextRecvSeq) + UAS_VALUE_NTOH(f->pktRecvWdw) - 1;
+	sp->attnSendSeq = UAL_VALUE_NTOH(op->pktAttnRecvSeq); /* on even boundry */
 
 		
-	sp->remCID = UAS_VALUE(f->CID);	/* Save Source CID as RemCID */
+	sp->remCID = UAS_VALUE_NTOH(f->CID);	/* Save Source CID as RemCID */
 	UAS_UAS(sp->of.dstCID, f->CID);	/* Save CID in open ctl packet */
 		
 	sp->remoteAddress = addr; /* Save his address */
 
     }
-	ATENABLE(s, sp->lock);
 
     if (m.t->action & A_DENY) {	/* We've been denied ! */
 	DoClose(sp, errOpenDenied, -1);
@@ -731,7 +725,6 @@ int adspPacket(gref, mp)
     int len;
     AddrUnion a;
     int dsoc;
-    int s;
     register DDPX_FRAME *ddp;	/* DDP frame pointer */
     register ADSP_FRAMEPtr f;	/* Frame */
     CCBPtr sp;
@@ -743,7 +736,7 @@ int adspPacket(gref, mp)
 	return -1;
     f = (ADSP_FRAMEPtr)(bp + DDPL_FRAME_LEN);
 
-    len = UAS_VALUE(ddp->ddpx_length) & 0x3ff; /* (ten bits of length) */
+    len = UAS_VALUE_NTOH(ddp->ddpx_length) & 0x3ff; /* (ten bits of length) */
     len -= DDPL_FRAME_LEN;
     if (len < (sizeof(ADSP_FRAME) - 1))	/* Packet too small */
 	return -1;		/* mark the failure */
@@ -805,12 +798,10 @@ int adspPacket(gref, mp)
 		/* This pkt may also ack some data we sent */
 		CheckRecvSeq(sp, f); 
 		RemoveTimerElem(&adspGlobal.fastTimers, &sp->RetryTimer);
-		ATDISABLE(s, sp->lock);
 		sp->sendSeq = sp->firstRtmtSeq;
 		sp->pktSendCnt = 0;
 		sp->waitingAck = 0;
 		sp->callSend = 1;
-		ATENABLE(s, sp->lock);
 	    } else
 		goto ignore;
 	    break;
diff --git a/bsd/netat/adsp_Read.c b/bsd/netat/adsp_Read.c
index 0d270a350..265b2e723 100644
--- a/bsd/netat/adsp_Read.c
+++ b/bsd/netat/adsp_Read.c
@@ -68,7 +68,6 @@ int CheckReadQueue(sp)		/* (CCBPtr sp) */
     register CCBPtr sp;
 {
     register struct adspcmd *pb;
-    int s;
     unsigned short cnt;
     char eom = 0;
     register gbuf_t *mp;
@@ -78,7 +77,6 @@ int CheckReadQueue(sp)		/* (CCBPtr sp) */
     dPrintf(D_M_ADSP, D_L_TRACE, ("CheckReadQueue: sp=0x%x\n", (unsigned)sp));
     KERNEL_DEBUG(DBG_ADSP_READ, 0, sp, sp->rbuf_mb, sp->rpb, sp->delay);
     trace_mbufs(D_M_ADSP_LOW, "    bCQR m", sp->rbuf_mb);
-    ATDISABLE(s, sp->lock);
 
     while (sp->rData && (pb = sp->rpb)) {		/* have data */
         dPrintf(D_M_ADSP, D_L_TRACE, 
@@ -239,7 +237,6 @@ int CheckReadQueue(sp)		/* (CCBPtr sp) */
 	sp->sendDataAck = 1;
 	sp->callSend = 1;
     }
-    ATENABLE(s, sp->lock);
 
     KERNEL_DEBUG(DBG_ADSP_READ, 0x11, sp, 0, 0, 0);
     trace_mbufs(D_M_ADSP_LOW, "    eCQR m", sp->rbuf_mb);
@@ -263,14 +260,12 @@ int CheckAttn(sp, pb)		/* (CCBPtr sp) */
     register CCBPtr sp;
     register struct adspcmd *pb;
 {
-    int s;
     gbuf_t *mp;
     gref_t *gref;
 	
     dPrintf(D_M_ADSP, D_L_TRACE, 
 	    ("CheckAttn: sp=0x%x, pb=0x%x\n", (unsigned)sp, (unsigned)pb));
 
-    ATDISABLE(s, sp->lock);
     if (mp = sp->attn_mb) {
 
 	/*
@@ -302,7 +297,6 @@ int CheckAttn(sp, pb)		/* (CCBPtr sp) */
     if (mp) {
 	SndMsgUp(gref, mp);
 	}
-    ATENABLE(s, sp->lock);
     return 0;
 }
 
@@ -328,7 +322,6 @@ int adspRead(sp, pb)		/* (DSPPBPtr pb) */
     register struct adspcmd *pb;
 {
     register gbuf_t *mp;
-    int	s;
 
     dPrintf(D_M_ADSP, D_L_TRACE, 
 	    ("adspRead: sp=0x%x, pb=0x%x\n", (unsigned)sp, (unsigned)pb));
@@ -343,19 +336,15 @@ int adspRead(sp, pb)		/* (DSPPBPtr pb) */
     /*
      * It's OK to read on a closed, or closing session
      */
-    ATDISABLE(s, sp->lock);
     if (sp->state != sOpen && sp->state != sClosing && sp->state != sClosed) {
-	ATENABLE(s, sp->lock);
 	pb->ioResult = errState;
 	return EINVAL;
     }
     if (sp->rData && (sp->rpb == 0)) { /* if data, and no queue of pbs */
 	qAddToEnd(&sp->rpb, pb); /* deliver data to user directly */
-	ATENABLE(s, sp->lock);
 	CheckReadQueue(sp);
     } else if ((pb->u.ioParams.reqCount == 0) && (sp->rpb == 0)) {
 	    /* empty read */
-	    ATENABLE(s, sp->lock);
 	    pb->ioResult = 0;
 	    adspioc_ack(0, pb->ioc, pb->gref);
 	    return 0;
@@ -367,9 +356,7 @@ int adspRead(sp, pb)		/* (DSPPBPtr pb) */
 		pb->ioc = 0;
 		pb->mp = mp;
 		qAddToEnd(&sp->rpb, pb); /* and queue it for later */
-		ATENABLE(s, sp->lock);
 	} else {
-		ATENABLE(s, sp->lock);
 		pb->ioResult = errDSPQueueSize;
 		return ENOBUFS;
 	}
diff --git a/bsd/netat/adsp_RxAttn.c b/bsd/netat/adsp_RxAttn.c
index 2c398c82d..520ac9e7f 100644
--- a/bsd/netat/adsp_RxAttn.c
+++ b/bsd/netat/adsp_RxAttn.c
@@ -110,7 +110,7 @@ CCBPtr FindSender(f, a)		/* (ADSP_FRAMEPtr f, AddrUnion a) */
     MATCH_SENDER m;
 
     m.addr = a;
-    m.srcCID = UAS_VALUE(f->CID);
+    m.srcCID = UAS_VALUE_NTOH(f->CID);
     return (CCBPtr)qfind_m(AT_ADSP_STREAMS, &m, (ProcPtr)MatchSender);
 }
 
@@ -147,7 +147,7 @@ int RXAttention(sp, mp, f, len)	/* (CCBPtr sp, ADSP_FRAMEPtr f, word len) */
 	 (char)(ADSP_ATTENTION_BIT | ADSP_ACK_REQ_BIT)) && /* Attention Data */
 	((sp->userFlags & eAttention) == 0)) /* & he read the previous */
     {
-	diff = netdw(UAL_VALUE(f->pktFirstByteSeq)) - sp->attnRecvSeq;
+	diff = UAL_VALUE_NTOH(f->pktFirstByteSeq) - sp->attnRecvSeq;
 	if (diff > 0)		/* Hey, he missed one */
 	    return 1;
 
@@ -183,7 +183,7 @@ int RXAttention(sp, mp, f, len)	/* (CCBPtr sp, ADSP_FRAMEPtr f, word len) */
      * Interrupts are OFF here, otherwise we have to do this atomically
      */
     /* Check to see if this acknowledges anything */
-    if ((sp->attnSendSeq + 1) == netdw(UAL_VALUE(f->pktNextRecvSeq))) {
+    if ((sp->attnSendSeq + 1) == UAL_VALUE_NTOH(f->pktNextRecvSeq)) {
 	sp->attnSendSeq++;
 	if ((pb = sp->sapb) == 0) { /* We never sent data ? !!! */
 	    if (mp)
diff --git a/bsd/netat/adsp_RxData.c b/bsd/netat/adsp_RxData.c
index bf4205775..a1e4fadff 100644
--- a/bsd/netat/adsp_RxData.c
+++ b/bsd/netat/adsp_RxData.c
@@ -113,20 +113,18 @@ void CheckRecvSeq(sp, f)	/* (CCBPtr sp, ADSP_FRAMEPtr f) */
     register CCBPtr sp;
     register ADSP_FRAMEPtr f;
 {
-    int s;
     int pktNextRecvSeq;
     int sendWdwSeq;
     int eom;
     int hlen;
     register gbuf_t *mp;
 	
-    ATDISABLE(s, sp->lock);
     if (f->descriptor & ADSP_ACK_REQ_BIT) { /* He wants an Ack */
 	sp->sendDataAck = 1;
 	sp->callSend = 1;
     }
 	
-    pktNextRecvSeq = netdw(UAL_VALUE(f->pktNextRecvSeq)); /* Local copy */
+    pktNextRecvSeq = UAL_VALUE_NTOH(f->pktNextRecvSeq); /* Local copy */
 
     /*
      * Make sure the sequence number corresponds to reality -- i.e. for 
@@ -241,7 +239,7 @@ void CheckRecvSeq(sp, f)	/* (CCBPtr sp, ADSP_FRAMEPtr f) */
 	sp->callSend = 1;
 
 noack:
-    sendWdwSeq = netw(UAS_VALUE(f->pktRecvWdw)) - 1 + pktNextRecvSeq;
+    sendWdwSeq = UAS_VALUE_NTOH(f->pktRecvWdw) - 1 + pktNextRecvSeq;
 
     if (GT(sendWdwSeq, sp->sendWdwSeq))	/* Don't make send window smaller */
     {
@@ -249,7 +247,6 @@ noack:
 				/* if we can send more data */
 	sp->sendWdwSeq	= sendWdwSeq;
     }
-    ATENABLE(s, sp->lock);
 }
 
 /*
@@ -275,7 +272,7 @@ int RXData(sp, mp, f, len)	/* (CCBPtr sp, ADSP_FRAMEPtr f, word len) */
     ADSP_FRAMEPtr f;
     int len;
 {
-    int s, offset;
+    int offset;
     int PktFirstByteSeq;
     short cnt;
     char eom;
@@ -293,14 +290,12 @@ int RXData(sp, mp, f, len)	/* (CCBPtr sp, ADSP_FRAMEPtr f, word len) */
 
     trace_mbufs(D_M_ADSP, "  mp", mp);
 
-    PktFirstByteSeq = netdw(UAL_VALUE(f->pktFirstByteSeq)); /* Local copy */
+    PktFirstByteSeq = UAL_VALUE_NTOH(f->pktFirstByteSeq); /* Local copy */
 
-    ATDISABLE(s, sp->lock);
     if (GT(PktFirstByteSeq, sp->recvSeq)) /* missed a packet (out of order) */
     {
 	if (sp->badSeqCnt++ > sp->badSeqCnt) /* Need to send rexmit advice */
 	    sp->sendCtl |= B_CTL_RETRANSMIT;
-	ATENABLE(s, sp->lock);
 	CheckRecvSeq(sp, f);	/* Will set send ACK flag if requested */
 	CheckReadQueue(sp);
 	gbuf_freem(mp);
@@ -313,7 +308,6 @@ int RXData(sp, mp, f, len)	/* (CCBPtr sp, ADSP_FRAMEPtr f, word len) */
     }
 	
     if (LTE(PktFirstByteSeq + len + eom, sp->recvSeq)) { /* duplicate data? */
-	ATENABLE(s, sp->lock);
 	CheckRecvSeq(sp, f);	/* Will set send ACK flag if requested */
 	CheckReadQueue(sp);
 	gbuf_freem(mp);
@@ -388,7 +382,6 @@ int RXData(sp, mp, f, len)	/* (CCBPtr sp, ADSP_FRAMEPtr f, word len) */
      * doing anything that might take a long while
      */
 
-    ATENABLE(s, sp->lock);
     CheckRecvSeq(sp, f);	/* Will set send ACK flag if requested */
     CheckReadQueue(sp);
     KERNEL_DEBUG(DBG_ADSP_RCV, 5, sp, sp->rbuf_mb, 0, 0);
diff --git a/bsd/netat/adsp_Status.c b/bsd/netat/adsp_Status.c
index ae4de1fb8..505dd746c 100644
--- a/bsd/netat/adsp_Status.c
+++ b/bsd/netat/adsp_Status.c
@@ -114,7 +114,6 @@ int adspStatus(sp, pb)	/* (DSPPBPtr pb) */
 {
     short err;
     short bytes;
-    int	s;
 
     if (sp == 0) {
 	pb->ioResult = errRefNum;
@@ -122,7 +121,6 @@ int adspStatus(sp, pb)	/* (DSPPBPtr pb) */
     }
 	
     pb->u.statusParams.ccbPtr 	= (TPCCB)sp;
-    ATDISABLE(s, sp->lock);	
 	
     /*
      * pending bytes in send queue
@@ -148,7 +146,6 @@ int adspStatus(sp, pb)	/* (DSPPBPtr pb) */
 				/* available buffer space in receive queue */
     pb->u.statusParams.recvQFree = CalcRecvWdw(sp);
 
-    ATENABLE(s, sp->lock);	
     pb->ioResult = 0;
     adspioc_ack(0, pb->ioc, pb->gref);
     return 0;
diff --git a/bsd/netat/adsp_Timer.c b/bsd/netat/adsp_Timer.c
index edcc8ffca..9f551fd0e 100644
--- a/bsd/netat/adsp_Timer.c
+++ b/bsd/netat/adsp_Timer.c
@@ -76,13 +76,10 @@ void TimerTick();
 void TrashSession(sp)		/* (CCBPtr sp) */
     CCBPtr sp;
 {
-    int s;
 
-    ATDISABLE(s, sp->lock);
     sp->userFlags |= eTearDown;
     sp->removing = 1;
     sp->state = sClosed;
-    ATENABLE(s, sp->lock);
 
     DoClose(sp, errAborted, 1);
 }
@@ -100,10 +97,8 @@ void DoTimerElem(t) /* (TimerElemPtr t) */
     TimerElemPtr t;
 {
     CCBPtr sp;
-    int	s;
 
     sp = (CCBPtr)((Ptr)t - t->type); /* Recover stream pointer for this guy */
-    ATDISABLE(s, sp->lock);
 	
     if (t->type == kFlushTimerType) { /* flush write data time just fired */
 	if (sp->sData) {	/* If there's any data, flush it. */
@@ -142,7 +137,6 @@ void DoTimerElem(t) /* (TimerElemPtr t) */
     } else if (t->type == kProbeTimerType) {
 	if (sp->state == sOpen || sp->state == sClosing) {
 	    if (--sp->probeCntr == 0) { /* Connection died */
-		ATENABLE(s, sp->lock);
 		TrashSession(sp);
 		return;
 	    } else {
@@ -157,7 +151,6 @@ void DoTimerElem(t) /* (TimerElemPtr t) */
 	    {
 		if (--sp->openRetrys == 0) { /* Oops, didn't open */
 		    sp->state = sClosed;
-		    ATENABLE(s, sp->lock);
 		    DoClose(sp, errOpening, 1);
 		    return;
 		}		/* open failed */
@@ -175,11 +168,9 @@ void DoTimerElem(t) /* (TimerElemPtr t) */
 	dPrintf(D_M_ADSP, D_L_ERROR, ("DoTimerElem:Unknown timer type!\n"));
     }
 
-    ATENABLE(s, sp->lock);
 	return;
 	
 send:
-    ATENABLE(s, sp->lock);
     CheckSend(sp);
 }
 
diff --git a/bsd/netat/adsp_TimerElem.c b/bsd/netat/adsp_TimerElem.c
index 4cf461c88..047e762d9 100644
--- a/bsd/netat/adsp_TimerElem.c
+++ b/bsd/netat/adsp_TimerElem.c
@@ -52,7 +52,6 @@
 #include <netat/adsp.h>
 #include <netat/adsp_internal.h>
 
-atlock_t adsptmr_lock;
 
 extern	void DoTimerElem();	/* (TimerElemPtr t);  
 				 * External routine called to 
@@ -75,17 +74,13 @@ void InsertTimerElem(qhead, t, val)
 {
     TimerElemPtr p;		/* parent pointer */
     TimerElemPtr n;		/* current */
-    int	s;
-	
-    ATDISABLE(s, adsptmr_lock);
-	
+		
     if (t->onQ) {
         /*
-	 * someone else beat us to the punch and put this
-	 * element back on the queue, just return in this case
-	 */
-        ATENABLE(s, adsptmr_lock);
-	return;
+	 	* someone else beat us to the punch and put this
+	 	* element back on the queue, just return in this case
+	 	*/
+		return;
     }
     p = (TimerElemPtr)qhead;
 
@@ -107,7 +102,6 @@ void InsertTimerElem(qhead, t, val)
     t->timer = val;		/* this is our value */
     t->link = n;		/* we point to n */
     
-    ATENABLE(s, adsptmr_lock);
 }
 
 
@@ -125,17 +119,13 @@ void RemoveTimerElem(qhead, t)	/* (TimerElemPtr *qhead, TimerElemPtr t) */
 {
     TimerElemPtr p;		/* parent pointer */
     TimerElemPtr n;		/* current */
-    int	s;
-	
-    ATDISABLE(s, adsptmr_lock);
-	
+		
     if ( !t->onQ) {
         /*
-	 * someone else beat us to the punch and took this
-	 * element off of the queue, just return in this case
-	 */
-        ATENABLE(s, adsptmr_lock);
-	return;
+	 	* someone else beat us to the punch and took this
+	 	* element off of the queue, just return in this case
+	 	*/
+		return;
     }
     p = (TimerElemPtr)qhead;
 
@@ -153,7 +143,6 @@ void RemoveTimerElem(qhead, t)	/* (TimerElemPtr *qhead, TimerElemPtr t) */
 	p = n;
     }				/* while */
 	
-    ATENABLE(s, adsptmr_lock);
 }
 
 
@@ -171,29 +160,19 @@ void TimerQueueTick(qhead)	/* (TimerElemPtr *qhead) */
 {
     TimerElemPtr p;		/* parent pointer */
     TimerElemPtr n;		/* current */
-    int	s;
-	
-    ATDISABLE(s, adsptmr_lock);
-	
+		
     p = (TimerElemPtr)qhead;
-    if (p->link)		/* Is anything on queue? */
+    if (p->link) {		/* Is anything on queue? */
 	p->link->timer--;	/* Yes, decrement by a tick */
-    else
-	goto done;		/* No, we're outta' here */
-		
-    while ((n = p->link) && 
-	   (n->timer == 0)) /* Next guy needs to be serviced */
-    {
-	p->link = n->link;	/* Unlink us */
-	n->onQ	= 0;
+	while ((n = p->link) && 
+	       (n->timer == 0)) /* Next guy needs to be serviced */
+	{
+		p->link = n->link;	/* Unlink us */
+		n->onQ	= 0;
 
-	ATENABLE(s, adsptmr_lock);
-	DoTimerElem(n);
-	ATDISABLE(s, adsptmr_lock);
+		DoTimerElem(n);
 
-	p = (TimerElemPtr)qhead;
-    }				/* while */
-	
-done:
-    ATENABLE(s, adsptmr_lock);
+		p = (TimerElemPtr)qhead;
+	}				/* while */
+    }
 }
diff --git a/bsd/netat/adsp_Write.c b/bsd/netat/adsp_Write.c
index 280fb3892..9037c3dc0 100644
--- a/bsd/netat/adsp_Write.c
+++ b/bsd/netat/adsp_Write.c
@@ -73,7 +73,6 @@ int FillSendQueue(sp, pb)		/* (CCBPtr sp) */
 	int eom;		/* True if should set eom in header */
 	int cnt;		/* # of bytes in this write */
 	int err = 0;
-	int s;
 
 	cnt = pb->u.ioParams.reqCount - pb->u.ioParams.actCount;
 	eom = pb->u.ioParams.eom ? F_EOM : 0;
@@ -102,7 +101,6 @@ int FillSendQueue(sp, pb)		/* (CCBPtr sp) */
 	}
 	gbuf_cont(mb) = 0;
 
-	ATDISABLE(s, sp->lock);
 	sp->sData = 1;		/* note that there is data to send */
 	if ((mb = sp->csbuf_mb)) {	/* add to the current message */
 	    gbuf_linkb(mb, nmb);
@@ -118,7 +116,6 @@ int FillSendQueue(sp, pb)		/* (CCBPtr sp) */
 	    sp->csbuf_mb = 0;	/* if its done, no current buffer */
 	}
 	pb->u.ioParams.actCount += cnt; /* Update count field in param blk */
-	ATENABLE(s, sp->lock);
 	
 	if (pb->u.ioParams.actCount == pb->u.ioParams.reqCount) {
 	    /* Write is complete */
@@ -156,24 +153,20 @@ int adspWrite(sp, pb)		/* (DSPPBPtr pb) */
     CCBPtr sp;
     struct adspcmd *pb;
 {
-    int	s;
 	
     if (sp == 0) {
 	pb->ioResult = errRefNum;
 	return EINVAL;		/* no stream, so drop the message */
     }
 	
-    ATDISABLE(s, sp->lock);
     if (sp->state != sOpen) {	/* Not allowed */
 	pb->ioResult = errState;
-	ATENABLE(s, sp->lock);
 	atalk_notify(sp->gref, ENOTCONN);
 	gbuf_freem(pb->mp);
 	return 0;
     }
 	
     pb->u.ioParams.actCount = 0; /* Set # of bytes so far to zero */
-    ATENABLE(s, sp->lock);
     
     FillSendQueue(sp, pb);	/* Copy from write param block to send queue */
 
diff --git a/bsd/netat/adsp_attention.c b/bsd/netat/adsp_attention.c
index 79b811d43..40195fb38 100644
--- a/bsd/netat/adsp_attention.c
+++ b/bsd/netat/adsp_attention.c
@@ -78,7 +78,6 @@
  */
 int adspAttention(register struct adspcmd *pb, register CCBPtr sp)
 {
-    int	s;
     register gbuf_t *mp, *nmp;
     unsigned char uerr;
 	
@@ -121,16 +120,13 @@ l_err:
 	}
     }
     pb->ioDirection = 1;	/* outgoing attention data */
-    ATDISABLE(s, sp->lock);
     if (sp->sapb) {		/* Pending attentions already? */
-	qAddToEnd(&sp->sapb, pb); /* Just add to end of queue */
-	ATENABLE(s, sp->lock);
+		qAddToEnd(&sp->sapb, pb); /* Just add to end of queue */
     } else {
-	sp->sendAttnData = 1;	/* Start off this attention */
-	pb->qLink = 0;
-	sp->sapb = pb;
-	ATENABLE(s, sp->lock);
-	CheckSend(sp);
+		sp->sendAttnData = 1;	/* Start off this attention */
+		pb->qLink = 0;
+		sp->sapb = pb;
+		CheckSend(sp);
     }
     pb->ioResult = 1;	/* indicate that the IO is not complete */
     return 0;
diff --git a/bsd/netat/adsp_internal.h b/bsd/netat/adsp_internal.h
index 1c563d420..57a17c944 100644
--- a/bsd/netat/adsp_internal.h
+++ b/bsd/netat/adsp_internal.h
@@ -250,9 +250,6 @@ typedef struct ccb {
     ADSP_OPEN_DATA of;		/* Holds the data for the open exchange */
     gref_t *gref;			/* The queue associated with the CCB */
     gbuf_t *sp_mp;
-    atlock_t lock;
-    atlock_t lockClose;
-    atlock_t lockRemove;
 } CCB, *CCBPtr;
 
 
diff --git a/bsd/netat/adsp_misc.c b/bsd/netat/adsp_misc.c
index ad0d280a6..aa68178ae 100644
--- a/bsd/netat/adsp_misc.c
+++ b/bsd/netat/adsp_misc.c
@@ -50,7 +50,6 @@
  *    Modified for MP, 1996 by Tuyen Nguyen
  *   Modified, April 9, 1997 by Tuyen Nguyen for MacOSX.
  */
-extern atlock_t adspgen_lock;
 
 
 struct qlink {
@@ -130,17 +129,14 @@ void* qfind_m(qhead, match, compare_fnx)
 	void  *match;
 	ProcPtr compare_fnx;
 {
-	int s;
 	CCBPtr queue_item = qhead;
 
-	ATDISABLE(s, adspgen_lock);
 	while (queue_item) { 
 		if ((*compare_fnx)(queue_item,match)) 
 			break;
 		
 		queue_item = queue_item->ccbLink;
 	}
-	ATENABLE(s, adspgen_lock);
 
 	return (queue_item);
 }
diff --git a/bsd/netat/adsp_reset.c b/bsd/netat/adsp_reset.c
index 342a18c81..11eaf6e99 100644
--- a/bsd/netat/adsp_reset.c
+++ b/bsd/netat/adsp_reset.c
@@ -78,10 +78,8 @@ int RXFReset(sp, f)		/* (CCBPtr sp, ADSP_FRAMEPtr f) */
     unsigned int hi;
     register gbuf_t *mp;
     register struct adspcmd *pb;
-    int s;
 
-    ATDISABLE(s, sp->lock);
-    pktFirstByteSeq = netdw(UAL_VALUE(f->pktFirstByteSeq));
+    pktFirstByteSeq = UAL_VALUE_NTOH(f->pktFirstByteSeq);
     
     hi = sp->recvSeq + CalcRecvWdw(sp);
 
@@ -120,7 +118,6 @@ int RXFReset(sp, f)		/* (CCBPtr sp, ADSP_FRAMEPtr f) */
 	sp->callSend = 1;
     }
 
-    ATENABLE(s, sp->lock);
     return 0;
 }
 
@@ -143,13 +140,11 @@ int RXFResetAck(sp, f)		/* (CCBPtr sp, ADSP_FRAMEPtr f) */
     ADSP_FRAMEPtr f;
 {
     unsigned int  PktNextRecvSeq;
-    int s;
 
     if (sp->frpb == 0)		/* Not expecting frwd reset Ack packet */
 	return 1;
 
-    ATDISABLE(s, sp->lock);
-    PktNextRecvSeq = netdw(UAL_VALUE(f->pktNextRecvSeq));
+    PktNextRecvSeq = UAL_VALUE_NTOH(f->pktNextRecvSeq);
 
     if (BETWEEN(sp->sendSeq, PktNextRecvSeq, sp->sendWdwSeq+1)) {
 	struct adspcmd *pb;
@@ -176,7 +171,6 @@ int RXFResetAck(sp, f)		/* (CCBPtr sp, ADSP_FRAMEPtr f) */
 	}
     }
 
-    ATENABLE(s, sp->lock);
     return 0;
 }
 
@@ -199,7 +193,6 @@ int adspReset(sp, pb)		/* (DSPPBPtr pb) */
     CCBPtr sp;
     struct adspcmd *pb;
 {
-    int s;
     register gbuf_t *mp;
     register struct adspcmd *rpb;
 	
@@ -213,7 +206,6 @@ int adspReset(sp, pb)		/* (DSPPBPtr pb) */
 	return EINVAL;
     }
 	
-    ATDISABLE(s, sp->lock);
 
     while (mp = sp->sbuf_mb) { /* clear the send queue */
 	sp->sbuf_mb = gbuf_next(mp);
@@ -241,7 +233,6 @@ int adspReset(sp, pb)		/* (DSPPBPtr pb) */
 				 * bookkeeping for it.  yetch! */
 	    adspioc_ack(0, pb->ioc, pb->gref);
     }
-    ATENABLE(s, sp->lock);
 
     CheckSend(sp);
     return STR_IGNORE;
diff --git a/bsd/netat/adsp_stream.c b/bsd/netat/adsp_stream.c
index cd23ec2ac..c749294bd 100644
--- a/bsd/netat/adsp_stream.c
+++ b/bsd/netat/adsp_stream.c
@@ -72,8 +72,6 @@ unsigned char adspAssignSocket();
 int adspallocate(), adsprelease();
 int adspInited = 0;
 
-atlock_t adspall_lock;
-atlock_t adspgen_lock;
 GLOBAL adspGlobal;
 
 /**********/
@@ -92,17 +90,14 @@ void adsp_input(mp)
 	gref_t *gref;
 	CCBPtr sp;
 	at_ddp_t *p;
-	int s, l;
 	gbuf_t *mb;
 
 	switch (gbuf_type(mp)) {
 	case MSG_DATA:
 		p = (at_ddp_t *)gbuf_rptr(mp);
-		ATDISABLE(s, adspall_lock);
 		sp = adsp_inputQ[p->dst_socket];
 		if ((sp == 0) || (sp->gref==0) || (sp->state==sClosed))
 		{
-			ATENABLE(s, adspall_lock);
 			gbuf_freem(mp);
 			return;
 		}
@@ -115,7 +110,6 @@ void adsp_input(mp)
 			} while ((sp = sp->otccbLink) != 0);
 			if (sp == 0)
 			{
-				ATENABLE(s, adspall_lock);
 				gbuf_freem(mp);
 				return;
 			}
@@ -127,12 +121,9 @@ void adsp_input(mp)
 				gbuf_next(mb) = mp;
 			} else
 				sp->deferred_mb = mp;
-			ATENABLE(s, adspall_lock);
 			return;
 		}
-		ATDISABLE(l, sp->lockRemove);
 		sp->lockFlag = 1;
-		ATENABLE(l, adspall_lock);
 		while (mp) {
 			adsp_rput(sp->gref, mp);
 			if ((mp = sp->deferred_mb) != 0) {
@@ -141,7 +132,6 @@ void adsp_input(mp)
 			}
 		}
 		sp->lockFlag = 0;
-		ATENABLE(s, sp->lockRemove);
 		return;
 
 	case MSG_IOCACK:
@@ -191,7 +181,7 @@ int adsp_readable(gref)
 int adsp_writeable(gref)
 	gref_t *gref;
 {
-	int s, rc;
+	int rc;
 	CCBPtr sp;
 
 	if (gref->info == 0)
@@ -207,9 +197,7 @@ int adsp_writeable(gref)
 	        return(1);
 
 	sp = (CCBPtr)gbuf_rptr(((gbuf_t *)gref->info));
-	ATDISABLE(s, sp->lock);
 	rc = CalcSendQFree(sp);
-	ATENABLE(s, sp->lock);
 
 	return rc;
 }
@@ -235,7 +223,6 @@ int adsp_open(gref)
 	gref_t *gref;
 {
     register CCBPtr sp;
-    int s;
     
     if (!adspInited)
 		adsp_init();
@@ -246,28 +233,21 @@ int adsp_open(gref)
 	sp = (CCBPtr)gbuf_rptr(((gbuf_t *)gref->info));
 	gref->readable = adsp_readable;
 	gref->writeable = adsp_writeable;
-	ATDISABLE(s, adspall_lock);
 	if ((sp->otccbLink = ccb_used_list) != 0)
 		sp->otccbLink->ccbLink = sp;
 	ccb_used_list = sp;
-	ATENABLE(s, adspall_lock);
 	return 0;
 }
 
 int adsp_close(gref)
 	gref_t *gref;
 {
-  int s, l;
   unsigned char localSocket;
 
   /* make sure we've not yet removed the CCB (e.g., due to TrashSession) */
-  ATDISABLE(l, adspgen_lock);
   if (gref->info) {
 	CCBPtr sp = (CCBPtr)gbuf_rptr(((gbuf_t *)gref->info));
-	ATDISABLE(s, sp->lock);
-	ATENABLE(s, adspgen_lock);
 	localSocket = sp->localSocket;
-	ATENABLE(l, sp->lock);
 	if (localSocket)
 		adspRelease(gref);
 	else
@@ -276,7 +256,6 @@ int adsp_close(gref)
 		gbuf_freeb((gbuf_t *)gref->info);
 	}
   } else
-	ATENABLE(l, adspgen_lock);
     return 0;
 }
 
@@ -342,7 +321,6 @@ int adsp_wput(gref, mp)
     gbuf_t *mp;
 {
 	int rc;
-	int s;
 	gbuf_t *xm;
 	ioc_t *iocbp;
 	CCBPtr sp;
@@ -364,17 +342,14 @@ int adsp_wput(gref, mp)
 				adsp_iocnak(gref, mp, EINVAL);
 			}
 			v = *(unsigned char *)gbuf_rptr(gbuf_cont(mp));
-			ATDISABLE(s, adspall_lock);
 			if ( (v != 0)
 			     && ((v > DDP_SOCKET_LAST) || (v < 2)
 				 || ddp_socket_inuse(v, DDP_ADSP))) {
-				ATENABLE(s, adspall_lock);
 				iocbp->ioc_rval = -1;
 				adsp_iocnak(gref, mp, EINVAL);
 			}
 			else {
 				if (v == 0) {
-					ATENABLE(s, adspall_lock);
 					if ((v = adspAssignSocket(gref, 0)) == 0) {
 						iocbp->ioc_rval = -1;
 						adsp_iocnak(gref, mp, EINVAL);
@@ -384,7 +359,6 @@ int adsp_wput(gref, mp)
 					adsp_inputC[v] = 1;
 					adsp_inputQ[v] = sp;
 					adsp_pidM[v] = sp->pid;
-					ATENABLE(s, adspall_lock);
 					adsp_dequeue_ccb(sp);
 				}
 				*(unsigned char *)gbuf_rptr(gbuf_cont(mp)) = v;
@@ -456,9 +430,7 @@ int adsp_wput(gref, mp)
 	if (!gref->info)
 	    gbuf_freem(mp);
 	else {
-	    ATDISABLE(s, sp->lockClose);
 	    rc = adspWriteHandler(gref, mp);
-	    ATENABLE(s, sp->lockClose);
 
 	    switch (rc) {
 	    case STR_PUTNEXT:
@@ -543,12 +515,10 @@ adspAssignSocket(gref, flag)
 {
 	unsigned char sVal, sMax, sMin, sSav, inputC;
 	CCBPtr sp;
-	int s;
 
 	sMax = flag ? DDP_SOCKET_LAST-46 : DDP_SOCKET_LAST-6;
 	sMin = DDP_SOCKET_1st_DYNAMIC;
 
-	ATDISABLE(s, adspall_lock);
 	for (inputC=255, sVal=sMax; sVal >= sMin; sVal--) {
 		if (!ddp_socket_inuse(sVal, DDP_ADSP))
 			break;
@@ -563,22 +533,17 @@ adspAssignSocket(gref, flag)
 		}
 	}
 	if (sVal < sMin) {
-		if (!flag || (inputC == 255)) {
-			ATENABLE(s, adspall_lock);
+		if (!flag || (inputC == 255))
 			return 0;
-		}
 		sVal = sSav;
 	}
 	sp = (CCBPtr)gbuf_rptr(((gbuf_t *)gref->info));
-	ATENABLE(s, adspall_lock);
 	adsp_dequeue_ccb(sp);
-	ATDISABLE(s, adspall_lock);
 	adsp_inputC[sVal]++;
 	sp->otccbLink = adsp_inputQ[sVal];
 	adsp_inputQ[sVal] = sp;
 	if (!flag)
 		adsp_pidM[sVal] = sp->pid;
-	ATENABLE(s, adspall_lock);
 	return sVal;
 }
 
@@ -590,11 +555,9 @@ adspDeassignSocket(sp)
 	CCBPtr curr_sp;
 	CCBPtr prev_sp;
 	int pid = 0;
-	int s, l;
 
 	dPrintf(D_M_ADSP, D_L_TRACE, ("adspDeassignSocket: pid=%d,s=%d\n",
 		sp->pid, sp->localSocket));
-	ATDISABLE(s, adspall_lock);
 	sVal = sp->localSocket;
 	if ((curr_sp = adsp_inputQ[sVal]) != 0) {
 		prev_sp = 0;
@@ -603,12 +566,10 @@ adspDeassignSocket(sp)
 			curr_sp = curr_sp->otccbLink;
 		}
 		if (curr_sp) {
-			ATDISABLE(l, sp->lockRemove);
 			if (prev_sp)
 				prev_sp->otccbLink = sp->otccbLink;
 			else
 				adsp_inputQ[sVal] = sp->otccbLink;
-			ATENABLE(l, sp->lockRemove);
 			if (adsp_inputQ[sVal])
 				adsp_inputC[sVal]--;
 			else {
@@ -619,11 +580,9 @@ adspDeassignSocket(sp)
 			sp->ccbLink = 0;
 			sp->otccbLink = 0;
 			sp->localSocket = 0;
-			ATENABLE(s, adspall_lock);
 		    return pid ? 0 : 1;
 		}
 	}
-	ATENABLE(s, adspall_lock);
 
 	dPrintf(D_M_ADSP, D_L_ERROR, 
 		("adspDeassignSocket: closing, no CCB block, trouble ahead\n"));
@@ -637,9 +596,7 @@ void
 adsp_dequeue_ccb(sp)
 	CCB *sp;
 {
-	int s;
 
-	ATDISABLE(s, adspall_lock);
 	if (sp == ccb_used_list) {
 		if ((ccb_used_list = sp->otccbLink) != 0)
 			sp->otccbLink->ccbLink = 0;
@@ -650,7 +607,6 @@ adsp_dequeue_ccb(sp)
 
 	sp->otccbLink = 0;
 	sp->ccbLink = 0;
-	ATENABLE(s, adspall_lock);
 }
 
 void SndMsgUp(gref, mp)
diff --git a/bsd/netat/appletalk.h b/bsd/netat/appletalk.h
index d98d75454..db08c693d 100644
--- a/bsd/netat/appletalk.h
+++ b/bsd/netat/appletalk.h
@@ -78,10 +78,22 @@ struct atalk_addr {
 #define	UAL_UAL(x,y)	*(unsigned long *) &(x[0]) = *(unsigned long *) &(y[0])
 #define	UAL_VALUE(x)	(*(unsigned long *) &(x[0]))
 
+/* Macros to assign unaligned fields with byte swapping */
+#define	UAS_ASSIGN_HTON(x,s)	*(unsigned short *) &(x[0]) = htons((unsigned short) (s))
+#define	UAS_ASSIGN_NTOH(x,s)	*(unsigned short *) &(x[0]) = ntohs((unsigned short) (s))
+#define	UAS_VALUE_HTON(x)		htons((*(unsigned short *) &(x[0])))
+#define	UAS_VALUE_NTOH(x)		ntohs((*(unsigned short *) &(x[0])))
+#define	UAL_ASSIGN_HTON(x,l)	*(unsigned long *) &(x[0]) = htonl((unsigned long) (l))
+#define	UAL_ASSIGN_NTOH(x,l)	*(unsigned long *) &(x[0]) = ntohl((unsigned long) (l))
+#define	UAL_VALUE_HTON(x)		htonl((*(unsigned long *) &(x[0])))
+#define	UAL_VALUE_NTOH(x)		ntohl((*(unsigned long *) &(x[0])))
+
 /* Macros to manipulate at_net variables */
-#define	NET_ASSIGN(x,s)	*(unsigned short *)&(x[0]) = (unsigned short)(s)
-#define	NET_NET(x, y)	*(unsigned short *)&(x[0]) = *(unsigned short *)&(y[0])
-#define	NET_VALUE(x)	(*(unsigned short *) &(x[0]))
+#define	NET_ASSIGN(x,s)		*(unsigned short *)&(x[0]) = htons((unsigned short)(s))
+#define	NET_ASSIGN_NOSWAP(x,s)		*(unsigned short *)&(x[0]) = (unsigned short)(s)
+#define	NET_NET(x, y)		*(unsigned short *)&(x[0]) = *(unsigned short *)&(y[0])
+#define	NET_VALUE(x)		ntohs((*(unsigned short *) &(x[0])))
+#define	NET_VALUE_NOSWAP(x)		(*(unsigned short *) &(x[0]))
 #define ATALK_ASSIGN(a, net, node, unused ) \
   a.atalk_unused = unused; a.atalk_node = node; NET_ASSIGN(a.atalk_net, net)
 
@@ -267,7 +279,6 @@ typedef struct {
 #define	IFID_HOME	1 		/* home port in ifID_table */
 
 #define	ATALK_VALUE(a)		((*(u_long *) &(a))&0x00ffffff)
-#define	ATALK_EQUAL(a, b)	(ATALK_VALUE(a) == ATALK_VALUE(b))
 
 #define VERSION_LENGTH		80	/* length of version string */
 
diff --git a/bsd/netat/asp_proto.c b/bsd/netat/asp_proto.c
index 29e6d6df7..517f83dfd 100644
--- a/bsd/netat/asp_proto.c
+++ b/bsd/netat/asp_proto.c
@@ -121,7 +121,6 @@ static unsigned char scb_tmo_cnt;
 asp_scb_t *scb_used_list;
 static asp_scb_t *scb_tmo_list;
 asp_scb_t *scb_free_list;
-atlock_t aspall_lock, asptmo_lock;
 
 int
 asp_readable(gref)
@@ -147,7 +146,6 @@ asp_init()
 int asp_open(gref)
 	gref_t *gref;
 {
-	int s;
 	asp_scb_t *scb;
 
 	/*
@@ -172,11 +170,9 @@ int asp_open(gref)
 	scb->gref = gref;
 	scb->session_timer = DEF_SESS_TMO;
 	scb->cmd_retry = asp_def_retry;
-	ATDISABLE(s, aspall_lock);
 	if ((scb->next_scb = scb_used_list) != 0)
 		scb->next_scb->prev_scb = scb;
 	scb_used_list = scb;
-	ATENABLE(s, aspall_lock);
 
 	/*
 	 * return success
@@ -192,7 +188,6 @@ int
 asp_close(gref)
 	gref_t *gref;
 {
-	int s;
 	unsigned char sock_num;
 	asp_scb_t *scb, *new_scb;
 	gbuf_t *m;
@@ -206,17 +201,14 @@ asp_close(gref)
 		 * send the CloseSess response to peer
 		 */
 		if (gbuf_type(scb->sess_ioc) != MSG_PROTO) {
-			ATDISABLE(s, scb->lock);
 			m = scb->sess_ioc;
 			scb->sess_ioc = gbuf_next(m);
-			ATENABLE(s, scb->lock);
 			atp_send_rsp(scb->gref, m, TRUE);
 		}
 	}
 
 	if (scb->atp_state) {
 		sock_num = scb->loc_addr.socket;
-		ATDISABLE(s, aspall_lock);
 		if ((scb->dflag != 1) && scb->stat_msg) {
 			untimeout(atp_retry_req, scb->stat_msg);
 			gbuf_freem(scb->stat_msg);
@@ -225,7 +217,6 @@ asp_close(gref)
 		if (asp_scbQ[sock_num]->next_scb == 0) {
 			asp_scbQ[sock_num] = 0;
 			asp_inpC[sock_num] = 0;
-			ATENABLE(s, aspall_lock);
 			dPrintf(D_M_ASP, D_L_INFO,
 			("         : atp_close(), loc=%d\n", scb->loc_addr.socket));
 			atp_close(gref, 0);
@@ -243,7 +234,6 @@ asp_close(gref)
 					scb->next_scb->prev_scb = scb->prev_scb;
 			}
 			scb->next_scb = 0;
-			ATENABLE(s, aspall_lock);
 		}
 	} else
 		asp_dequeue_scb(scb);
@@ -251,7 +241,6 @@ asp_close(gref)
 	/*
 	 * free all allocated blocks if any
 	 */
-	ATDISABLE(s, scb->lock);
 	if (scb->stat_msg) {
 		gbuf_freem(scb->stat_msg);
 		scb->stat_msg = 0;
@@ -266,7 +255,6 @@ asp_close(gref)
 	}
 
 	scb->rem_addr.node = 0;
-	ATENABLE(s, scb->lock);
 
 	/*
 	 * stop all timers
@@ -387,7 +375,7 @@ int asp_wput(gref, m)
 	gref_t *gref;
 	gbuf_t *m;
 {
-	int s, err;
+	int err;
 	unsigned char sockSav, sock_num;
 	gbuf_t *mioc, *mdata;
 	ioc_t *iocbp;
@@ -552,7 +540,6 @@ int asp_wput(gref, m)
 		if ((sock_num = (at_socket)atp_bind(gref, 0, &sockSav)) == 0) {
 			atp_close(gref, 0);
 			asp_dequeue_scb(scb);
-			ATDISABLE(s, aspall_lock);
 			sock_num = sockSav;
 			scb->loc_addr.socket = sock_num;
 			for (curr_scb = asp_scbQ[sock_num];
@@ -560,14 +547,11 @@ int asp_wput(gref, m)
 			scb->prev_scb = curr_scb;
 			curr_scb->next_scb = scb;
 			scb->atp_state = curr_scb->atp_state;
-			ATENABLE(s, aspall_lock);
 		} else {
 			asp_dequeue_scb(scb);
-			ATDISABLE(s, aspall_lock);
 			scb->loc_addr.socket = sock_num;
 			asp_scbQ[sock_num] = scb;
 			scb->atp_state->dflag = scb->dflag;
-			ATENABLE(s, aspall_lock);
 		}
 		gref->info = (void *)scb;
 		asp_inpC[sock_num]++;
@@ -608,7 +592,7 @@ int asp_wput(gref, m)
 		scb->session_timer = open_cmd->SessionTimer;
 		aw.func = ASPFUNC_OpenSess;
 		aw.param1 = scb->loc_addr.socket;
-		aw.param2 = ASP_Version;
+		aw.param2 = htons(ASP_Version);
 		scb->ioc_wait = (unsigned char)(iocbp->ioc_cmd & 0xff);
 		iocbp->ioc_cmd = AT_ATP_ISSUE_REQUEST_DEF;
 		asp_send_req(gref, mioc, &open_cmd->SLSEntityIdentifier,
@@ -854,7 +838,6 @@ asp_accept(scb, sess_scb, m)
 	asp_scb_t *sess_scb;
 	gbuf_t *m;
 {
-	int s;
 	gbuf_t *mdata;
 	at_ddp_t *ddp;
 	at_atp_t *atp;
@@ -886,9 +869,7 @@ asp_accept(scb, sess_scb, m)
 	awp->param1 = sess_scb->sess_id;
 	awp->param2 = 0;
 	gbuf_freeb(m);
-	ATDISABLE(s, scb->lock);
 	scb->sess_ioc = gbuf_next(mdata);
-	ATENABLE(s, scb->lock);
 	gbuf_next(mdata) = 0;
 	asp_timout(asp_hangup, sess_scb, sess_scb->session_timer);
 	atp_send_rsp(scb->gref, mdata, TRUE);
@@ -917,11 +898,9 @@ void asp_clock_locked(arg)
 void asp_clock(arg)
 	void *arg;
 {
-	int s;
 	asp_scb_t *scb;
 	void (*tmo_func)();
 
-	ATDISABLE(s, asptmo_lock);
 	if (scb_tmo_list)
 		scb_tmo_list->tmo_delta--;
 	while (((scb = scb_tmo_list) != 0) && (scb_tmo_list->tmo_delta == 0)) {
@@ -929,12 +908,9 @@ void asp_clock(arg)
 			scb_tmo_list->prev_tmo = 0;
 		if ((tmo_func = scb->tmo_func) != 0) {
 			scb->tmo_func = 0;
-			ATENABLE(s, asptmo_lock);
 			(*tmo_func)(scb);
-			ATDISABLE(s, asptmo_lock);
 		}
 	}
-	ATENABLE(s, asptmo_lock);
 
 	if (++scb_tmo_cnt == 0) scb_tmo_cnt++;
 	timeout(asp_clock_locked, (void *)arg, (1<<SESS_TMO_RES)*TICKS_PER_SEC);
@@ -949,7 +925,6 @@ asp_ack_reply(gref, mioc)
 	register gref_t *gref;
 	register gbuf_t *mioc;
 {
-	int s;
 	int len, msize, nbds;
 	register gbuf_t *mdata, *m, *mx;
 	struct atpBDS *atpBDS;
@@ -1089,7 +1064,7 @@ asp_ack_reply(gref, mioc)
 				 */
 				awp->func = 0;
 				awp->param1 = 0;
-				awp->param2 = (unsigned short)ASPERR_BadVersNum;
+				awp->param2 = htons((unsigned short)ASPERR_BadVersNum);
 				dPrintf(D_M_ASP, D_L_INFO,
 					("             : version=%d\n",
 					ASPERR_BadVersNum));
@@ -1106,7 +1081,6 @@ asp_ack_reply(gref, mioc)
 			/*
 			 * queue the connection request
 			 */
-			ATDISABLE(s, scb->lock);
 			gbuf_next(mdata) = 0;
 			if ((m = scb->sess_ioc) == 0) {
 				scb->sess_ioc = mdata;
@@ -1119,7 +1093,6 @@ asp_ack_reply(gref, mioc)
 					m = gbuf_next(m);
 				gbuf_next(m) = mdata;
 			}
-			ATENABLE(s, scb->lock);
 			dPrintf(D_M_ASP, D_L_INFO,
 				("             : QUEUE connect request\n"));
 
@@ -1128,7 +1101,7 @@ asp_ack_reply(gref, mioc)
 		case ASPFUNC_Command:
 		case ASPFUNC_Write:
 			if ( (scb->sess_id != awp->param1)
-			     || (scb->rcv_seq_num != awp->param2)
+			     || (scb->rcv_seq_num != ntohs(awp->param2))
 			     || BAD_REMADDR(rem_addr) ) {
 				char era[8], ra[8];
 				sprintf(era,"%d.%d", scb->rem_addr.node,scb->rem_addr.socket);
@@ -1150,12 +1123,11 @@ asp_ack_reply(gref, mioc)
 			command_ind = (asp_command_ind_t *)gbuf_rptr(mioc);
 			command_ind->Primitive = (int)awp->func;
 			command_ind->ReqRefNum =
-				*(unsigned short *)atp->tid;
+				ntohs(*(unsigned short *)atp->tid);
 			command_ind->ReqType = awp->func;
 
 			mdata = gbuf_strip(mdata);
 			gbuf_cont(mioc) = mdata;
-			ATDISABLE(s, scb->lock);
 			if (scb->req_flag) {
 				if ((mx = scb->req_msgq) != 0) {
 					while (gbuf_next(mx))
@@ -1163,10 +1135,8 @@ asp_ack_reply(gref, mioc)
 					gbuf_next(mx) = mioc;
 				} else
 					scb->req_msgq = mioc;
-				ATENABLE(s, scb->lock);
 			} else {
 				scb->req_flag = 1;
-				ATENABLE(s, scb->lock);
 				asp_putnext(scb->gref, mioc);
 			}
 			goto l_done;
@@ -1183,7 +1153,7 @@ asp_ack_reply(gref, mioc)
 			command_ind = (asp_command_ind_t *)gbuf_rptr(mioc);
 			command_ind->Primitive = (int)awp->func;
 			command_ind->ReqRefNum =
-				*(unsigned short *)atp->tid;
+				ntohs(*(unsigned short *)atp->tid);
 			command_ind->ReqType = awp->func;
 
 			mdata = gbuf_strip(mdata);
@@ -1252,12 +1222,10 @@ asp_ack_reply(gref, mioc)
 				scb->rem_addr.socket));
 
 			gbuf_next(mdata) = 0;
-			ATDISABLE(s, scb->lock);
 			if (scb->sess_ioc)
 				gbuf_freel(scb->sess_ioc);
 			scb->sess_ioc = mdata;
 			scb->state = ASPSTATE_Close;
-			ATENABLE(s, scb->lock);
 
 			/*
 			 * notify upstream of the CloseSess from peer
@@ -1276,7 +1244,7 @@ asp_ack_reply(gref, mioc)
 			command_ind = (asp_command_ind_t *)gbuf_rptr(mioc);
 			command_ind->Primitive = (int)awp->func;
 			command_ind->ReqRefNum =
-				*(unsigned short *)atp->tid;
+				ntohs(*(unsigned short *)atp->tid);
 			command_ind->ReqType = awp->func;
 			scb->attn_tid = *(unsigned short *)atp->tid;
 			scb->attn_flag = 1;
@@ -1322,12 +1290,10 @@ asp_ack_reply(gref, mioc)
 		case ASPSTATE_Idle:
 			scb->rem_addr.node = 0;
 			gbuf_freem(mioc);
-			ATDISABLE(s, scb->lock);
 			if (scb->get_wait)
 				wakeup(&scb->event);
 			else
 				atalk_notify_sel(gref);
-			ATENABLE(s, scb->lock);
 			return;
 
 		case ASPSTATE_WaitingForGetStatusRsp:
@@ -1396,7 +1362,7 @@ asp_ack_reply(gref, mioc)
 			atpBDS = (struct atpBDS *)gbuf_rptr(mx);
 			cmdreply_ind = (asp_cmdreply_ind_t *)gbuf_rptr(mioc);
 			cmdreply_ind->Primitive = ASPFUNC_CmdReply;
-			cmdreply_ind->CmdResult = *(int *)atpBDS->bdsUserData;
+			cmdreply_ind->CmdResult = ntohl(*(int *)atpBDS->bdsUserData);
 			gbuf_wset(mioc,sizeof(asp_cmdreply_ind_t));
 			gbuf_freeb(mx);
 			asp_putnext(scb->gref, mioc);
@@ -1592,9 +1558,7 @@ StaticProc void
 asp_dequeue_scb(scb)
 	asp_scb_t *scb;
 {
-	int s;
 
-	ATDISABLE(s, aspall_lock);
 	if (scb == scb_used_list) {
 		if ((scb_used_list = scb->next_scb) != 0)
 			scb->next_scb->prev_scb = 0;
@@ -1602,7 +1566,6 @@ asp_dequeue_scb(scb)
 		if ((scb->prev_scb->next_scb = scb->next_scb) != 0)
 			scb->next_scb->prev_scb = scb->prev_scb;
 	}
-	ATENABLE(s, aspall_lock);
 
 	scb->next_scb = 0;
 	scb->prev_scb = 0;
@@ -1616,11 +1579,9 @@ asp_find_scb(sock_num, rem_addr)
 	unsigned char sock_num;
 	at_inet_t *rem_addr;
 {
-	int s;
 	asp_scb_t *scb;
 	asp_scb_t *alt_scb = 0;
 
-	ATDISABLE(s, aspall_lock);
 	for (scb = asp_scbQ[sock_num]; scb; scb = scb->next_scb) {
 		if ((scb->rem_addr.net == rem_addr->net)
 			&& (scb->rem_addr.node == rem_addr->node)) {
@@ -1640,7 +1601,6 @@ asp_find_scb(sock_num, rem_addr)
 			rem_addr->node,
 			rem_addr->socket));
 	}
-	ATENABLE(s, aspall_lock);
 
 	return scb;
 }
@@ -1654,7 +1614,6 @@ asp_timout(func, scb, seconds)
 	register asp_scb_t *scb;
 	int seconds;
 {
-	int s;
 	unsigned char sum;
 	register asp_scb_t *curr_scb, *prev_scb;
 
@@ -1665,11 +1624,9 @@ asp_timout(func, scb, seconds)
 	scb->tmo_delta = (seconds>>SESS_TMO_RES);
 	scb->tmo_cnt = scb_tmo_cnt;
 
-	ATDISABLE(s, asptmo_lock);
 	if (scb_tmo_list == 0) {
 		scb->next_tmo = scb->prev_tmo = 0;
 		scb_tmo_list = scb;
-		ATENABLE(s, asptmo_lock);
 		return;
 	}
 
@@ -1703,7 +1660,6 @@ asp_timout(func, scb, seconds)
 		scb_tmo_list->prev_tmo = scb;
 		scb_tmo_list = scb;
 	}
-	ATENABLE(s, asptmo_lock);
 }
 
 /*
@@ -1714,12 +1670,10 @@ asp_untimout(func, scb)
 	void (*func)();
 	register asp_scb_t *scb;
 {
-	int s;
 
 	if ((scb->tmo_cnt == scb_tmo_cnt) || (scb->tmo_func == 0))
 		return;
 
-	ATDISABLE(s, asptmo_lock);
 	if (scb_tmo_list == scb) {
 		if ((scb_tmo_list = scb->next_tmo) != 0) {
 			scb_tmo_list->prev_tmo = 0;
@@ -1733,7 +1687,6 @@ asp_untimout(func, scb)
 		scb->prev_tmo = 0;
 	}
 	scb->tmo_func = 0;
-	ATENABLE(s, asptmo_lock);
 }
 
 /*
@@ -1748,7 +1701,6 @@ asp_hangup(scb)
 	/*
 	 * set the state to Close
 	 */
-	ATDISABLE(s, scb->lock);
 	scb->state = ASPSTATE_Close;
 	if (scb->tickle_tid) {
 		atp_cancel_req(scb->gref, (unsigned int)scb->tickle_tid);
@@ -1759,15 +1711,11 @@ asp_hangup(scb)
 	 * notify upstream of the hangup
 	 */
 	if (scb->rem_addr.node) {
-		if (scb->get_wait) {
+		if (scb->get_wait)
 			wakeup(&scb->event);
-			ATENABLE(s, scb->lock);
-		} else {
-			ATENABLE(s, scb->lock);
+		else
 			atalk_notify_sel(scb->gref);
-		}
-	} else
-		ATENABLE(s, scb->lock);
+	}
 }
 
 StaticProc void
@@ -1810,17 +1758,13 @@ asp_iocnak(gref, mioc, err)
 StaticProc asp_scb_t *
 asp_scb_alloc()
 {
-	int s, i;
+	int i;
 	gbuf_t *m;
 	asp_scb_t *scb, *scb_array;
 
-	ATDISABLE(s, aspall_lock);
 	if (scb_free_list == 0) {
 		if ((m = gbuf_alloc(SCBS_PER_BLK*sizeof(asp_scb_t), PRI_MED)) == 0)
-		{
-			ATENABLE(s, aspall_lock);
 			return (asp_scb_t *)0;
-		}
 		bzero((char *)gbuf_rptr(m), SCBS_PER_BLK*sizeof(asp_scb_t));
 		gbuf_cont(m) = scb_resource_m;
 		scb_resource_m = m;
@@ -1833,9 +1777,6 @@ asp_scb_alloc()
 
 	scb = scb_free_list;
 	scb_free_list = scb->next_scb;
-	ATENABLE(s, aspall_lock);
-	ATLOCKINIT(scb->lock);
-	ATLOCKINIT(scb->delay_lock);
 	ATEVENTINIT(scb->event);
 	ATEVENTINIT(scb->delay_event);
 
@@ -1849,13 +1790,10 @@ StaticProc void
 asp_scb_free(scb)
 	asp_scb_t *scb;
 {
-	int s;
 
 	bzero((char *)scb, sizeof(asp_scb_t));
-	ATDISABLE(s, aspall_lock);
 	scb->next_scb = scb_free_list;
 	scb_free_list = scb;
-	ATENABLE(s, aspall_lock);
 }
 
 /*
@@ -1866,7 +1804,6 @@ asp_putnext(gref, mproto)
 	gref_t *gref;
 	gbuf_t *mproto;
 {
-	int s;
 	gbuf_t *m;
 	asp_scb_t *scb;
 
@@ -1875,7 +1812,6 @@ asp_putnext(gref, mproto)
 	/*
 	 * queue the message.
 	 */
-	ATDISABLE(s, scb->lock);
 	gbuf_next(mproto) = 0;
 	if ((m = scb->sess_ioc) == 0)
 		scb->sess_ioc = mproto;
@@ -1888,14 +1824,11 @@ asp_putnext(gref, mproto)
 	if (scb->rcv_cnt >= MAX_RCV_CNT)
 		scb->snd_stop = 1;
 
-	if (scb->get_wait) {
+	if (scb->get_wait)
 		wakeup(&scb->event);
-		ATENABLE(s, scb->lock);
-	} else if (mproto == scb->sess_ioc) {
-		ATENABLE(s, scb->lock);
+	else if (mproto == scb->sess_ioc)
 		atalk_notify_sel(gref);
-	} else
-		ATENABLE(s, scb->lock);
+
 } /* asp_putnext */
 
 /*
@@ -1919,7 +1852,7 @@ asp_putnext(gref, mproto)
 
 int ASPputmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t *mreq, int flags, int *errp)
 {
-    int s, i, err, len, offset, remain, size, copy_len;
+    int i, err, len, offset, remain, size, copy_len;
     gbuf_t *mioc, *mdata, *mx, *m0;
     ioc_t *iocbp;
     strbuf_t ctlbuf;
@@ -2103,7 +2036,7 @@ int ASPputmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t *mreq, in
         awp = (asp_word_t *)atp->user_bytes;
         awp->func = (unsigned char)Primitive;
         awp->param1 = scb->sess_id;
-        awp->param2 = scb->snd_seq_num;
+        awp->param2 = htons(scb->snd_seq_num);
         iocbp->ioc_private = (void *)scb;
         iocbp->ioc_count = gbuf_len(mdata);
         iocbp->ioc_rval = 0;
@@ -2112,7 +2045,6 @@ int ASPputmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t *mreq, in
         /*
          * send the command/write/write_continue/attention request
          */
-        ATDISABLE(s, scb->lock);
         switch (awp->func) {
         case ASPFUNC_Command:
             scb->state = ASPSTATE_WaitingForCommandRsp;
@@ -2122,7 +2054,7 @@ int ASPputmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t *mreq, in
             break;
         case ASPFUNC_WriteContinue:
             scb->state = ASPSTATE_WaitingForWriteContinueRsp;
-            awp->param2 = scb->wrt_seq_num;
+            awp->param2 = htons(scb->wrt_seq_num);
             break;
         case ASPFUNC_Attention:
             scb->state = ASPSTATE_WaitingForCommandRsp;
@@ -2130,10 +2062,9 @@ int ASPputmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t *mreq, in
             atp->xo_relt = 0;
             atp->bitmap = 0x01;
             gbuf_wdec(mdata,2);
-            awp->param2 = *(unsigned short *)gbuf_wptr(mdata);
+            awp->param2 = htons(*(unsigned short *)gbuf_wptr(mdata));
             break;
         }
-        ATENABLE(s, scb->lock);
         dPrintf(D_M_ASP,D_L_INFO,
             ("ASPputmsg: %s, loc=%d, rem=%x.%x.%d\n",
              (awp->func == ASPFUNC_Command ? "CommandReq" :
@@ -2146,17 +2077,14 @@ int ASPputmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t *mreq, in
 
     case ASPFUNC_CmdReply:
 
-        ATDISABLE(s, scb->lock);
         if (scb->req_msgq) {
             mx = scb->req_msgq;
             scb->req_msgq = gbuf_next(mx);
             gbuf_next(mx) = 0;
-            ATENABLE(s, scb->lock);
             asp_putnext(scb->gref, mx);
-        } else {
+        } else
             scb->req_flag = 0;
-            ATENABLE(s, scb->lock);
-        }
+
         result = primitives->CmdReplyReq.CmdResult;
         tid = primitives->CmdReplyReq.ReqRefNum;
 
@@ -2174,7 +2102,7 @@ int ASPputmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t *mreq, in
         ddp->dst_socket = scb->reply_socket;
         ddp->type = DDP_ATP;
         UAS_ASSIGN(ddp->checksum, 0);
-        UAS_ASSIGN(atp->tid, tid);
+        UAS_ASSIGN(atp->tid, htons(tid));
         if (scb->attn_flag && (tid == scb->attn_tid)) {
            scb->attn_flag = 0;
             atp->xo = 0;
@@ -2221,7 +2149,7 @@ l_err:
 /* bms:  make this callable from kernel.  reply date is passed back as a mbuf chain in *mreply  */
 int ASPgetmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t **mreply, int *flags, int *errp)
 {
-    int err, s, len, sum, rval;
+    int err, len, sum, rval;
     gbuf_t *mproto, *mdata;
     strbuf_t ctlbuf;
     strbuf_t datbuf;
@@ -2237,11 +2165,8 @@ int ASPgetmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t **mreply,
 		return -1;
 	}
 
-    ATDISABLE(s, scb->lock);
-    if (scb->state == ASPSTATE_Close) {
-        ATENABLE(s, scb->lock);
+    if (scb->state == ASPSTATE_Close)
         return 0;
-    }
 
     /*
      * get receive data
@@ -2252,25 +2177,20 @@ int ASPgetmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t **mreply,
         err = msleep(&scb->event, atalk_mutex, PSOCK | PCATCH, "aspgetmsg", 0);
         if (err != 0) {
             scb->get_wait = 0;
-            ATENABLE(s, scb->lock);
             *errp = err;
             return -1;
         }
         if (scb->state == ASPSTATE_Close) {
             scb->get_wait = 0;
-            ATENABLE(s, scb->lock);
             return 0;
         }
     }
     get_wait = scb->get_wait;
     scb->get_wait = 0;
-    if ((ctlptr == 0) && (datptr == 0)) {
-        ATENABLE(s, scb->lock);
+    if ((ctlptr == 0) && (datptr == 0))
         return 0;
-    }
     scb->sess_ioc = gbuf_next(mproto);
     mdata = gbuf_cont(mproto);
-    ATENABLE(s, scb->lock);
 
     /* last remaining use of MSG_ERROR */
     if (gbuf_type(mproto) == MSG_ERROR) {
@@ -2295,10 +2215,8 @@ int ASPgetmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t **mreply,
             goto l_err;
     }
     if ((datbuf.maxlen < 0) || (datbuf.maxlen < gbuf_msgsize(mdata))) {
-        ATDISABLE(s, scb->lock);
         gbuf_next(mproto) = scb->sess_ioc;
         scb->sess_ioc = mproto;
-        ATENABLE(s, scb->lock);
         return MOREDATA;
     }
 
@@ -2308,13 +2226,9 @@ int ASPgetmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t **mreply,
          * we're not supposed to dequeue messages in the Streams 
          * head's read queue this way; but there is no better way.
          */
-        ATDISABLE(s, scb->lock);
-        if (scb->sess_ioc == 0) {
-            ATENABLE(s, scb->lock);
-        } else {
-            ATENABLE(s, scb->lock);
+        if (scb->sess_ioc != 0)
             atalk_notify_sel(gref); 
-        }
+        
     }
 
     /*
@@ -2379,21 +2293,17 @@ int ASPgetmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t **mreply,
         gbuf_freem(mproto);
     }
 
-    ATDISABLE(s, scb->lock);
     if (scb->sess_ioc)
         scb->rcv_cnt--;
     else {
         scb->rcv_cnt = 0;
         scb->snd_stop = 0;
     }
-    ATENABLE(s, scb->lock);
     return rval;
 
 l_err:
-    ATDISABLE(s, scb->lock);
     gbuf_next(mproto) = scb->sess_ioc;
     scb->sess_ioc = mproto;
-    ATENABLE(s, scb->lock);
     *errp = err;
     return -1;
 }
diff --git a/bsd/netat/at.c b/bsd/netat/at.c
index 7c0b6ca39..526f002f2 100644
--- a/bsd/netat/at.c
+++ b/bsd/netat/at.c
@@ -77,8 +77,7 @@ struct  etalk_addr      ttalk_multicast_addr = {
   {0xC0, 0x00, 0x40, 0x00, 0x00, 0x00}};
 
 /* called only in router mode */
-static int set_zones(ifz)
-	zone_usage_t *ifz;
+static int set_zones(zone_usage_t *ifz)
 
 /* 1. adds zone to table
    2. looks up each route entry from zone list
@@ -147,7 +146,6 @@ at_control(so, cmd, data, ifp)
 
     if ((cmd & 0xffff) == 0xff99) {
 		u_long 	fixed_command;
-		char ioctl_buffer[32];
 		/* *** this is a temporary hack to get at_send_to_dev() to
 		   work with BSD-style sockets instead of the special purpose 
 		   system calls, ATsocket() and ATioctl().
@@ -392,13 +390,11 @@ at_control(so, cmd, data, ifp)
 		/* Normal case; no tuple found for this name, so insert
 		 * this tuple in the registry and return ok response.
 		 */
-		ATDISABLE(nve_lock, NVE_LOCK);
 		if ((error2 = nbp_new_nve_entry(&nve, ifID)) == 0) {
 			nbpP->addr.net = ifID->ifThisNode.s_net;
 			nbpP->addr.node = ifID->ifThisNode.s_node;
 			nbpP->unique_nbp_id = nve.unique_nbp_id;
 		}
-		ATENABLE(nve_lock, NVE_LOCK);
 
 		return(error2);
 		break;
@@ -414,16 +410,13 @@ at_control(so, cmd, data, ifp)
 
 		/* delete by id */
 		if (nbpP->unique_nbp_id) {
-			ATDISABLE(nve_lock, NVE_LOCK);
 			TAILQ_FOREACH(nve_entry, &name_registry, nve_link) {
 				if (nve_entry->unique_nbp_id == nbpP->unique_nbp_id) {
 					/* Found a match! */
 					nbp_delete_entry(nve_entry);
-					ATENABLE(nve_lock, NVE_LOCK);
 					return(0);
 				}
 			}
-			ATENABLE(nve_lock, NVE_LOCK);
 			return(EADDRNOTAVAIL);
 		}
 
@@ -443,9 +436,7 @@ at_control(so, cmd, data, ifp)
 				if ((nve_entry = nbp_find_nve(&nve)) == NULL) 
 					continue;
 
-				ATDISABLE(nve_lock, NVE_LOCK);
 				nbp_delete_entry(nve_entry);
-				ATENABLE(nve_lock, NVE_LOCK);
 				found = TRUE;
 			}
 			if (found) 
@@ -461,9 +452,7 @@ at_control(so, cmd, data, ifp)
 		/* Normal case; tuple found for this name, so delete
 		 * the entry from the registry and return ok response.
 		 */
-		ATDISABLE(nve_lock, NVE_LOCK);
 		nbp_delete_entry(nve_entry);
-		ATENABLE(nve_lock, NVE_LOCK);
 		return(0);
 
 		break;
@@ -671,21 +660,17 @@ at_control(so, cmd, data, ifp)
 #endif
 
     case SIOCSETOT: {
-        int				s;
         struct atpcb	*at_pcb, *clonedat_pcb;
         int				cloned_fd = *(int *)data;
 
-        s = splnet();		/* XXX */
         at_pcb = sotoatpcb(so);
         
         /* let's make sure it's either -1 or a valid file descriptor */
         if (cloned_fd != -1) {
             struct socket	*cloned_so;
 			error = file_socket(cloned_fd, &cloned_so);
-            if (error){
-                splx(s);	/* XXX */
+            if (error)
                 break;
-            }
             clonedat_pcb = sotoatpcb(cloned_so);
         } else {
             clonedat_pcb = NULL;
@@ -696,7 +681,6 @@ at_control(so, cmd, data, ifp)
         } else {
             at_pcb->ddp_flags = clonedat_pcb->ddp_flags;
         }
-        splx(s);		/* XXX */
 		file_drop(cloned_fd);
         break;
     }
diff --git a/bsd/netat/at_aarp.h b/bsd/netat/at_aarp.h
index f6c5a6a94..4f361feb5 100644
--- a/bsd/netat/at_aarp.h
+++ b/bsd/netat/at_aarp.h
@@ -109,7 +109,7 @@ typedef struct {
 /*************************************************/
 
 typedef struct {
-	struct atalk_addr	dest_at_addr;
+	struct atalk_addr	dest_at_addr;		/* net# in network byte order */
 	struct etalk_addr	dest_addr;
 	char                    dummy[2];       /* pad out to struct size of 32 */
 	time_t			last_time;	/* the last time that this addr
@@ -134,19 +134,22 @@ typedef struct {
 #define	AMT_HASH(a) 								\
 	((NET_VALUE(((struct atalk_addr *)&a)->atalk_net) + ((struct atalk_addr *)&a)->atalk_node) % AMT_NB)
 
+/* at_addr - net # in network byte order */
 #define	AMT_LOOK(at, at_addr, elapp) {							\
 	register n; 								\
 	at = &aarp_table[elapp->ifPort]->et_aarp_amt[AMT_HASH(at_addr) * AMT_BSIZ];	 		\
 	for (n = 0 ; ; at++) {					                \
-	    if (ATALK_EQUAL(at->dest_at_addr, at_addr))	                        \
+	    if (at->dest_at_addr.atalk_node == (at_addr).atalk_node &&									\
+	    	NET_EQUAL(at->dest_at_addr.atalk_net, (at_addr).atalk_net))	                        \
 		break; 							        \
 	    if (++n >= AMT_BSIZ) {					        \
 	        at = NULL;                                                      \
 		break;                                                          \
             }									\
 	}                                                                       \
-        }
+}
 
+/* at_addr - net # in network byte order */
 #define	NEW_AMT(at, at_addr, elapp) {							\
 	register n; 								\
 	register aarp_amt_t *myat;                                              \
@@ -159,7 +162,7 @@ typedef struct {
 		break;                                                          \
             }                                                                   \
 	}                                                                       \
-	}
+}
 
 #define	AARP_NET_MCAST(p, elapp)						\
  	(NET_VALUE((p)->dst_net) == elapp->ifThisNode.s_net)		\
diff --git a/bsd/netat/at_pcb.c b/bsd/netat/at_pcb.c
index 36b866a4f..b37174524 100644
--- a/bsd/netat/at_pcb.c
+++ b/bsd/netat/at_pcb.c
@@ -189,7 +189,6 @@ int at_pcbbind(pcb, nam)
 	register struct atpcb *pcb;
 	struct sockaddr *nam;
 {
-	register struct socket *so = pcb->atpcb_socket;
 	register struct sockaddr_at *local = (struct sockaddr_at *) nam;
 	u_char ddpsock = local->sat_port;
 
diff --git a/bsd/netat/at_proto.c b/bsd/netat/at_proto.c
index de59959de..79498bf91 100644
--- a/bsd/netat/at_proto.c
+++ b/bsd/netat/at_proto.c
@@ -77,7 +77,6 @@ extern int	ddp_pru_send(struct socket *so, int flags, struct mbuf *m,
 extern int	ddp_pru_shutdown(struct socket *so);
 extern int	ddp_pru_sockaddr(struct socket *so, 
 				 struct sockaddr **nam);
-void atalk_dominit();
 
 /*
  * Dummy usrreqs struct created by Ted for FreeBSD 3.x integration. 
@@ -91,31 +90,45 @@ struct pr_usrreqs ddp_usrreqs = {
 	ddp_pru_sockaddr, sosend, soreceive, pru_sopoll_notsupp
 };
 
-struct domain atalkdomain;
+extern struct domain atalkdomain;
+extern void atalk_dominit(void);
+
 struct protosw atalksw[] = {
   { SOCK_RAW,	&atalkdomain,	/*protocol*/ 0,	PR_ATOMIC|PR_ADDR,
     /*input*/ 0, /*output*/ 0, /*clinput*/ 0, ddp_ctloutput,
     /*ousrreq*/ 0, 
     ddp_init, /*fastto*/ 0, /*slowto*/ 0, /*drain*/ 0, 
     /*sysctl*/ 0, &ddp_usrreqs,
-	0, 0, 0
+	0, 0, 0, /*lock, unlock, getlock */
+   {0, 0}, 0, {0} /* filters */
   }
 };
 
 struct domain atalkdomain =
-{ AF_APPLETALK, "appletalk", atalk_dominit, 0, 0, 
-  atalksw, 0,
-  0, 0, 0,
-  DDP_X_HDR_SIZE, 0
+{ AF_APPLETALK,
+  "appletalk",
+  atalk_dominit,
+  0,
+  0, 
+  atalksw,
+  0,
+  0, /* dom_rtattach */
+  0, 0, /* dom_rtoffset, dom_maxrtkey */
+  DDP_X_HDR_SIZE, 0,
+  0, /* domain global mutex */
+  0,  /* domain flags */
+  {0, 0} /*reserved[2] */
 };
 
 struct domain * atalkdom = &atalkdomain;
 lck_mtx_t  *atalk_mutex = NULL;
 
+static int at_saved_lock, at_saved_unlock;
+
 SYSCTL_NODE(_net, PF_APPLETALK, appletalk, CTLFLAG_RW, 0, "AppleTalk Family");
 
 void
-atalk_dominit()
+atalk_dominit(void)
 {
 	atalk_mutex = atalkdom->dom_mtx;
 }
@@ -123,24 +136,22 @@ atalk_dominit()
 void
 atalk_lock()
 {
-	int error = 0, lr, lr_saved;
-#ifdef __ppc__
-	__asm__ volatile("mflr %0" : "=r" (lr));
-	lr_saved = lr;
-#endif 
+	int lr_saved;
+	lr_saved = (unsigned int) __builtin_return_address(0);
+
 	lck_mtx_assert(atalkdom->dom_mtx, LCK_MTX_ASSERT_NOTOWNED);
 	lck_mtx_lock(atalkdom->dom_mtx);
+	at_saved_lock = lr_saved;
 }
 	
 void
 atalk_unlock()
 {
-	int error = 0, lr, lr_saved;
-#ifdef __ppc__
-	__asm__ volatile("mflr %0" : "=r" (lr));
-	lr_saved = lr;
-#endif 
+	int lr_saved;
+	lr_saved = (unsigned int) __builtin_return_address(0);
+
 	lck_mtx_assert(atalkdom->dom_mtx, LCK_MTX_ASSERT_OWNED);
+	at_saved_unlock = lr_saved;
 	lck_mtx_unlock(atalkdom->dom_mtx);
 
 }
diff --git a/bsd/netat/at_var.h b/bsd/netat/at_var.h
index f1e2c6f5f..b9a3df2e2 100644
--- a/bsd/netat/at_var.h
+++ b/bsd/netat/at_var.h
@@ -316,8 +316,8 @@ struct kev_atalk_data {
 
 void atalk_post_msg(struct ifnet *ifp, u_long event_code, struct at_addr *address, at_nvestr_t *zone);
 void aarp_sched_probe(void *);
-void atalk_lock();
-void atalk_unlock();
+void atalk_lock(void);
+void atalk_unlock(void);
 
 #endif /* KERNEL_PRIVATE */
 #endif /* __APPLE_API_OBSOLETE */
diff --git a/bsd/netat/atp.h b/bsd/netat/atp.h
index fa1ae7391..1336e8fc8 100644
--- a/bsd/netat/atp.h
+++ b/bsd/netat/atp.h
@@ -85,13 +85,22 @@
 #define	ATP_XO_8MIN		4
 
 typedef struct {
+#if BYTE_ORDER == BIG_ENDIAN
         unsigned       cmd : 2,
                        xo : 1,
                        eom : 1,
                        sts : 1,
                        xo_relt : 3;
+#endif
+#if BYTE_ORDER == LITTLE_ENDIAN
+		unsigned		xo_relt : 3,
+						sts : 1,
+						eom : 1,
+						xo : 1,
+						cmd : 2;
+#endif
         u_char         bitmap;
-	ua_short       tid;
+		ua_short       tid;
         ua_long        user_bytes;
         u_char         data[ATP_DATA_SIZE];
 } at_atp_t;
diff --git a/bsd/netat/atp_alloc.c b/bsd/netat/atp_alloc.c
index 208ad056f..6ff87fe50 100644
--- a/bsd/netat/atp_alloc.c
+++ b/bsd/netat/atp_alloc.c
@@ -57,28 +57,23 @@
 #define TRPS_PER_BLK 16
 
 gbuf_t *atp_resource_m = 0;
-extern atlock_t atpgen_lock;
 extern caddr_t atp_free_cluster_list;
 extern void atp_delete_free_clusters();
 
 struct atp_trans *atp_trans_alloc(atp)
 struct  atp_state *atp;
 {
-	int s;
 	int i;
 	gbuf_t *m;
 	register struct atp_trans *trp, *trp_array;
 
-	ATDISABLE(s, atpgen_lock);
 	if (atp_trans_free_list == 0) {
-		ATENABLE(s, atpgen_lock);
 		if ((m = gbuf_alloc(TRPS_PER_BLK*sizeof(struct atp_trans),PRI_HI)) == 0)
 			return (struct atp_trans *)0;
 		bzero(gbuf_rptr(m), TRPS_PER_BLK*sizeof(struct atp_trans));
 		trp_array = (struct atp_trans *)gbuf_rptr(m);
 		for (i=0; i < TRPS_PER_BLK-1; i++)
 			trp_array[i].tr_list.next = (struct atp_trans *)&trp_array[i+1];
-		ATDISABLE(s, atpgen_lock);
 		gbuf_cont(m) = atp_resource_m;
 		atp_resource_m = m;
 		trp_array[i].tr_list.next = atp_trans_free_list;
@@ -87,11 +82,9 @@ struct  atp_state *atp;
 
 	trp = atp_trans_free_list;
 	atp_trans_free_list = trp->tr_list.next;
-	ATENABLE(s, atpgen_lock);
 	trp->tr_queue = atp;
 	trp->tr_state = TRANS_TIMEOUT;
 	trp->tr_local_node = 0;
-	ATLOCKINIT(trp->tr_lock);
 	ATEVENTINIT(trp->tr_event);
 
 	dPrintf(D_M_ATP_LOW, D_L_TRACE,
@@ -108,13 +101,10 @@ struct  atp_state *atp;
 void atp_trans_free(trp)
 register struct atp_trans *trp;
 {
-	int s;
 
-	ATDISABLE(s, atpgen_lock);
 	trp->tr_queue = 0;
 	trp->tr_list.next = atp_trans_free_list;
 	atp_trans_free_list = trp;
-	ATENABLE(s, atpgen_lock);
 }
 
 /*
@@ -127,16 +117,13 @@ struct atp_rcb *atp_rcb_alloc(atp)
 struct  atp_state *atp;
 {
 	register struct atp_rcb *rcbp;
-	int s;
 
-	ATDISABLE(s, atpgen_lock);
 	if ((rcbp = atp_rcb_free_list) != NULL) {
 		atp_rcb_free_list = rcbp->rc_list.next;
 		rcbp->rc_queue = atp;
 		rcbp->rc_pktcnt = 0;
 		rcbp->rc_local_node = 0;
 	}
-	ATENABLE(s, atpgen_lock);
 	dPrintf(D_M_ATP_LOW, D_L_TRACE,
 		("atp_rcb_alloc: allocated rcbp 0x%x\n", (u_int) rcbp));
 	return(rcbp);
@@ -153,14 +140,11 @@ register struct atp_rcb *rcbp;
 	register struct atp_state *atp;
 	register int i;
 	register int rc_state;
-	int s;
 
 	dPrintf(D_M_ATP_LOW, D_L_TRACE,
 		("atp_rcb_free: freeing rcbp 0x%x\n", (u_int) rcbp));
-	ATDISABLE(s, atpgen_lock);
 	atp = rcbp->rc_queue;
 	if ((rc_state = rcbp->rc_state) == -1) {
-		ATENABLE(s, atpgen_lock);
 		dPrintf(D_M_ATP, D_L_WARNING,
 			("atp_rcb_free(%d): tid=%d,loc=%d,rem=%d\n",
 			0, rcbp->rc_tid,
@@ -201,5 +185,4 @@ register struct atp_rcb *rcbp;
 	}
 	rcbp->rc_list.next = atp_rcb_free_list;
 	atp_rcb_free_list = rcbp;
-	ATENABLE(s, atpgen_lock);
 }
diff --git a/bsd/netat/atp_misc.c b/bsd/netat/atp_misc.c
index 13dc7235d..e08001b1e 100644
--- a/bsd/netat/atp_misc.c
+++ b/bsd/netat/atp_misc.c
@@ -54,7 +54,6 @@
 #include <netat/atp.h>
 #include <netat/debug.h>
 
-extern atlock_t atpgen_lock;
 void atp_free();
 void atp_send(struct atp_trans *);
 
@@ -67,7 +66,6 @@ void
 atp_req_timeout(trp)
 register struct atp_trans *trp;
 {
-	int s;
 	register gbuf_t *m;
 	gref_t *gref;
 	struct atp_state *atp;
@@ -75,19 +73,15 @@ register struct atp_trans *trp;
 
 	if ((atp = trp->tr_queue) == 0)
 		return;
-	ATDISABLE(s, atp->atp_lock);
-	if (atp->atp_flags & ATP_CLOSING) {
-		ATENABLE(s, atp->atp_lock);
+	if (atp->atp_flags & ATP_CLOSING)
 		return;
-	}
+	
 	for (ctrp = atp->atp_trans_wait.head; ctrp; ctrp = ctrp->tr_list.next) {
 		if (ctrp == trp)
 			break;
 	}
-	if (ctrp != trp) {
-		ATENABLE(s, atp->atp_lock);
+	if (ctrp != trp)
 		return;
-	}
 
 	if ((m = gbuf_cont(trp->tr_xmt)) == NULL)
 	        m = trp->tr_xmt;               /* issued via the new interface */
@@ -101,7 +95,6 @@ l_notify:
 				*gbuf_rptr(m) = 99;
 				gbuf_set_type(m, MSG_DATA);
 				gref = trp->tr_queue->atp_gref;
-				ATENABLE(s, atp->atp_lock);
 				atalk_putnext(gref, m);
 
 			return;
@@ -115,13 +108,11 @@ l_notify:
 			if (trp->tr_queue->dflag)
 				((ioc_t *)gbuf_rptr(m))->ioc_cmd = AT_ATP_REQUEST_COMPLETE;
 			else if (trp->tr_bdsp == NULL) {
-				ATENABLE(s, atp->atp_lock);
 				gbuf_freem(m);
 				if (trp->tr_rsp_wait)
 					wakeup(&trp->tr_event);
 				break;
 			}
-			ATENABLE(s, atp->atp_lock);
 			atp_iocnak(trp->tr_queue, m, ETIMEDOUT);
 			atp_free(trp);
 			return;
@@ -137,7 +128,6 @@ l_notify:
 
 		if (trp->tr_retry != (unsigned int) ATP_INFINITE_RETRIES)
 			trp->tr_retry--;
-		ATENABLE(s, atp->atp_lock);
 		atp_send(trp);
 	}
 }
@@ -154,12 +144,10 @@ register struct atp_trans *trp;
 {	
 	register struct atp_state *atp;
 	register int i;
-	int s;
 	
 	dPrintf(D_M_ATP_LOW, D_L_TRACE,
 		("atp_free: freeing trp 0x%x\n", (u_int) trp));
 
-	ATDISABLE(s, atpgen_lock);
 
 	if (trp->tr_state == TRANS_ABORTING) {
 		ATP_Q_REMOVE(atp_trans_abort, trp, tr_list);
@@ -191,12 +179,10 @@ register struct atp_trans *trp;
 			trp->tr_state = TRANS_ABORTING;
 			ATP_Q_APPEND(atp_trans_abort, trp, tr_list);
 			wakeup(&trp->tr_event);
-			ATENABLE(s, atpgen_lock);
 			return;
 		}
 	}
 	
-	ATENABLE(s, atpgen_lock);
 	atp_trans_free(trp);
 } /* atp_free */
 
@@ -251,10 +237,8 @@ register struct atp_rcb *rcbp;
 {
 	register struct atp_state *atp;
 	register int i;
-	int s;
 
   if ((atp = rcbp->rc_queue) != 0) {
-	ATDISABLE(s, atp->atp_lock);
 	for (i = 0; i < rcbp->rc_pktcnt; i++) {
 		if (rcbp->rc_bitmap&atp_mask[i])
 			rcbp->rc_snd[i] = 1;
@@ -264,10 +248,8 @@ register struct atp_rcb *rcbp;
         if (rcbp->rc_rep_waiting == 0) {
 	        rcbp->rc_state = RCB_SENDING;
 	        rcbp->rc_rep_waiting = 1;
-	        ATENABLE(s, atp->atp_lock);
 	        atp_send_replies(atp, rcbp);
-	} else
-	ATENABLE(s, atp->atp_lock);
+	}
   }
 }
 
@@ -278,26 +260,22 @@ register struct atp_rcb *rcbp;
 
 void atp_rcb_timer()
 {  
-	int s;
-        register struct atp_rcb *rcbp;
+    register struct atp_rcb *rcbp;
 	register struct atp_rcb *next_rcbp;
 	extern   struct atp_rcb_qhead atp_need_rel;
 	extern struct atp_trans *trp_tmo_rcb;
 	struct timeval timenow;
 
 l_again:
-	ATDISABLE(s, atpgen_lock);
 	getmicrouptime(&timenow);
 	for (rcbp = atp_need_rel.head; rcbp; rcbp = next_rcbp) {
 	        next_rcbp = rcbp->rc_tlist.next;
 
-	        if (abs(timenow.tv_sec - rcbp->rc_timestamp) > 30) {
-		        ATENABLE(s, atpgen_lock);
+	        if ((timenow.tv_sec - rcbp->rc_timestamp) > 30) {
 		        atp_rcb_free(rcbp);
 		        goto l_again;
 		}
 	}
-	ATENABLE(s, atpgen_lock);
 	atp_timout(atp_rcb_timer, trp_tmo_rcb, 10 * HZ);
 }
 
@@ -348,9 +326,7 @@ register struct atp_state *atp;
 {
 	register int i;
 	register struct atp_trans *trp;
-	int s;
 
-	ATDISABLE(s, atpgen_lock);
 	for (i = lasttid;;) {
 		i = (i+1)&0xffff;
 
@@ -360,7 +336,6 @@ register struct atp_state *atp;
 		}
 		if (trp == NULL) {
 			lasttid = i;
-			ATENABLE(s, atpgen_lock);
 			return(i);
 		}
 	}
diff --git a/bsd/netat/atp_open.c b/bsd/netat/atp_open.c
index d77532757..92fd2b162 100644
--- a/bsd/netat/atp_open.c
+++ b/bsd/netat/atp_open.c
@@ -64,9 +64,6 @@
 
 int atp_inited = 0;
 struct atp_rcb_qhead atp_need_rel;
-atlock_t atpall_lock;
-atlock_t atptmo_lock;
-atlock_t atpgen_lock;
 
 /**********/
 int atp_pidM[256];
@@ -136,7 +133,7 @@ int atp_open(gref, flag)
 	int flag;
 {
 	register struct atp_state *atp;
-	register int s, i;
+	register int i;
 	vm_offset_t	temp;
 	
 	/*
@@ -176,18 +173,14 @@ int atp_open(gref, flag)
 	 *	If no atp structure available return failure
 	 */
 
-	ATDISABLE(s, atpall_lock);
-	if ((atp = atp_free_list) == NULL) {
-		ATENABLE(s, atpall_lock);
+	if ((atp = atp_free_list) == NULL)
 		return(EAGAIN);
-	}
 
 	/*
 	 *	Update free list
 	 */
 
 	atp_free_list = atp->atp_trans_waiting;
-	ATENABLE(s, atpall_lock);
 
 	/*
 	 *	Initialize the data structure
@@ -206,8 +199,6 @@ int atp_open(gref, flag)
 	atp->atp_socket_no = -1;
 	atp->atp_pid = gref->pid;
 	atp->atp_msgq = 0;
-	ATLOCKINIT(atp->atp_lock);
-	ATLOCKINIT(atp->atp_delay_lock);
 	ATEVENTINIT(atp->atp_event);
 	ATEVENTINIT(atp->atp_delay_event);
 	gref->info = (void *)atp;
@@ -217,11 +208,9 @@ int atp_open(gref, flag)
 	 */
 
 	if (flag) {
-		ATDISABLE(s, atpall_lock);
 		if ((atp->atp_trans_waiting = atp_used_list) != 0)
 			atp->atp_trans_waiting->atp_rcb_waiting = atp;
 		atp_used_list = atp;
-		ATENABLE(s, atpall_lock);
 	}
 	return(0);
 }
@@ -239,7 +228,6 @@ int atp_close(gref, flag)
 	register struct atp_state *atp;
 	register struct atp_trans *trp;
 	register struct atp_rcb *rcbp;
-	register int s;
 	int socket;
 	pid_t pid;
 
@@ -251,7 +239,6 @@ int atp_close(gref, flag)
 		atp->atp_msgq = 0;
 	}
 
-	ATDISABLE(s, atp->atp_lock);
 	atp->atp_flags |= ATP_CLOSING;
 	socket = atp->atp_socket_no;
 	if (socket != -1)
@@ -272,7 +259,6 @@ int atp_close(gref, flag)
 		atp_rcb_free(rcbp);
 	while ((rcbp = atp->atp_attached.head))
 		atp_rcb_free(rcbp);
-	ATENABLE(s, atp->atp_lock);
 
 	if (flag && (socket == -1))
 		atp_dequeue_atp(atp);
@@ -280,11 +266,9 @@ int atp_close(gref, flag)
 	/*
 	 *	free the state variable
 	 */
-	ATDISABLE(s, atpall_lock);
 	atp->atp_socket_no = -1;
 	atp->atp_trans_waiting = atp_free_list;
 	atp_free_list = atp;
-	ATENABLE(s, atpall_lock);
 
 	if (socket != -1) {
 		pid = (pid_t)atp_pidM[socket];
diff --git a/bsd/netat/atp_read.c b/bsd/netat/atp_read.c
index 823a8475c..9a4424731 100644
--- a/bsd/netat/atp_read.c
+++ b/bsd/netat/atp_read.c
@@ -100,9 +100,10 @@ gbuf_t   *m;
 {
 	register at_atp_t *athp;
 	register struct atp_state *atp;
-	register int s, s_gen;
+	register int s_gen;
 	gbuf_t *m_asp = NULL;
 	struct timeval timenow;
+	u_short		temp_net;
 
 	atp = (struct atp_state *)gref->info;	
 	if (atp->dflag)
@@ -137,15 +138,14 @@ gbuf_t   *m;
 	    {   
 		register struct atp_trans *trp;
 		register int    seqno;
-	        register at_ddp_t       *ddp;
+	    register at_ddp_t       *ddp;
 
 		/*
 		 * we just got a response, find the trans record
 		 */
 
-		ATDISABLE(s, atp->atp_lock);
 		for (trp = atp->atp_trans_wait.head; trp; trp = trp->tr_list.next) {
-		    if (trp->tr_tid == UAS_VALUE(athp->tid))
+		    if (trp->tr_tid == UAS_VALUE_NTOH(athp->tid))
 			break;
 		}
 
@@ -154,12 +154,11 @@ gbuf_t   *m;
 		 */
 		seqno = athp->bitmap;
 		if (trp == NULL) {
-	        ATENABLE(s, atp->atp_lock);
 	        ddp = AT_DDP_HDR(m);
 		    dPrintf(D_M_ATP_LOW, (D_L_INPUT|D_L_ERROR),
 		("atp_rput: dropping TRESP, no trp,tid=%d,loc=%d,rem=%d.%d,seqno=%d\n",
-			    UAS_VALUE(athp->tid),
-			    ddp->dst_socket,ddp->src_node,ddp->src_socket,seqno));
+			    UAS_VALUE_NTOH(athp->tid),
+			    ddp->dst_socket, ddp->src_node, ddp->src_socket, seqno));
 		    gbuf_freem(m);
 		    return;
 		}
@@ -168,11 +167,10 @@ gbuf_t   *m;
 		 * If no longer valid, drop it
 		 */
 		if (trp->tr_state == TRANS_FAILED) {
-	        ATENABLE(s, atp->atp_lock);
 	        ddp = AT_DDP_HDR(m);
 		    dPrintf(D_M_ATP_LOW, (D_L_INPUT|D_L_ERROR),
 		("atp_rput: dropping TRESP, failed trp,tid=%d,loc=%d,rem=%d.%d\n",
-			    UAS_VALUE(athp->tid),
+			    UAS_VALUE_NTOH(athp->tid),
 			    ddp->dst_socket, ddp->src_node, ddp->src_socket));
 		    gbuf_freem(m);
 		    return;
@@ -182,11 +180,10 @@ gbuf_t   *m;
 		 * If we have already received it, ignore it
 		 */
 		if (!(trp->tr_bitmap&atp_mask[seqno]) || trp->tr_rcv[seqno]) {
-	        ATENABLE(s, atp->atp_lock);
 	        ddp = AT_DDP_HDR(m);
 		    dPrintf(D_M_ATP_LOW, (D_L_INPUT|D_L_ERROR),
 		("atp_rput: dropping TRESP, duplicate,tid=%d,loc=%d,rem=%d.%d,seqno=%d\n",
-			    UAS_VALUE(athp->tid),
+			    UAS_VALUE_NTOH(athp->tid),
 			    ddp->dst_socket, ddp->src_node, ddp->src_socket, seqno));
 		    gbuf_freem(m);
 		    return;
@@ -217,7 +214,6 @@ gbuf_t   *m;
 		 *		the message to the user
 		 */
 		if (trp->tr_bitmap == 0) {
-		    ATENABLE(s, atp->atp_lock);
 
 		    /*
 		     *	Cancel the request timer and any
@@ -235,12 +231,10 @@ gbuf_t   *m;
 		    /*
 		     *	If they want treq again, send them
 		     */
-		    ATENABLE(s, atp->atp_lock);
 		    atp_untimout(atp_req_timeout, trp);
 		    atp_send(trp);
 		    return;
 		}
-		ATENABLE(s, atp->atp_lock);
 		return;
 	    }
 
@@ -253,9 +247,8 @@ gbuf_t   *m;
 		 */
 	        ddp = AT_DDP_HDR(m);
 
-		ATDISABLE(s, atp->atp_lock);
 		for (rcbp = atp->atp_rcb.head; rcbp; rcbp = rcbp->rc_list.next) {
-		    if (rcbp->rc_tid == UAS_VALUE(athp->tid) &&
+		    if (rcbp->rc_tid == UAS_VALUE_NTOH(athp->tid) &&
 			rcbp->rc_socket.node == ddp->src_node &&
 			rcbp->rc_socket.net == NET_VALUE(ddp->src_net) &&
 			rcbp->rc_socket.socket == ddp->src_socket) {
@@ -269,14 +262,11 @@ gbuf_t   *m;
 				{
 				ddp = 0;
 				atp_rcb_free(rcbp);
-				ATENABLE(s, atp->atp_lock);
 				}
 			    break;
 		    }
 		}
 
-		if (ddp)
-			ATENABLE(s, atp->atp_lock);
 		gbuf_freem(m);
 		return;
 	   }
@@ -295,9 +285,8 @@ gbuf_t   *m;
 		 */
 	        ddp = AT_DDP_HDR(m);
 
-		ATDISABLE(s, atp->atp_lock);
 		for (rcbp = atp->atp_rcb.head; rcbp; rcbp = rcbp->rc_list.next) {
-		    if (rcbp->rc_tid == UAS_VALUE(athp->tid) &&
+		    if (rcbp->rc_tid == UAS_VALUE_NTOH(athp->tid) &&
 			rcbp->rc_socket.node == ddp->src_node &&
 			rcbp->rc_socket.net == NET_VALUE(ddp->src_net) &&
 			rcbp->rc_socket.socket == ddp->src_socket)
@@ -316,11 +305,10 @@ gbuf_t   *m;
 		     */
 					/* we just did this, why do again? -jjs 4-10-95 */
 		    for (rcbp = atp->atp_attached.head; rcbp; rcbp = rcbp->rc_list.next) {
-		        if (rcbp->rc_tid == UAS_VALUE(athp->tid) &&
+		        if (rcbp->rc_tid == UAS_VALUE_NTOH(athp->tid) &&
 			    rcbp->rc_socket.node == ddp->src_node &&
 			    rcbp->rc_socket.net == NET_VALUE(ddp->src_net) &&
 			    rcbp->rc_socket.socket == ddp->src_socket) {
-			    ATENABLE(s, atp->atp_lock);
 			    gbuf_freem(m);
 			    dPrintf(D_M_ATP_LOW, D_L_INPUT, 
 				    ("atp_rput: dropping TREQ, matches req queue\n"));
@@ -332,20 +320,19 @@ gbuf_t   *m;
 			 * assume someone is interested in 
 			 * in an asynchronous incoming request
 			 */
-			ATENABLE(s, atp->atp_lock);
 			if ((rcbp = atp_rcb_alloc(atp)) == NULL) {
 			    gbuf_freem(m);
 			    return;
 			}
 			rcbp->rc_state = RCB_UNQUEUED;
-			ATDISABLE(s, atp->atp_lock);
 
 		    rcbp->rc_local_node = ddp->dst_node;
-		    NET_NET(rcbp->rc_local_net, ddp->dst_net);
+		    temp_net = NET_VALUE(ddp->dst_net);
+		    NET_ASSIGN_NOSWAP(rcbp->rc_local_net, temp_net);
 		    rcbp->rc_socket.socket = ddp->src_socket;
 		    rcbp->rc_socket.node = ddp->src_node;
 		    rcbp->rc_socket.net = NET_VALUE(ddp->src_net);
-		    rcbp->rc_tid = UAS_VALUE(athp->tid);
+		    rcbp->rc_tid = UAS_VALUE_NTOH(athp->tid);
 		    rcbp->rc_bitmap = athp->bitmap;
 		    rcbp->rc_not_sent_bitmap = athp->bitmap;
 		    rcbp->rc_xo = athp->xo;
@@ -382,7 +369,6 @@ gbuf_t   *m;
 			rcbp->rc_state = RCB_PENDING;
 			ATP_Q_APPEND(atp->atp_attached, rcbp, rc_list);
 			if (m_asp != NULL) {
-			    ATENABLE(s, atp->atp_lock);
 			    atp_req_ind(atp, m_asp);
 			    return;
 			}
@@ -406,16 +392,13 @@ gbuf_t   *m;
 			 *		the replies
 			 */
 			getmicrouptime(&timenow);
-			ATDISABLE(s_gen, atpgen_lock);
 			if (rcbp->rc_timestamp) {
 			  rcbp->rc_timestamp = timenow.tv_sec;
 			  if (rcbp->rc_timestamp == 0)
 			    rcbp->rc_timestamp = 1;
 			}
-			ATENABLE(s_gen, atpgen_lock);
 			rcbp->rc_bitmap = athp->bitmap;
 			rcbp->rc_not_sent_bitmap = athp->bitmap;
-			ATENABLE(s, atp->atp_lock);
 			gbuf_freem(m);
 			atp_reply(rcbp);
 			return;
@@ -427,12 +410,10 @@ gbuf_t   *m;
 			 *      we haven't sent any data yet
 			 *      ignore the request
 			 */
-			ATENABLE(s, atp->atp_lock);
 			gbuf_freem(m);
 			return;
 		    }
 		}
-		ATENABLE(s, atp->atp_lock);
 		return;
 	   }
 
diff --git a/bsd/netat/atp_write.c b/bsd/netat/atp_write.c
index 34d18be4b..27526a78c 100644
--- a/bsd/netat/atp_write.c
+++ b/bsd/netat/atp_write.c
@@ -80,9 +80,6 @@ extern struct atp_rcb_qhead atp_need_rel;
 extern int atp_inited;
 extern struct atp_state *atp_used_list;
 extern asp_scb_t *scb_free_list;
-extern atlock_t atpgen_lock;
-extern atlock_t atpall_lock;
-extern atlock_t atptmo_lock;
 
 extern gbuf_t *scb_resource_m;
 extern gbuf_t *atp_resource_m;
@@ -145,7 +142,7 @@ atp_wput(gref, m)
 	register gbuf_t *m;
 {
 	register ioc_t    *iocbp;
-	int i, xcnt, s;
+	int i, xcnt;
 	struct atp_state *atp;
 	struct atp_trans *trp;
 	struct atp_rcb   *rcbp;
@@ -243,18 +240,16 @@ atp_wput(gref, m)
 				atp->atp_msgq = 0;
 			}
 
-			ATDISABLE(s, atp->atp_lock);
 			/*
 			 *	search for the corresponding rcb
 			 */
 			for (rcbp = atp->atp_rcb.head; rcbp; rcbp = rcbp->rc_list.next) {
-			    if (rcbp->rc_tid == UAS_VALUE(athp->tid) &&
+			    if (rcbp->rc_tid == UAS_VALUE_NTOH(athp->tid) &&
 				rcbp->rc_socket.node == ddp->dst_node &&
 				rcbp->rc_socket.net == NET_VALUE(ddp->dst_net) &&
 				rcbp->rc_socket.socket == ddp->dst_socket)
 				break;
 			}
-			ATENABLE(s, atp->atp_lock);
 
 			/*
 			 *	If it has already been sent then return an error
@@ -275,13 +270,11 @@ atp_wput(gref, m)
 			    rcbp->rc_socket.socket = ddp->dst_socket;
 			    rcbp->rc_socket.node = ddp->dst_node;
 			    rcbp->rc_socket.net = NET_VALUE(ddp->dst_net);
-			    rcbp->rc_tid = UAS_VALUE(athp->tid);
+			    rcbp->rc_tid = UAS_VALUE_NTOH(athp->tid);
 			    rcbp->rc_bitmap = 0xff;
 			    rcbp->rc_xo = 0;
-			    ATDISABLE(s, atp->atp_lock);
 			    rcbp->rc_state = RCB_SENDING;
 			    ATP_Q_APPEND(atp->atp_rcb, rcbp, rc_list);
-			    ATENABLE(s, atp->atp_lock);
 			}
 			xcnt = get_bds_entries(m2);
 			if ((i = atp_unpack_bdsp(atp, m2, rcbp, xcnt, FALSE))) {
@@ -309,7 +302,6 @@ atp_wput(gref, m)
 			/*
 			 *	search for a waiting request
 			 */
-			ATDISABLE(s, atp->atp_lock);
 			if ((rcbp = atp->atp_attached.head)) {
 			    /*
 			     *	Got one, move it to the active response Q
@@ -326,13 +318,11 @@ atp_wput(gref, m)
 				 */
 				atp_rcb_free(rcbp);
 			    }
-			    ATENABLE(s, atp->atp_lock);
 			    atp_iocack(atp, m);
 			} else {
 				/*
 				 *	None available - can out
 				 */
-				ATENABLE(s, atp->atp_lock);
 				atp_iocnak(atp, m, EAGAIN);
 			}
 			break;
@@ -349,16 +339,13 @@ atp_wput(gref, m)
 			i = *(int *)gbuf_rptr(gbuf_cont(m));
 			gbuf_freem(gbuf_cont(m));
 			gbuf_cont(m) = NULL;
-			ATDISABLE(s, atp->atp_lock);
 			for (trp = atp->atp_trans_wait.head; trp; trp = trp->tr_list.next) {
 			  if (trp->tr_tid == i)
 				break;
 			}
-			if (trp == NULL) {
-				ATENABLE(s, atp->atp_lock);
+			if (trp == NULL)
 				atp_iocnak(atp, m, ENOENT);
-			} else {
-				ATENABLE(s, atp->atp_lock);
+			else {
 				atp_free(trp);
 				atp_iocack(atp, m);
 			}
@@ -436,7 +423,7 @@ register struct atp_trans *trp;
 		gbuf_wset(m,TOTAL_ATP_HDR_SIZE);
 		ddp = AT_DDP_HDR(m);
 		ddp->type = DDP_ATP;
-		UAS_ASSIGN(ddp->checksum, 0);
+		UAS_ASSIGN_HTON(ddp->checksum, 0);
 		ddp->dst_socket = trp->tr_socket.socket;
 		ddp->dst_node = trp->tr_socket.node;
 		NET_ASSIGN(ddp->dst_net, trp->tr_socket.net);
@@ -449,7 +436,7 @@ register struct atp_trans *trp;
 		athp = AT_ATP_HDR(m);
 		ATP_CLEAR_CONTROL(athp);
 		athp->cmd = ATP_CMD_TREL;
-		UAS_ASSIGN(athp->tid, trp->tr_tid);
+		UAS_ASSIGN_HTON(athp->tid, trp->tr_tid);
 	}
 
 	return (m);
@@ -460,7 +447,7 @@ void atp_send_replies(atp, rcbp)
      register struct atp_rcb   *rcbp;
 {       register gbuf_t *m;
 	register int     i, len;
-	int              s_gen, s, cnt, err, offset, space;
+	int              s_gen, cnt, err, offset, space;
 	unsigned char *m0_rptr = NULL, *m0_wptr = NULL;
 	register at_atp_t *athp;
 	register struct atpBDS *bdsp;
@@ -474,11 +461,8 @@ void atp_send_replies(atp, rcbp)
 	};
 	struct timeval timenow;
 
-	ATDISABLE(s, atp->atp_lock);
-	if (rcbp->rc_queue != atp) {
-		ATENABLE(s, atp->atp_lock);
+	if (rcbp->rc_queue != atp)
 		return;
-	}
 	if (rcbp->rc_not_sent_bitmap == 0)
 		goto nothing_to_send;
 
@@ -566,21 +550,17 @@ void atp_send_replies(atp, rcbp)
 	   	 */
 	  	bdsp++;
 	}
-	if (mlist) {
-		ATENABLE(s, atp->atp_lock);
+	if (mlist)
 		DDP_OUTPUT(mlist);
-		ATDISABLE(s, atp->atp_lock);
-	}
+	
 
 nothing_to_send:
 	/*
 	 *	If all replies from this reply block have been sent then 
 	 *		remove it from the queue and mark it so
 	 */
-	if (rcbp->rc_queue != atp) {
-		ATENABLE(s, atp->atp_lock);
+	if (rcbp->rc_queue != atp)
 		return;
-	}
 	rcbp->rc_rep_waiting = 0;
 
 	/*
@@ -595,7 +575,6 @@ nothing_to_send:
 	 */
 	if (rcbp->rc_xo && rcbp->rc_state != RCB_RELEASED) {
 		getmicrouptime(&timenow);
-		ATDISABLE(s_gen, atpgen_lock);
 		if (rcbp->rc_timestamp == 0) {
 	        	rcbp->rc_timestamp = timenow.tv_sec;
 			if (rcbp->rc_timestamp == 0)
@@ -603,10 +582,8 @@ nothing_to_send:
 			ATP_Q_APPEND(atp_need_rel, rcbp, rc_tlist);
 		}
 		rcbp->rc_state = RCB_RESPONSE_FULL;
-		ATENABLE(s_gen, atpgen_lock);
 	} else
 		atp_rcb_free(rcbp);
-	ATENABLE(s, atp->atp_lock);
 } /* atp_send_replies */
 
 
@@ -694,7 +671,7 @@ atp_unpack_bdsp(atp, m, rcbp, cnt, wait)
 	register	 gbuf_t        *m2, *m1, *m0, *mhdr;
 	caddr_t 	lastPage;
     at_atp_t    *athp;
-	int  		i, len, s_gen;
+	int  		i, len;
 	at_socket 	src_socket;
 	
 	struct ddp_atp {
@@ -840,13 +817,11 @@ atp_unpack_bdsp(atp, m, rcbp, cnt, wait)
 l_send:
 	if (rcbp->rc_xo) {
 		getmicrouptime(&timenow);
-		ATDISABLE(s_gen, atpgen_lock);
 		if (rcbp->rc_timestamp == 0) {
 			if ((rcbp->rc_timestamp = timenow.tv_sec) == 0)
 				rcbp->rc_timestamp = 1;
 			ATP_Q_APPEND(atp_need_rel, rcbp, rc_tlist);
 		}
-		ATENABLE(s_gen, atpgen_lock);
 	}
 
 	DDP_OUTPUT(mlist);
@@ -868,7 +843,6 @@ int atp_bind(gref, sVal, flag)
 	unsigned char inpC, sNextUsed = 0;
 	unsigned int sMin, sMax, sSav;
 	struct atp_state *atp;
-	int s;
 
 	atp = (struct atp_state *)gref->info;
 	if (atp->dflag)
@@ -876,7 +850,6 @@ int atp_bind(gref, sVal, flag)
 
 	sMax = ATP_SOCKET_LAST;
 	sMin = ATP_SOCKET_FIRST;
-	ATDISABLE(s, atpgen_lock);
 	if (flag && (*flag == 3)) {
 		sMin += 40;
 		if (sMin < sNext) {
@@ -889,7 +862,6 @@ int atp_bind(gref, sVal, flag)
 	     ((sVal > sMax) || (sVal < 2) || (sVal == 6) || 
 	      (ddp_socket_inuse(sVal, DDP_ATP) &&
 	       (atp_inputQ[sVal] != (gref_t *)1)))) {
-		ATENABLE(s, atpgen_lock);
 		return 0;
 	}
 
@@ -919,7 +891,6 @@ again:
 				sNext = 0;
 				*flag = (unsigned char)sSav;
 			}
-			ATENABLE(s, atpgen_lock);
 			return 0;
 		}
 	}
@@ -933,7 +904,6 @@ again:
 			sNext = 0;
 	}
 
-	ATENABLE(s, atpgen_lock);
 	return (int)sVal;
 }
 
@@ -942,19 +912,16 @@ void atp_req_ind(atp, mioc)
 	register gbuf_t *mioc;
 {
 	register struct atp_rcb *rcbp;
-	int s;
 
 	if ((rcbp = atp->atp_attached.head) != 0) {
 		gbuf_cont(mioc) = rcbp->rc_ioctl;
 		rcbp->rc_ioctl = NULL;
-		ATDISABLE(s, atp->atp_lock);
 		if (rcbp->rc_xo) {
 			ATP_Q_REMOVE(atp->atp_attached, rcbp, rc_list);
 			rcbp->rc_state = RCB_NOTIFIED;
 			ATP_Q_APPEND(atp->atp_rcb, rcbp, rc_list);
 		} else
 			atp_rcb_free(rcbp);
-		ATENABLE(s, atp->atp_lock);
 		if (gbuf_cont(mioc))
 		  ((ioc_t *)gbuf_rptr(mioc))->ioc_count = gbuf_msgsize(gbuf_cont(mioc));
 		else
@@ -1007,7 +974,6 @@ void atp_cancel_req(gref, tid)
 	gref_t *gref;
 	unsigned short tid;
 {
-	int s;
 	struct atp_state *atp;
 	struct atp_trans *trp;
 
@@ -1015,12 +981,10 @@ void atp_cancel_req(gref, tid)
 	if (atp->dflag)
 		atp = (struct atp_state *)atp->atp_msgq;
 
-	ATDISABLE(s, atp->atp_lock);
 	for (trp = atp->atp_trans_wait.head; trp; trp = trp->tr_list.next) {
 	    if (trp->tr_tid == tid)
 			break;
 	}
-	ATENABLE(s, atp->atp_lock);
 	if (trp != NULL)
 		atp_free(trp);
 }
@@ -1032,9 +996,7 @@ void
 atp_dequeue_atp(atp)
 	struct atp_state *atp;
 {
-	int s;
 
-	ATDISABLE(s, atpall_lock);
 	if (atp == atp_used_list) {
 		if ((atp_used_list = atp->atp_trans_waiting) != 0)
 			atp->atp_trans_waiting->atp_rcb_waiting = 0;
@@ -1046,7 +1008,6 @@ atp_dequeue_atp(atp)
 
 	atp->atp_trans_waiting = 0;
 	atp->atp_rcb_waiting = 0;
-	ATENABLE(s, atpall_lock);
 }
 
 void
@@ -1055,15 +1016,11 @@ atp_timout(func, trp, ticks)
 	struct atp_trans *trp;
 	int ticks;
 {
-	int s;
 	unsigned int sum;
 	struct atp_trans *curr_trp, *prev_trp;
 
-	ATDISABLE(s, atptmo_lock);
-	if (trp->tr_tmo_func) {
-		ATENABLE(s, atptmo_lock);
+	if (trp->tr_tmo_func)
 		return;
-	}
 
 	trp->tr_tmo_func = func;
 	trp->tr_tmo_delta = 1+(ticks>>5);
@@ -1071,7 +1028,6 @@ atp_timout(func, trp, ticks)
 	if (trp_tmo_list == 0) {
 		trp->tr_tmo_next = trp->tr_tmo_prev = 0;
 		trp_tmo_list = trp;
-		ATENABLE(s, atptmo_lock);
 		return;
 	}
 
@@ -1105,7 +1061,6 @@ atp_timout(func, trp, ticks)
 		trp_tmo_list->tr_tmo_prev = trp;
 		trp_tmo_list = trp;
 	}
-	ATENABLE(s, atptmo_lock);
 }
 
 void
@@ -1113,13 +1068,9 @@ atp_untimout(func, trp)
 	void (*func)();
 	struct atp_trans *trp;
 {
-	int s;
 
-	ATDISABLE(s, atptmo_lock);
-	if (trp->tr_tmo_func == 0) {
-		ATENABLE(s, atptmo_lock);
+	if (trp->tr_tmo_func == 0)
 		return;
-	}
 
 	if (trp_tmo_list == trp) {
 		if ((trp_tmo_list = trp->tr_tmo_next) != 0) {
@@ -1133,7 +1084,6 @@ atp_untimout(func, trp)
 		}
 	}
 	trp->tr_tmo_func = 0;
-	ATENABLE(s, atptmo_lock);
 }
 
 void
@@ -1149,11 +1099,9 @@ void
 atp_trp_clock(arg)
 	void *arg;
 {
-	int s;
 	struct atp_trans *trp;
 	void (*tr_tmo_func)();
 
-	ATDISABLE(s, atptmo_lock);
 	if (trp_tmo_list)
 		trp_tmo_list->tr_tmo_delta--;
 	while (((trp = trp_tmo_list) != 0) && (trp_tmo_list->tr_tmo_delta == 0)) {
@@ -1161,12 +1109,9 @@ atp_trp_clock(arg)
 			trp_tmo_list->tr_tmo_prev = 0;
 		if ((tr_tmo_func = trp->tr_tmo_func) != 0) {
 			trp->tr_tmo_func = 0;
-			ATENABLE(s, atptmo_lock);
 			(*tr_tmo_func)(trp);
-			ATDISABLE(s, atptmo_lock);
 		}
 	}
-	ATENABLE(s, atptmo_lock);
 
 	timeout(atp_trp_clock_locked, (void *)arg, (1<<5));
 }
@@ -1183,8 +1128,9 @@ atp_send_req(gref, mioc)
 	register at_ddp_t *ddp;
 	gbuf_t *m, *m2, *bds;
 	struct atp_set_default *sdb;
-	int s, old;
+	int old;
 	unsigned int timer;
+	u_short		temp_net;
 
 	atp = (struct atp_state *)((struct atp_state *)gref->info)->atp_msgq;
 	iocbp = (ioc_t *)gbuf_rptr(mioc);
@@ -1232,7 +1178,7 @@ l_retry:
 	 */
 	athp = AT_ATP_HDR(m2);
 	athp->cmd = ATP_CMD_TREQ;
-	UAS_ASSIGN(athp->tid, trp->tr_tid);
+	UAS_ASSIGN_HTON(athp->tid, trp->tr_tid);
 	athp->eom = 0;
 	athp->sts = 0;
 	trp->tr_xo = athp->xo;
@@ -1245,7 +1191,8 @@ l_retry:
 	trp->tr_socket.net = NET_VALUE(ddp->dst_net);
 	trp->tr_local_socket = atp->atp_socket_no;
 	trp->tr_local_node = ddp->src_node;
-	NET_NET(trp->tr_local_net, ddp->src_net);
+	temp_net = NET_VALUE(ddp->src_net);
+	NET_ASSIGN_NOSWAP(trp->tr_local_net, temp_net);
 
 #ifdef NOT_YET
 	/* save the local information in the gref */
@@ -1258,9 +1205,7 @@ l_retry:
 	/*
 	 *	Put us in the transaction waiting queue
 	 */
-	ATDISABLE(s, atp->atp_lock);
 	ATP_Q_APPEND(atp->atp_trans_wait, trp, tr_list);
-	ATENABLE(s, atp->atp_lock);
 
 	/*
 	 * Send the message and set the timer	
@@ -1302,6 +1247,7 @@ void atp_send_rsp(gref, m, wait)
 	register at_atp_t *athp;
 	register at_ddp_t *ddp;
 	int s, xcnt;
+	u_short	temp_net;
 
 	atp = (struct atp_state *)gref->info;
 	if (atp->dflag)
@@ -1312,9 +1258,8 @@ void atp_send_rsp(gref, m, wait)
 	/*
 	 *	search for the corresponding rcb
 	 */
-	ATDISABLE(s, atp->atp_lock);
 	for (rcbp = atp->atp_rcb.head; rcbp; rcbp = rcbp->rc_list.next) {
-	    if ( (rcbp->rc_tid == UAS_VALUE(athp->tid)) &&
+	    if ( (rcbp->rc_tid == UAS_VALUE_NTOH(athp->tid)) &&
 			(rcbp->rc_socket.node == ddp->dst_node) &&
 			(rcbp->rc_socket.net == NET_VALUE(ddp->dst_net)) &&
 			(rcbp->rc_socket.socket == ddp->dst_socket) )
@@ -1326,11 +1271,9 @@ void atp_send_rsp(gref, m, wait)
 	 */
 	if ((rcbp && (rcbp->rc_state != RCB_NOTIFIED)) ||
 			(rcbp == NULL && athp->xo) ) {
-		ATENABLE(s, atp->atp_lock);
 		gbuf_freem(m);
 		return;
 	}
-	ATENABLE(s, atp->atp_lock);
 
 	if (rcbp == NULL) { /* a response is being sent for an ALO transaction */
 	    if ((rcbp = atp_rcb_alloc(atp)) == NULL) {
@@ -1341,16 +1284,15 @@ void atp_send_rsp(gref, m, wait)
 	    rcbp->rc_socket.socket = ddp->dst_socket;
 	    rcbp->rc_socket.node = ddp->dst_node;
 	    rcbp->rc_socket.net = NET_VALUE(ddp->dst_net);
-	    rcbp->rc_tid = UAS_VALUE(athp->tid);
+	    rcbp->rc_tid = UAS_VALUE_NTOH(athp->tid);
 	    rcbp->rc_bitmap = 0xff;
 	    rcbp->rc_xo = 0;
 	    rcbp->rc_state = RCB_RESPONSE_FULL;
-	    ATDISABLE(s, atp->atp_lock);
 	    ATP_Q_APPEND(atp->atp_rcb, rcbp, rc_list);
-		ATENABLE(s, atp->atp_lock);
 	}
 	else if (ddp->src_node == 0) {
-		NET_NET(ddp->src_net, rcbp->rc_local_net);
+		temp_net = NET_VALUE_NOSWAP(rcbp->rc_local_net);
+		NET_ASSIGN(ddp->src_net, temp_net);
 		ddp->src_node = rcbp->rc_local_node;
 	}
 
@@ -1448,7 +1390,7 @@ _ATPsndreq(fd, buf, len, nowait, err, proc)
 	void *proc;
 {
 	gref_t *gref;
-	int s, rc;
+	int rc;
 	unsigned short tid;
 	unsigned int timer;
 	register struct atp_state *atp;
@@ -1486,9 +1428,7 @@ _ATPsndreq(fd, buf, len, nowait, err, proc)
 		ts.tv_sec = 0;
 		ts.tv_nsec = 100 *1000 * NSEC_PER_USEC;
 
-		ATDISABLE(s, atp->atp_delay_lock);
 		rc = msleep(&atp->atp_delay_event, atalk_mutex, PSOCK | PCATCH, "atpmioc", &ts);
-		ATENABLE(s, atp->atp_delay_lock);
 		if (rc != 0) {
 			*err = rc;
 			file_drop(fd);
@@ -1504,9 +1444,7 @@ _ATPsndreq(fd, buf, len, nowait, err, proc)
         ts.tv_sec = 0;
         ts.tv_nsec = 100 *1000 * NSEC_PER_USEC;
 
-		ATDISABLE(s, atp->atp_delay_lock);
 		rc = msleep(&atp->atp_delay_event, atalk_mutex, PSOCK | PCATCH, "atpm2", &ts);
-		ATENABLE(s, atp->atp_delay_lock);
 		if (rc != 0) {
 			gbuf_freeb(mioc);
 			file_drop(fd);
@@ -1547,9 +1485,7 @@ _ATPsndreq(fd, buf, len, nowait, err, proc)
         ts.tv_sec = 0;
         ts.tv_nsec = 100 *1000 * NSEC_PER_USEC;
 
-		ATDISABLE(s, atp->atp_delay_lock);
 		rc = msleep(&atp->atp_delay_event, atalk_mutex, PSOCK | PCATCH, "atptrp", &ts);
-		ATENABLE(s, atp->atp_delay_lock);
 		if (rc != 0) {
 			gbuf_freem(mioc);
 			file_drop(fd);
@@ -1575,7 +1511,7 @@ _ATPsndreq(fd, buf, len, nowait, err, proc)
 	 */
 	athp = AT_ATP_HDR(m2);
 	athp->cmd = ATP_CMD_TREQ;
-	UAS_ASSIGN(athp->tid, trp->tr_tid);
+	UAS_ASSIGN_HTON(athp->tid, trp->tr_tid);
 	athp->eom = 0;
 	athp->sts = 0;
 	trp->tr_xo = athp->xo;
@@ -1600,9 +1536,7 @@ _ATPsndreq(fd, buf, len, nowait, err, proc)
 	/*
 	 *	Put us in the transaction waiting queue
 	 */
-	ATDISABLE(s, atp->atp_lock);
 	ATP_Q_APPEND(atp->atp_trans_wait, trp, tr_list);
-	ATENABLE(s, atp->atp_lock);
 
 	/*
 	 * Send the message and set the timer	
@@ -1623,21 +1557,18 @@ _ATPsndreq(fd, buf, len, nowait, err, proc)
 	/*
 	 * wait for the transaction to complete
 	 */
-	ATDISABLE(s, trp->tr_lock);
 	while ((trp->tr_state != TRANS_DONE) && (trp->tr_state != TRANS_FAILED) &&
 				(trp->tr_state != TRANS_ABORTING)) {
 		trp->tr_rsp_wait = 1;
 		rc = msleep(&trp->tr_event, atalk_mutex, PSOCK | PCATCH, "atpsndreq", 0);
 		if (rc != 0) {
 			trp->tr_rsp_wait = 0;
-			ATENABLE(s, trp->tr_lock);
 			file_drop(fd);
 			*err = rc;
 			return -1;
 		}
 	}
 	trp->tr_rsp_wait = 0;
-	ATENABLE(s, trp->tr_lock);
 
 
 	if (trp->tr_state == TRANS_FAILED || trp->tr_state == TRANS_ABORTING) {
@@ -1830,7 +1761,7 @@ _ATPgetreq(fd, buf, buflen, err, proc)
 	register struct atp_state *atp;
 	register struct atp_rcb *rcbp;
 	register gbuf_t *m, *m_head;
-	int s, size, len;
+	int size, len;
 
 	if ((*err = atalk_getref(0, fd, &gref, proc, 1)) != 0)
 		return -1;
@@ -1850,7 +1781,6 @@ _ATPgetreq(fd, buf, buflen, err, proc)
 		return -1;
 	}
 
-	ATDISABLE(s, atp->atp_lock);
 	if ((rcbp = atp->atp_attached.head) != NULL) {
 	    /*
 	     * Got one, move it to the active response Q
@@ -1868,7 +1798,6 @@ _ATPgetreq(fd, buf, buflen, err, proc)
 			 */
 			atp_rcb_free(rcbp);
 		}
-		ATENABLE(s, atp->atp_lock);
 
 		/*
 		 * copyout the request data, including the protocol header
@@ -1886,7 +1815,6 @@ _ATPgetreq(fd, buf, buflen, err, proc)
 		file_drop(fd);
 		return size;
 	}
-	ATENABLE(s, atp->atp_lock);
 
 	file_drop(fd);
 	return -1;
@@ -1902,7 +1830,7 @@ _ATPgetrsp(fd, bdsp, err, proc)
 	gref_t *gref;
 	register struct atp_state *atp;
 	register struct atp_trans *trp;
-	int s, tid;
+	int tid;
 	char bds[atpBDSsize];
 
 	if ((*err = atalk_getref(0, fd, &gref, proc, 1)) != 0)
@@ -1917,7 +1845,6 @@ _ATPgetrsp(fd, bdsp, err, proc)
 		return -1;
 	}
 
-	ATDISABLE(s, atp->atp_lock);
 	for (trp = atp->atp_trans_wait.head; trp; trp = trp->tr_list.next) {
 		dPrintf(D_M_ATP, D_L_INFO,
 			("ATPgetrsp: atp:0x%x, trp:0x%x, state:%d\n",
@@ -1925,7 +1852,6 @@ _ATPgetrsp(fd, bdsp, err, proc)
 
 		switch (trp->tr_state) {
 		case TRANS_DONE:
-	    	ATENABLE(s, atp->atp_lock);
 			if ((*err = copyin(CAST_USER_ADDR_T(bdsp),
 					(caddr_t)bds, sizeof(bds))) != 0) {
 				atp_free(trp);
@@ -1950,7 +1876,6 @@ _ATPgetrsp(fd, bdsp, err, proc)
 			/*
 			 * transaction timed out, return error
 			 */
-	    	ATENABLE(s, atp->atp_lock);
 			atp_free(trp);
 			file_drop(fd);
 			*err = ETIMEDOUT;
@@ -1960,7 +1885,6 @@ _ATPgetrsp(fd, bdsp, err, proc)
 			continue;
 	    }
 	}
-	ATENABLE(s, atp->atp_lock);
 
 	file_drop(fd);
 	*err = EINVAL;
@@ -1972,7 +1896,6 @@ atp_drop_req(gref, m)
 	gref_t *gref;
 	gbuf_t *m;
 {
-	int s;
 	struct atp_state *atp;
 	struct atp_rcb *rcbp;
 	at_atp_t *athp;
@@ -1987,9 +1910,8 @@ atp_drop_req(gref, m)
 	/*
 	 *	search for the corresponding rcb
 	 */
-	ATDISABLE(s, atp->atp_lock);
 	for (rcbp = atp->atp_rcb.head; rcbp; rcbp = rcbp->rc_list.next) {
-	    if ( (rcbp->rc_tid == UAS_VALUE(athp->tid)) &&
+	    if ( (rcbp->rc_tid == UAS_VALUE_NTOH(athp->tid)) &&
 			(rcbp->rc_socket.node == ddp->src_node) &&
 			(rcbp->rc_socket.net == NET_VALUE(ddp->src_net)) &&
 			(rcbp->rc_socket.socket == ddp->src_socket) )
@@ -2001,7 +1923,6 @@ atp_drop_req(gref, m)
 	 */
 	if (rcbp)
 		atp_rcb_free(rcbp);
-	ATENABLE(s, atp->atp_lock);
 
 	gbuf_freem(m);
 }
diff --git a/bsd/netat/aurp.h b/bsd/netat/aurp.h
index 718092d4c..9df123b1b 100644
--- a/bsd/netat/aurp.h
+++ b/bsd/netat/aurp.h
@@ -33,6 +33,8 @@
  *	File: aurp.h
  */
 
+#ifdef AURP_SUPPORT
+
 #ifndef _NETAT_AURP_H_
 #define	_NETAT_AURP_H_
 #include <sys/appleapiopts.h>
@@ -172,9 +174,6 @@ typedef struct {
 	unsigned short flags;
 } aurp_hdr_t;
 
-#ifdef AURP_SUPPORT
-
-extern atlock_t aurpgen_lock;
 extern gref_t *aurp_gref;
 extern unsigned char dst_addr_cnt;
 extern unsigned char net_access_cnt;
@@ -184,7 +183,6 @@ extern int net_port;
 extern int update_tmo;
 extern aurp_state_t aurp_state[];
 extern unsigned short net_access[];
-#endif
 
 struct myq
 {	struct mbuf *q_head;
@@ -192,7 +190,6 @@ struct myq
 	int q_cnt;
 };
 
-#define LOCK_DECL(x)	atlock_t x
 
 #include <sys/uio_internal.h>
 
@@ -295,3 +292,5 @@ extern struct aurp_global_t aurp_global;
 #endif /* KERNEL_PRIVATE */
 #endif /* __APPLE_API_OBSOLETE */
 #endif /* _NETAT_AURP_H_ */
+
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_aurpd.c b/bsd/netat/aurp_aurpd.c
index 757aa3188..7fd1e7eed 100644
--- a/bsd/netat/aurp_aurpd.c
+++ b/bsd/netat/aurp_aurpd.c
@@ -38,6 +38,7 @@
  * Kernel process to implement the AURP daemon:
  *  manage tunnels to remote AURP servers across IP networks
  */
+#ifdef AURP_SUPPORT
 
 #include <sys/errno.h>
 #include <sys/types.h>
@@ -109,7 +110,6 @@ aurpd_start()
 	 */
 	bzero((char *)&aurp_global.tunnel, sizeof(aurp_global.tunnel));
 	/*lock_alloc(&aurp_global.glock, LOCK_ALLOC_PIN, AURP_EVNT_LOCK, -1);*/
-	ATLOCKINIT(aurp_global.glock);
 	ATEVENTINIT(aurp_global.event_anchor);
 
 	/* open udp socket */
@@ -209,7 +209,6 @@ AURPgetmsg(err)
 		 *	     when a packet arrives
 		 */
 
-		ATDISABLE(s, aurp_global.glock);
 		events = aurp_global.event;
 		if (((*err == 0) || (*err == EWOULDBLOCK)) && events == 0)
 		  {
@@ -218,7 +217,6 @@ AURPgetmsg(err)
 		    events = aurp_global.event;
 		    aurp_global.event = 0;
 		  }	
-		ATENABLE(s, aurp_global.glock);	 
 
 		/*
 		 * Shut down if we have the AE_SHUTDOWN event or if we got
@@ -288,9 +286,7 @@ AURPgetmsg(err)
 				 * which will wake us from the sleep at
 				 * the top of the outer loop.
 				 */
-				ATDISABLE(s, aurp_global.glock);
 				aurp_global.event &= ~AE_UDPIP;
-				ATENABLE(s, aurp_global.glock);
 				dPrintf(D_M_AURP, D_L_WARNING, ("AURPgetmsg: spurious soreceive, err==%d, p_mbuf==0x%x\n", *err, (unsigned int) p_mbuf));
 			  break;
 		}
@@ -310,9 +306,7 @@ void aurp_wakeup(__unused struct socket *so, register caddr_t p, __unused int st
 	register int bit;
 
 	bit = (int) p;
-	ATDISABLE(s, aurp_global.glock);
 	aurp_global.event |= bit;
-	ATENABLE(s, aurp_global.glock);
 
 	dPrintf(D_M_AURP, D_L_STATE_CHG,
 		("aurp_wakeup: bit 0x%x, aurp_global.event now 0x%x\n",
@@ -428,14 +422,10 @@ atalk_to_ip(register gbuf_t *m)
 	domain = (aurp_domain_t *)gbuf_rptr(m);
 	*(long *) &rem_addr.sin_addr = domain->dst_address;
 
-	ATDISABLE(s, aurp_global.glock);
 	aurp_global.running++;
-	ATENABLE(s, aurp_global.glock);
 	if (aurp_global.shutdown) {
 		gbuf_freem(m);
-			ATDISABLE(s, aurp_global.glock);
 		aurp_global.running--;
-		ATENABLE(s, aurp_global.glock);
 		dPrintf(D_M_AURP, D_L_SHUTDN_INFO,
 			("atalk_to_ip: detected aurp_global.shutdown state\n"));
 		return;
@@ -448,9 +438,8 @@ atalk_to_ip(register gbuf_t *m)
 		  error));
 	}
 
-	ATDISABLE(s, aurp_global.glock);
 	aurp_global.running--;
-	ATENABLE(s, aurp_global.glock);
 	return;
 }
 
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_cfg.c b/bsd/netat/aurp_cfg.c
index 052caef8e..7f610a987 100644
--- a/bsd/netat/aurp_cfg.c
+++ b/bsd/netat/aurp_cfg.c
@@ -33,6 +33,9 @@
  *
  *	File: cfg.c
  */
+ 
+#ifdef AURP_SUPPORT
+
 #define RESOLVE_DBG
 #include <sys/errno.h>
 #include <sys/types.h>
@@ -55,7 +58,6 @@
 #include <netat/at_pcb.h>
 #include <netat/aurp.h>
 
-extern atlock_t aurpgen_lock;
 static int aurp_inited = 0;
 static char aurp_minor_no[4];
 
@@ -65,10 +67,8 @@ int aurp_open(gref)
 	extern void AURPcmdx();
 	int i;
 
-	if (!aurp_inited) {
+	if (!aurp_inited)
 		aurp_inited = 1;
-		ATLOCKINIT(aurpgen_lock);
-	}
 
 	for (i=1; i < sizeof(aurp_minor_no); i++) {
 		if (aurp_minor_no[i] == 0) {
@@ -104,3 +104,5 @@ int aurp_close(gref)
 	gref->info = 0;
 	return 0;
 }
+
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_gdata.c b/bsd/netat/aurp_gdata.c
index 6851a6005..3d1e9eeae 100644
--- a/bsd/netat/aurp_gdata.c
+++ b/bsd/netat/aurp_gdata.c
@@ -33,6 +33,9 @@
  *
  *	File: gdata.c
  */
+ 
+#ifdef AURP_SUPPORT
+
 #include <sys/errno.h>
 #include <sys/types.h>
 #include <sys/param.h>
@@ -55,7 +58,6 @@
 #include <netat/at_pcb.h>
 #include <netat/aurp.h>
 
-atlock_t aurpgen_lock;
 gref_t *aurp_gref;
 unsigned char dst_addr_cnt;
 unsigned char net_access_cnt;
@@ -65,3 +67,5 @@ int net_port;
 int update_tmo;
 aurp_state_t aurp_state[256];
 unsigned short net_access[AURP_MaxNetAccess];
+
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_misc.c b/bsd/netat/aurp_misc.c
index 95ca09a33..570e0fe86 100644
--- a/bsd/netat/aurp_misc.c
+++ b/bsd/netat/aurp_misc.c
@@ -33,6 +33,9 @@
  *
  *	File: misc.c
  */
+ 
+#ifdef AURP_SUPPORT
+
 #include <sys/errno.h>
 #include <sys/types.h>
 #include <sys/param.h>
@@ -214,3 +217,5 @@ void AURPaccess()
 			entry->AURPFlag = net_export ? 0 : AURP_NetHiden;
 	}
 }
+
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_open.c b/bsd/netat/aurp_open.c
index 40967a809..1d8fac9d8 100644
--- a/bsd/netat/aurp_open.c
+++ b/bsd/netat/aurp_open.c
@@ -33,6 +33,9 @@
  *
  *	File: open.c
  */
+ 
+#ifdef AURP_SUPPORT
+
 #include <sys/errno.h>
 #include <sys/types.h>
 #include <sys/param.h>
@@ -252,3 +255,5 @@ void AURPrcvOpenRsp(state, m)
 	/* get routing info */
 	AURPsndRIReq(state);
 }
+
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_rd.c b/bsd/netat/aurp_rd.c
index dc333e2fc..d0016917d 100644
--- a/bsd/netat/aurp_rd.c
+++ b/bsd/netat/aurp_rd.c
@@ -33,6 +33,9 @@
  *
  *	File: rd.c
  */
+ 
+#ifdef AURP_SUPPORT
+
 #include <sys/errno.h>
 #include <sys/types.h>
 #include <sys/param.h>
@@ -121,3 +124,5 @@ void AURPrcvRDReq(state, m)
 	/* respond to the going-down peer with an RI Ack packet */
 	AURPsndRIAck(state, m, 0);
 }
+
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_ri.c b/bsd/netat/aurp_ri.c
index c7ad8b63e..c4f36f4a0 100644
--- a/bsd/netat/aurp_ri.c
+++ b/bsd/netat/aurp_ri.c
@@ -33,6 +33,9 @@
  *
  *	File: ri.c
  */
+ 
+#ifdef AURP_SUPPORT
+
 #include <sys/errno.h>
 #include <sys/types.h>
 #include <sys/param.h>
@@ -158,14 +161,12 @@ void AURPsndRIRsp(state)
 	gbuf_t *m;
 	aurp_hdr_t *hdrp;
 	short len = 0;
-	int s, msize = 0;
+	int msize = 0;
 
-	ATDISABLE(s, aurpgen_lock);
 
 	/* make sure we're in a valid state to send RI response */
 	if ((state->snd_state == AURPSTATE_Unconnected) ||
 		(state->snd_state == AURPSTATE_WaitingForRIAck2)) {
-		ATENABLE(s, aurpgen_lock);
 		return;
 	}
 
@@ -173,7 +174,6 @@ void AURPsndRIRsp(state)
 	state->snd_state = AURPSTATE_WaitingForRIAck1;
 
 	if (state->rsp_m == 0) {
-		ATENABLE(s, aurpgen_lock);
 		msize = sizeof(aurp_hdr_t);
 		if ((m = (gbuf_t *)gbuf_alloc(msize+AURP_MaxPktSize, PRI_MED)) == 0) {
 			timeout(AURPsndRIRsp_locked, state, AURP_RetryInterval*HZ);
@@ -207,8 +207,6 @@ void AURPsndRIRsp(state)
 	timeout(AURPsndRIRsp_locked, state, AURP_RetryInterval*HZ);
 	state->snd_tmo = 1;
 
-	if (msize == 0)
-		ATENABLE(s, aurpgen_lock);
 
 	/* send the packet */
 	if (m) {
@@ -235,13 +233,11 @@ void AURPsndRIUpd(state)
 	short len = 0;
 	int s, msize = 0;
 
-	ATDISABLE(s, aurpgen_lock);
 
 	/* make sure we're in a valid state to send update */
 	if (state->snd_next_entry || (state->upd_m == 0) ||
 		(state->snd_state == AURPSTATE_Unconnected) ||
 			(state->snd_state == AURPSTATE_WaitingForRIAck1)) {
-		ATENABLE(s, aurpgen_lock);
 		return;
 	}
 
@@ -249,7 +245,6 @@ void AURPsndRIUpd(state)
 	state->snd_state = AURPSTATE_WaitingForRIAck2;
 
 	if (state->snd_tmo == 0) {
-		ATENABLE(s, aurpgen_lock);
 		msize = sizeof(aurp_hdr_t);
 		m = state->upd_m;
 		len = gbuf_len(m);
@@ -270,8 +265,6 @@ void AURPsndRIUpd(state)
 	timeout(AURPsndRIUpd_locked, state, AURP_RetryInterval*HZ);
 	state->snd_tmo = 1;
 
-	if (msize == 0)
-		ATENABLE(s, aurpgen_lock);
 
 	/* send the packet */
 	if (m) {
@@ -289,12 +282,10 @@ void AURPrcvRIReq(state, m)
 	aurp_hdr_t *hdrp = (aurp_hdr_t *)gbuf_rptr(m);
 	int s;
 
-	ATDISABLE(s, aurpgen_lock);
 
 	/* make sure we're in a valid state to accept it */
 	if ((state->snd_state == AURPSTATE_Unconnected) ||
 			(state->snd_state == AURPSTATE_WaitingForRIAck2)) {
-		ATENABLE(s, aurpgen_lock);
 		dPrintf(D_M_AURP, D_L_WARNING, ("AURPrcvRIReq: unexpected request\n"));
 		gbuf_freem(m);
 		return;
@@ -302,7 +293,6 @@ void AURPrcvRIReq(state, m)
 
 	/* check for the correct connection id */
 	if (hdrp->connection_id != state->snd_connection_id) {
-		ATENABLE(s, aurpgen_lock);
 		dPrintf(D_M_AURP, D_L_WARNING,
 			("AURPrcvRIReq: invalid connection id, r=%d, m=%d\n",
 			hdrp->connection_id, state->snd_connection_id));
@@ -316,10 +306,8 @@ void AURPrcvRIReq(state, m)
 			gbuf_freem(state->rsp_m);
 			state->rsp_m = 0;
 		}
-		ATENABLE(s, aurpgen_lock);
 		AURPsndRIRsp(state);
-	} else
-		ATENABLE(s, aurpgen_lock);
+	} 
 
 	gbuf_freem(m);
 }
@@ -330,13 +318,10 @@ void AURPrcvRIRsp(state, m)
 	gbuf_t *m;
 {
 	aurp_hdr_t *hdrp = (aurp_hdr_t *)gbuf_rptr(m);
-	int s;
 
-	ATDISABLE(s, aurpgen_lock);
 
 	/* make sure we're in a valid state to accept it */
 	if (state->rcv_state != AURPSTATE_WaitingForRIRsp) {
-		ATENABLE(s, aurpgen_lock);
 		dPrintf(D_M_AURP, D_L_WARNING, ("AURPrcvRIRsp: unexpected response\n"));
 		gbuf_freem(m);
 		return;
@@ -344,7 +329,6 @@ void AURPrcvRIRsp(state, m)
 
 	/* check for the correct connection id */
 	if (hdrp->connection_id != state->rcv_connection_id) {
-		ATENABLE(s, aurpgen_lock);
 		dPrintf(D_M_AURP, D_L_WARNING,
 			("AURPrcvRIRsp: invalid connection id, r=%d, m=%d\n",
 			hdrp->connection_id, state->rcv_connection_id));
@@ -354,7 +338,6 @@ void AURPrcvRIRsp(state, m)
 
 	/* check for the correct sequence number */
 	if (hdrp->sequence_number != state->rcv_sequence_number) {
-		ATENABLE(s, aurpgen_lock);
 		if ( ((state->rcv_sequence_number == AURP_FirstSeqNum) &&
 			(hdrp->sequence_number == AURP_LastSeqNum)) ||
 		(hdrp->sequence_number == (state->rcv_sequence_number-1)) ) {
@@ -370,7 +353,6 @@ void AURPrcvRIRsp(state, m)
 	gbuf_rinc(m,sizeof(*hdrp));
 	if (hdrp->flags & AURPFLG_LAST)
 		state->rcv_state = AURPSTATE_Connected;
-	ATENABLE(s, aurpgen_lock);
 
 	dPrintf(D_M_AURP, D_L_INFO, ("AURPrcvRIRsp: len=%ld\n", gbuf_len(m)));
 
@@ -463,12 +445,10 @@ void AURPrcvRIAck(state, m)
 	gbuf_t *dat_m;
 	aurp_hdr_t *hdrp = (aurp_hdr_t *)gbuf_rptr(m);
 	unsigned char snd_state;
-	int s;
 	int flag;
 
 	dPrintf(D_M_AURP, D_L_INFO, ("AURPrcvRIAck: state=%d\n",
 		state->snd_state));
-	ATDISABLE(s, aurpgen_lock);
 
 	/* make sure we're in a valid state to accept it */
 	snd_state = state->snd_state;
@@ -498,7 +478,6 @@ void AURPrcvRIAck(state, m)
 
 		/* update state info */
 		state->snd_state = AURPSTATE_Connected;
-		ATENABLE(s, aurpgen_lock);
 
 		if (state->snd_next_entry) /* more RI responses to send? */
 			AURPsndRIRsp(state);
@@ -508,8 +487,7 @@ void AURPrcvRIAck(state, m)
 			AURPsndZRsp(state, dat_m, flag);
 		else if (dat_m)
 			gbuf_freem(dat_m);
-	} else
-		ATENABLE(s, aurpgen_lock);
+	} 
 
 	gbuf_freem(m);
 }
@@ -837,17 +815,14 @@ void AURPrtupdate(entry, ev)
 				(!(state->snd_sui & AURPFLG_ND))) continue;
 		if ((ev == AURPEV_NetDistChange) &&
 				(!(state->snd_sui & AURPFLG_NDC))) continue;
-		ATDISABLE(s, aurpgen_lock);
 	  if ((state->snd_state != AURPSTATE_Unconnected) &&
 			(state->snd_state != AURPSTATE_WaitingForRIAck2)) {
 		if ((m = state->upd_m) == 0) {
 			/*
 			 * we don't have the RI update buffer yet, allocate one
 			 */
-			ATENABLE(s, aurpgen_lock);
 			if ((m = (gbuf_t *)gbuf_alloc(msize+AURP_MaxPktSize, PRI_HI)) == 0)
 				continue;
-			ATDISABLE(s, aurpgen_lock);
 			state->upd_m = m;
 			gbuf_rinc(m,msize);
 			gbuf_wset(m,0);
@@ -865,11 +840,11 @@ void AURPrtupdate(entry, ev)
 		 * if the RI update buffer is full, send the RI update now
 		 */
 		if (gbuf_len(m) > (AURP_MaxPktSize-6)) {
-			ATENABLE(s, aurpgen_lock);
 			AURPsndRIUpd(state);
 			continue;
 		}
 	  }
-		ATENABLE(s, aurpgen_lock);
 	}
 }
+
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_rx.c b/bsd/netat/aurp_rx.c
index 861f7c2d1..a10dbaf62 100644
--- a/bsd/netat/aurp_rx.c
+++ b/bsd/netat/aurp_rx.c
@@ -34,6 +34,8 @@
  *
  *	File: rx.c
  */
+#ifdef AURP_SUPPORT
+
 #include <sys/errno.h>
 #include <sys/types.h>
 #include <sys/param.h>
@@ -213,3 +215,5 @@ at_insert(m, type, node)
 
 	return 0;
 }
+
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_tickle.c b/bsd/netat/aurp_tickle.c
index 09dce012f..dd6d9769e 100644
--- a/bsd/netat/aurp_tickle.c
+++ b/bsd/netat/aurp_tickle.c
@@ -33,6 +33,9 @@
  *
  *	File: tickle.c
  */
+ 
+#ifdef AURP_SUPPORT
+
 #include <sys/errno.h>
 #include <sys/types.h>
 #include <sys/param.h>
@@ -162,3 +165,5 @@ void AURPrcvTickleAck(state, m)
 	/* update state info */
 	state->tickle_retry = 0;
 }
+
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_tx.c b/bsd/netat/aurp_tx.c
index 729b6e953..ee87d91c2 100644
--- a/bsd/netat/aurp_tx.c
+++ b/bsd/netat/aurp_tx.c
@@ -33,6 +33,9 @@
  *
  *	File: tx.c
  */
+ 
+#ifdef AURP_SUPPORT
+
 #include <sys/errno.h>
 #include <sys/types.h>
 #include <sys/param.h>
@@ -145,3 +148,5 @@ void AURPcmdx(code, mdata, param)
 		dPrintf(D_M_AURP, D_L_ERROR, ("AURPcmdx: bad code, %d\n", code));
 	}
 }
+
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_zi.c b/bsd/netat/aurp_zi.c
index d491a751f..beb7146a0 100644
--- a/bsd/netat/aurp_zi.c
+++ b/bsd/netat/aurp_zi.c
@@ -33,6 +33,9 @@
  *
  *	File: zi.c
  */
+ 
+#ifdef AURP_SUPPORT
+
 #include <sys/errno.h>
 #include <sys/types.h>
 #include <sys/param.h>
@@ -617,3 +620,5 @@ AURPsetzi(node, m, sub_code, tuples_cnt)
 			buf += zname->len+1;
 	}
 }
+
+#endif /* AURP_SUPPORT */
diff --git a/bsd/netat/ddp.c b/bsd/netat/ddp.c
index 187a05676..8b46b79eb 100644
--- a/bsd/netat/ddp.c
+++ b/bsd/netat/ddp.c
@@ -94,8 +94,6 @@ void (*ddp_AURPsendx)();
 at_ifaddr_t *aurp_ifID = 0;
 extern pktsIn,pktsOut;
 int pktsDropped,pktsHome;
-atlock_t ddpall_lock;
-atlock_t ddpinp_lock;
 
 extern int *atp_pidM;
 extern int *adsp_pidM;
@@ -458,7 +456,6 @@ void ddp_notify_nbp(socket, pid, ddptype)
 
 	if (at_state.flags & AT_ST_STARTED) {
 		/* *** NBP_CLOSE_NOTE processing (from ddp_nbp.c) *** */
-   		ATDISABLE(nve_lock, NVE_LOCK);
                 for ((nve_entry = TAILQ_FIRST(&name_registry)); nve_entry; nve_entry = nve_next) {
                         nve_next = TAILQ_NEXT(nve_entry, nve_link);
 			if ((at_socket)socket == nve_entry->address.socket &&
@@ -470,7 +467,6 @@ void ddp_notify_nbp(socket, pid, ddptype)
 				nbp_delete_entry(nve_entry);
 			}
 		}
-		ATENABLE(nve_lock, NVE_LOCK);
 	}
 } /* ddp_notify_nbp */
 
@@ -485,12 +481,12 @@ static void fillin_pkt_chain(m)
 
 	if (UAS_VALUE(ddp->checksum)) {
 		tmp = ddp_checksum(m, 4);
-		UAS_ASSIGN(ddp->checksum, tmp);
+		UAS_ASSIGN_HTON(ddp->checksum, tmp);
 	}
 
 	for (tmp_m=gbuf_next(tmp_m); tmp_m; tmp_m=gbuf_next(tmp_m)) {
 		tmp_ddp = (at_ddp_t *)gbuf_rptr(tmp_m);
-		tmp_ddp->length = gbuf_msgsize(tmp_m);
+		DDPLEN_ASSIGN(tmp_ddp, gbuf_msgsize(tmp_m));
 		tmp_ddp->hopcount = 
 		  tmp_ddp->unused = 0;
 		NET_NET(tmp_ddp->src_net, ddp->src_net);
@@ -498,7 +494,7 @@ static void fillin_pkt_chain(m)
 		tmp_ddp->src_socket = ddp->src_socket;
 		if (UAS_VALUE(tmp_ddp->checksum)) {
 			tmp = ddp_checksum(tmp_m, 4);
-			UAS_ASSIGN(tmp_ddp->checksum, tmp);
+			UAS_ASSIGN_HTON(ddp->checksum, tmp);
 		}
 	}
 }
@@ -579,7 +575,7 @@ int ddp_output(mp, src_socket, src_addr_included)
 	at_ddp_stats.xmit_bytes += len;
 	at_ddp_stats.xmit_packets++;
 
-	ddp->length = len;
+	DDPLEN_ASSIGN(ddp, len);
 	ddp->hopcount = 
 	  ddp->unused = 0;
 
@@ -859,7 +855,7 @@ int ddp_output(mp, src_socket, src_addr_included)
 	     * it doesn't know net#, consequently can't do 
 	     * AMT_LOOKUP.  That task left to aarp now.
 	     */
-	    aarp_send_data(m,ifID,&dest_at_addr, loop);
+	    aarp_send_data(m,ifID, &dest_at_addr, loop);
 	    break;
 	case ET_ADDR :
 	    pat_output(ifID, m, &dest_addr, 0);
@@ -942,7 +938,7 @@ void ddp_input(mp, ifID)
 	 * {extended ddp, ... }.
 	 */
 	ddp = (at_ddp_t *)gbuf_rptr(mp);
-	len = ddp->length;
+	len = DDPLEN_VALUE(ddp);
 
 	if (msgsize != len) {
 	        if ((unsigned) msgsize > len) {
@@ -991,10 +987,10 @@ void ddp_input(mp, ifID)
 	 * if the checksum is true, then upstream wants us to calc
 	 */
 	if (UAS_VALUE(ddp->checksum) && 
-           (UAS_VALUE(ddp->checksum) != ddp_checksum(mp, 4))) {
+           (UAS_VALUE_NTOH(ddp->checksum) != ddp_checksum(mp, 4))) {
 		dPrintf(D_M_DDP, D_L_WARNING,
 			("Checksum error on incoming pkt, calc 0x%x, exp 0x%x",
-			ddp_checksum(mp, 4), UAS_VALUE(ddp->checksum)));
+			ddp_checksum(mp, 4), UAS_VALUE_NTOH(ddp->checksum)));
 		snmpStats.dd_checkSum++;
 		at_ddp_stats.rcv_bad_checksum++;
 		gbuf_freem(mp);
@@ -1153,6 +1149,7 @@ int ddp_router_output(mp, ifID, addr_type, router_net, router_node, enet_addr)
 	}
 	ddp = (at_ddp_t *)gbuf_rptr(mp);
 
+#ifdef AURP_SUPPORT
 	if (ifID->ifFlags & AT_IFF_AURP) { /* AURP link? */
 		if (ddp_AURPsendx) {
 			fillin_pkt_chain(mp);
@@ -1165,6 +1162,7 @@ int ddp_router_output(mp, ifID, addr_type, router_net, router_node, enet_addr)
 			return EPROTOTYPE;
 		}
 	}
+#endif
 
 	/* keep some of the tests for now ####### */
 
@@ -1317,15 +1315,14 @@ void rt_delete(NetStop, NetStart)
 	RT_entry *found;
 	int s;
 
-	ATDISABLE(s, ddpinp_lock);
 	if ((found = rt_bdelete(NetStop, NetStart)) != 0) {
 		bzero(found, sizeof(RT_entry));
 		found->right = RT_table_freelist;
 		RT_table_freelist = found;
 	}
-	ATENABLE(s, ddpinp_lock);
 }
 
+#ifdef AURP_SUPPORT
 int ddp_AURPfuncx(code, param, node)
 	int code;
 	void *param;
@@ -1397,7 +1394,7 @@ int ddp_AURPfuncx(code, param, node)
 
 	return 0;
 }
-
+#endif
 
 /* checks to see if address of packet is for one of our interfaces
    returns *ifID if it's for us, NULL if not
diff --git a/bsd/netat/ddp.h b/bsd/netat/ddp.h
index e2e8b3909..0554ddf0e 100644
--- a/bsd/netat/ddp.h
+++ b/bsd/netat/ddp.h
@@ -68,9 +68,17 @@
 /* DDP extended header packet format */
 
 typedef struct {
-        unsigned   unused:2,
-		   hopcount:4,
-		   length:10;  		/* Datagram length */
+#if BYTE_ORDER == BIG_ENDIAN
+		unsigned	unused:2,
+        		    hopcount:4,  	/* hop count/len high order  */
+        			length_H:2;	
+#endif
+#if BYTE_ORDER == LITTLE_ENDIAN
+		unsigned	length_H:2,
+					hopcount:4,
+					unused:2;
+#endif
+        u_char	   length_L;			/* len low order */
         ua_short   checksum;    	/* Checksum */
         at_net     dst_net;  		/* Destination network number */
         at_net     src_net;  		/* Source network number */
@@ -78,12 +86,17 @@ typedef struct {
         at_node    src_node;  		/* Source node ID */
         at_socket  dst_socket; 		/* Destination socket number */
         at_socket  src_socket; 		/* Source socket number */
-        u_char	   type;  		/* Protocol type */
+        u_char	   type;  			/* Protocol type */
         char       data[DDP_DATA_SIZE];
 } at_ddp_t;
 
-#define	DDPLEN_ASSIGN(ddp, len)		ddp->length = len
-#define	DDPLEN_VALUE(ddp)		ddp->length
+
+#define	DDPLEN_ASSIGN(ddp, len)		\
+		ddp->length_H = 0x03 & (len >> 8); \
+		ddp->length_L = len & 0xff;
+		
+#define	DDPLEN_VALUE(ddp)			\
+		(((u_short)ddp->length_H) << 8) + ddp->length_L
 
 /* DDP module statistics and configuration */
 
diff --git a/bsd/netat/ddp_aarp.c b/bsd/netat/ddp_aarp.c
index d808f1892..9e6975b89 100644
--- a/bsd/netat/ddp_aarp.c
+++ b/bsd/netat/ddp_aarp.c
@@ -99,7 +99,6 @@ StaticProc void aarp_sched_req(void *);
 StaticProc int aarp_get_rand_node();
 StaticProc int aarp_get_next_node();
 StaticProc int aarp_get_rand_net();
-atlock_t arpinp_lock;
 
 extern void AARPwakeup(aarp_amt_t *);
 extern int pat_output(at_ifaddr_t *, gbuf_t *, unsigned char *, int);
@@ -187,7 +186,7 @@ int aarp_rcv_pkt(pkt, elapp)
      aarp_pkt_t *pkt;
      at_ifaddr_t *elapp;
 {
-	switch (pkt->aarp_cmd) {
+	switch (ntohs(pkt->aarp_cmd)) {
 	case AARP_REQ_CMD:
 		return (aarp_req_cmd_in (pkt, elapp));
 	case AARP_RESP_CMD:
@@ -287,7 +286,7 @@ StaticProc int aarp_resp_cmd_in (pkt, elapp)
 		break;
 
 	case PROBE_DONE :
-		AMT_LOOK(amt_ptr, pkt->src_at_addr, elapp);
+		AMT_LOOK(amt_ptr, pkt->src_at_addr, elapp)
 		if (amt_ptr == NULL)
 			return(-1);
 		if (amt_ptr->tmo) {
@@ -428,18 +427,18 @@ int aarp_chk_addr(ddp_hdrp, elapp)
  * 	will ALWAYS be removed.  If the message is dropped,
  *	it's not an "error".
  *
+ *  Parameter dest_at_addr must have the net # in network byte order
  ****************************************************************************/
 
 int	aarp_send_data(m, elapp, dest_at_addr, loop)
      register gbuf_t	*m;
      register at_ifaddr_t  *elapp;
-     struct  atalk_addr	   *dest_at_addr;
+     struct  atalk_addr	   *dest_at_addr;	/* net# in network byte order */
      int		loop;			/* if true, loopback broadcasts */
 {
 	register aarp_amt_t	*amt_ptr;
 	register at_ddp_t	*ddp_hdrp;
 	int			error;
-	int s;
 	struct timeval timenow;
 	getmicrouptime(&timenow);
 
@@ -457,22 +456,19 @@ int	aarp_send_data(m, elapp, dest_at_addr, loop)
 		ddp_input(m, elapp);
 		return(0);
 	}
-	ATDISABLE(s, arpinp_lock);
 	AMT_LOOK(amt_ptr, *dest_at_addr, elapp);
 
 
 	if (amt_ptr) {
-	        if (amt_ptr->m) {
+	    if (amt_ptr->m) {
 		        /*
 			 * there's already a packet awaiting transmission, so
 			 * drop this one and let the upper layer retransmit
 			 * later.
 			 */
-			ATENABLE(s, arpinp_lock);
 		        gbuf_freel(m);
 			return (0);
 		}
-		ATENABLE(s, arpinp_lock);
 		return (pat_output(elapp, m,
 				   (unsigned char *)&amt_ptr->dest_addr, 0));
         }
@@ -484,7 +480,6 @@ int	aarp_send_data(m, elapp, dest_at_addr, loop)
 	        gbuf_t	             *newm = 0;
 		struct	etalk_addr   *dest_addr;
 
-		ATENABLE(s, arpinp_lock);
 		dest_addr =  &elapp->cable_multicast_addr;
 		if (loop)
 			newm = (gbuf_t *)gbuf_dupm(m);
@@ -507,7 +502,7 @@ int	aarp_send_data(m, elapp, dest_at_addr, loop)
 		}
 		return (error);
 	}
-	NEW_AMT(amt_ptr, *dest_at_addr,elapp);
+	NEW_AMT(amt_ptr, *dest_at_addr, elapp)
 
         if (amt_ptr->m) {
 	        /*
@@ -515,7 +510,6 @@ int	aarp_send_data(m, elapp, dest_at_addr, loop)
 		 * drop this one and let the upper layer retransmit
 		 * later.
 		 */
-		ATENABLE(s, arpinp_lock);
 	        gbuf_freel(m);
 		return (0);
 	}
@@ -527,7 +521,6 @@ int	aarp_send_data(m, elapp, dest_at_addr, loop)
 	amt_ptr->m = m;
 	amt_ptr->elapp = elapp;
 	amt_ptr->no_of_retries = 0;
-	ATENABLE(s, arpinp_lock);
 
 	if ((error = aarp_send_req(amt_ptr))) {
 		aarp_delete_amt_info(amt_ptr);
@@ -563,7 +556,7 @@ StaticProc   int	aarp_send_resp(elapp, pkt)
 	new_pkt = (aarp_pkt_t *)gbuf_rptr(m);
 	aarp_build_pkt(new_pkt, elapp);
 
-	new_pkt->aarp_cmd = AARP_RESP_CMD;
+	new_pkt->aarp_cmd = htons(AARP_RESP_CMD);
 	new_pkt->dest_addr =  pkt->src_addr;
 
 	new_pkt->dest_at_addr = pkt->src_at_addr;
@@ -605,7 +598,7 @@ register aarp_amt_t 	*amt_ptr;
 	pkt = (aarp_pkt_t *)gbuf_rptr(m);
 	aarp_build_pkt(pkt, amt_ptr->elapp);
 
-	pkt->aarp_cmd = AARP_REQ_CMD;
+	pkt->aarp_cmd = htons(AARP_REQ_CMD);
 	pkt->dest_addr = et_zeroaddr;
 	pkt->dest_at_addr = amt_ptr->dest_at_addr;
 	pkt->dest_at_addr.atalk_unused = 0;
@@ -648,7 +641,7 @@ StaticProc  int	aarp_send_probe()
 	pkt = (aarp_pkt_t *)gbuf_rptr(m);
 	aarp_build_pkt(pkt, probe_cb.elapp);
 
-	pkt->aarp_cmd = AARP_PROBE_CMD;
+	pkt->aarp_cmd = htons(AARP_PROBE_CMD);
 	pkt->dest_addr = et_zeroaddr;
 
 	ATALK_ASSIGN(pkt->src_at_addr, probe_cb.elapp->initial_addr.s_net,
@@ -706,9 +699,7 @@ register aarp_pkt_t	*pkt;
 at_ifaddr_t	*elapp;
 {
     register aarp_amt_t   *amt_ptr;
-	int s;
 
-	ATDISABLE(s, arpinp_lock);
 	AMT_LOOK(amt_ptr, pkt->src_at_addr, elapp);
 
 	if (amt_ptr == NULL) {
@@ -718,10 +709,7 @@ at_ifaddr_t	*elapp;
 		NEW_AMT(amt_ptr, pkt->src_at_addr,elapp); 
 
 		if (amt_ptr->m)
-		{
-		ATENABLE(s, arpinp_lock);
-		        return(0);     /* no non-busy slots available in the cache */
-		}
+			return(0);     /* no non-busy slots available in the cache */
 		amt_ptr->dest_at_addr = pkt->src_at_addr;
 		amt_ptr->dest_at_addr.atalk_unused = 0;
 
@@ -734,7 +722,6 @@ at_ifaddr_t	*elapp;
 	amt_ptr->dest_addr = pkt->src_addr;
 	if (FDDI_OR_TOKENRING(elapp->aa_ifp->if_type))
 		ddp_bit_reverse(&amt_ptr->dest_addr);
-	ATENABLE(s, arpinp_lock);
 	return(1);
 }
 
@@ -747,9 +734,7 @@ at_ifaddr_t	*elapp;
 StaticProc   int	aarp_delete_amt_info(amt_ptr)
 register aarp_amt_t	*amt_ptr;
 {
-	register s;
 	register gbuf_t		*m;
-	ATDISABLE(s, arpinp_lock);
 	amt_ptr->last_time = 0;
 	ATALK_ASSIGN(amt_ptr->dest_at_addr, 0, 0, 0);
 	amt_ptr->no_of_retries = 0;
@@ -757,11 +742,8 @@ register aarp_amt_t	*amt_ptr;
 	if (amt_ptr->m) {
 	    m = amt_ptr->m;
 	    amt_ptr->m = NULL;    
- 	    ATENABLE(s, arpinp_lock);
 	    gbuf_freel(m);
-        }
-	else
-		ATENABLE(s, arpinp_lock);
+    }
 	return(0);
 }
 
@@ -800,8 +782,8 @@ StaticProc void aarp_build_pkt(pkt, elapp)
      register aarp_pkt_t *pkt;
      at_ifaddr_t *elapp;
 {
-	pkt->hardware_type = AARP_ETHER_HW_TYPE;
-	pkt->stack_type = AARP_AT_PROTO;
+	pkt->hardware_type = htons(AARP_ETHER_HW_TYPE);
+	pkt->stack_type = htons(AARP_AT_PROTO);
 	pkt->hw_addr_len = ETHERNET_ADDR_LEN;
 	pkt->stack_addr_len = AARP_AT_ADDR_LEN;
 	bcopy(elapp->xaddr, pkt->src_addr.etalk_addr_octet, sizeof(elapp->xaddr));
@@ -817,7 +799,7 @@ StaticProc void aarp_build_pkt(pkt, elapp)
 StaticProc void	aarp_sched_req(arg)
      void *arg;
 {
-	int s, i;
+	int i;
 	aarp_amt_t *amt_ptr = (aarp_amt_t *)arg;
 
 	atalk_lock();
@@ -835,21 +817,16 @@ StaticProc void	aarp_sched_req(arg)
 	    /*
 	     * found match - pointer is valid
 	     */
-	    ATDISABLE(s, arpinp_lock);
 	    if (amt_ptr->tmo == 0) {
-	        ATENABLE(s, arpinp_lock);
 			atalk_unlock();
 	        return;
 	    }
 	    if (amt_ptr->no_of_retries < AARP_MAX_REQ_RETRIES) {
-	        ATENABLE(s, arpinp_lock);
 	        if (aarp_send_req(amt_ptr) == 0) {
 				atalk_unlock();
 	            return;
 	        }
-	        ATDISABLE(s, arpinp_lock);
 	    }
-	    ATENABLE(s, arpinp_lock);
 	    aarp_delete_amt_info(amt_ptr);
 	    break;
 	}	
@@ -983,6 +960,8 @@ snmpAarpEnt_t *getAarp(elapId)
 	aarp_amt_t *amtp;
 	static snmpAarpEnt_t  snmp[AMTSIZE];
 	snmpAarpEnt_t  *snmpp;
+	struct atalk_addr addr;
+	u_short  tmp_net;
 
 
 	if (*elapId <0 || *elapId >= IF_TOTAL_MAX)
@@ -999,7 +978,11 @@ snmpAarpEnt_t *getAarp(elapId)
 				 * & etalk_addr positions in the aarp_amt_t struct
 				 * has not changed and copy both at once
 				 */
-			bcopy(&amtp->dest_at_addr, &snmpp->ap_ddpAddr, ENTRY_SIZE);
+			addr.atalk_unused = 0;		
+			tmp_net = UAS_VALUE(amtp->dest_at_addr.atalk_net);
+			NET_ASSIGN(addr.atalk_net, tmp_net);
+			addr.atalk_node = amtp->dest_at_addr.atalk_node;
+			bcopy(&addr, &snmpp->ap_ddpAddr, ENTRY_SIZE);
 			snmpp++;
 			cnt++;
 			
diff --git a/bsd/netat/ddp_lap.c b/bsd/netat/ddp_lap.c
index a55c561ae..7b72871f1 100644
--- a/bsd/netat/ddp_lap.c
+++ b/bsd/netat/ddp_lap.c
@@ -121,8 +121,6 @@ int xpatcnt = 0;
 /* externs */
 extern TAILQ_HEAD(name_registry, _nve_) name_registry;
 extern snmpStats_t	snmpStats;
-extern atlock_t ddpinp_lock;
-extern atlock_t arpinp_lock;
 extern short appletalk_inited;
 extern int adspInited;
 extern struct atpcb ddp_head;
@@ -861,7 +859,7 @@ elap_dataput(m, elapp, addr_flag, addr)
 	     * it doesn't know net#, consequently can't do 
 	     * AMT_LOOKUP.  That task left to aarp now.
 	     */
-	    error = aarp_send_data(m,elapp,&dest_at_addr, loop);
+	    error = aarp_send_data(m, elapp, &dest_at_addr, loop);
 	    break;
 	case ET_ADDR :
 	    error = pat_output(elapp, m, &dest_addr, 0);
@@ -1032,7 +1030,6 @@ void elap_offline(elapp)
 {
 	void	zip_sched_getnetinfo(); /* forward reference */
 	int	errno;
-	int s;
 
 	dPrintf(D_M_ELAP, D_L_SHUTDN_INFO, ("elap_offline:%s\n", elapp->ifName));
 	if (elapp->ifState != LAP_OFFLINE) {
@@ -1046,11 +1043,9 @@ void elap_offline(elapp)
 		(void)at_unreg_mcast(elapp, (caddr_t)&elapp->cable_multicast_addr);
 		elapp->ifState = LAP_OFFLINE;
 
-		ATDISABLE(s, ddpinp_lock);
 		if (MULTIPORT_MODE)
 			RT_DELETE(elapp->ifThisCableEnd,
 				  elapp->ifThisCableStart);
-		ATENABLE(s, ddpinp_lock);
 
 		/* make sure no zip timeouts are left running */
 		elapp->ifGNIScheduled = 0;
@@ -1103,7 +1098,7 @@ int ddp_shutdown(count_only)
 	CCB *sp, *sp_next;
 	gref_t *gref;
 	vm_offset_t temp_rcb_data, temp_state_data;
-	int i, s, active_skts = 0;	/* count of active pids for non-socketized
+	int i, active_skts = 0;	/* count of active pids for non-socketized
 				   AppleTalk protocols */
 
 	/* Network is shutting down... send error messages up on each open
@@ -1112,8 +1107,6 @@ int ddp_shutdown(count_only)
 	     sockets, but return EBUSY and don't complete shutdown. *** 
 	 */
 
-	s = splimp();	/* *** previously contained mismatched locking 
-			   that was ifdef'ed to splimp() *** */
 	if (!count_only)
 		nbp_shutdown();	/* clear all known NVE */
 
@@ -1199,11 +1192,9 @@ int ddp_shutdown(count_only)
 		atalk_notify(gref, ESHUTDOWN);
 	    }
 	}
-	if (count_only) {
-		splx(s);
+	if (count_only)
 		return(active_skts);
 
-	}
 	/* if there are no interfaces in the process of going online, continue shutting down DDP */
 	for (i = 0; i < IF_TOTAL_MAX; i++) {
 		if (at_interfaces[i].startup_inprogress == TRUE)
@@ -1255,7 +1246,6 @@ int ddp_shutdown(count_only)
 	}
 	ddp_start();
 	
-	splx(s);
 	return(0);
 } /* ddp_shutdown */
 
@@ -1316,11 +1306,9 @@ void ZIPwakeup(elapp, ZipError)
      at_ifaddr_t *elapp;
      int ZipError;
 {
-	int s, error = ZipError;
+	int error = ZipError;
 
-	ATDISABLE(s, ddpinp_lock);
 	if ( (elapp != NULL) && elapp->startup_inprogress) {
-		ATENABLE(s, ddpinp_lock);
 
 		/* was ZIPContinue */
 		/* was elapp_online() with jump to ZIP_sleep */
@@ -1355,21 +1343,17 @@ void ZIPwakeup(elapp, ZipError)
 			dPrintf(D_M_ELAP, D_L_STARTUP_INFO,
 				("elap_online: ifZipError=%d\n", error));
 		}
-	} else
-		ATENABLE(s, ddpinp_lock);
+	}
 } /* ZIPwakeup */
 
 void AARPwakeup(probe_cb)
      aarp_amt_t *probe_cb;
 {
-	int s;
 	int errno;
 	at_ifaddr_t *elapp;
 
-	ATDISABLE(s, arpinp_lock);
 	elapp = probe_cb->elapp;
 	if ( (elapp != NULL) && elapp->startup_inprogress && elapp->aa_ifp != 0) {
-		ATENABLE(s, arpinp_lock);
 
 		/* was AARPContinue */
 		errno = aarp_init2(elapp);
@@ -1391,8 +1375,7 @@ void AARPwakeup(probe_cb)
 				("elap_online: aarp_init returns zero\n"));
 			elap_online2(elapp);
 		}
-	} else
-		ATENABLE(s, arpinp_lock);
+	}
 } /* AARPwakeup */
 
 void ddp_bit_reverse(addr)
diff --git a/bsd/netat/ddp_nbp.c b/bsd/netat/ddp_nbp.c
index 875d21c68..7a1db804a 100644
--- a/bsd/netat/ddp_nbp.c
+++ b/bsd/netat/ddp_nbp.c
@@ -82,7 +82,6 @@ extern at_ifaddr_t *ifID_home;
 
 TAILQ_HEAD(name_registry, _nve_) name_registry;
 
-atlock_t 	nve_lock;
 
 /* statics */
 static	int		errno;
@@ -156,14 +155,12 @@ void nbp_shutdown()
 	/* delete all NVE's and release buffers */
 	register nve_entry_t	*nve_entry, *nve_next;
 
-	ATDISABLE(nve_lock_pri,NVE_LOCK);
-        for ((nve_entry = TAILQ_FIRST(&name_registry)); nve_entry; nve_entry = nve_next) {
-                nve_next = TAILQ_NEXT(nve_entry, nve_link);
+	for ((nve_entry = TAILQ_FIRST(&name_registry)); nve_entry; nve_entry = nve_next) {
+			nve_next = TAILQ_NEXT(nve_entry, nve_link);
 
                 /* NB: nbp_delete_entry calls TAILQ_REMOVE */
 		nbp_delete_entry(nve_entry);
 	}
-	ATENABLE(nve_lock_pri,NVE_LOCK);
 
 	if (lzones) {
 		gbuf_freem(lzones);
@@ -439,7 +436,7 @@ void nbp_input(m, ifID)
 				   ("nbp_input: BRREQ: src changed to %d.%d.%d\n",
 					ifID->ifThisNode.s_net,
 					ifID->ifThisNode.s_node, ourSkt));
-				nbp->tuple[0].enu_addr.net = ifID->ifThisNode.s_net;
+				nbp->tuple[0].enu_addr.net = htons(ifID->ifThisNode.s_net);
 				nbp->tuple[0].enu_addr.node = ifID->ifThisNode.s_node;
 				nbp->tuple[0].enu_addr.socket = ourSkt; 
 				ddp->src_socket = NBP_SOCKET;
@@ -535,7 +532,7 @@ static	int	nbp_validate_n_hash (nbp_req, wild_ok, checkLocal)
 #ifdef COMMENTED_OUT
 	{
 		int net,node,skt;
-		net = tuple->enu_addr.net;
+		net = ntohs(tuple->enu_addr.net);
 		node = tuple->enu_addr.node;
 		skt = tuple->enu_addr.socket;
 		dPrintf(D_M_NBP_LOW,D_L_USR4,
@@ -740,7 +737,6 @@ static	nve_entry_t *nbp_search_nve (nbp_req, ifID)
 				(u_int) nbp_req->response));
 	}
 #endif /* NBP_DEBUG */
-	ATDISABLE(nve_lock_pri,NVE_LOCK);
 	TAILQ_FOREACH(nve_entry, &name_registry, nve_link) {
 		if ((nbp_req->nve.zone_hash) && 
 			((nbp_req->nve.zone_hash != 
@@ -823,15 +819,11 @@ static	nve_entry_t *nbp_search_nve (nbp_req, ifID)
 		if (nbp_req->func != NULL) {
 			if ((*(nbp_req->func))(nbp_req, nve_entry) != 0) {
 				/* errno expected to be set by func */
-				ATENABLE(nve_lock_pri,NVE_LOCK);
 				return (NULL);
 			}
-		} else {
-			ATENABLE(nve_lock_pri,NVE_LOCK);
+		} else
 			return (nve_entry);
-		}
 	}
-	ATENABLE(nve_lock_pri,NVE_LOCK);
 
 	errno = 0;
 	return (NULL);
@@ -871,7 +863,9 @@ register nve_entry_t	*nve_entry;
 	 * tuple we want to write. Write it!
 	 */
 	tuple = (at_nbptuple_t *)gbuf_wptr(nbp_req->response);
-	tuple->enu_addr = nve_entry->address;
+	tuple->enu_addr.net = htons(nve_entry->address.net);
+	tuple->enu_addr.node = nve_entry->address.node;
+	tuple->enu_addr.socket = nve_entry->address.socket;
 	tuple->enu_enum = nve_entry->enumerator;
 
         /* tuple is in the compressed (no "filler") format */
@@ -992,7 +986,7 @@ register nbp_req_t	*nbp_req;
 	case NBP_LKUP :
 		ddp->dst_socket = nbp_req->nve.address.socket;
 		ddp->dst_node = nbp_req->nve.address.node;
-		NET_ASSIGN(ddp->dst_net, nbp_req->nve.address.net);
+		NET_ASSIGN_NOSWAP(ddp->dst_net, nbp_req->nve.address.net);
 		nbp->control = NBP_LKUP_REPLY;
 		break;
 	}
@@ -1069,10 +1063,8 @@ getNbpTableSize()
 	register nve_entry_t *nve;
 	register int i=0;
 
-	ATDISABLE(nve_lock_pri,NVE_LOCK);
 	for (nve = TAILQ_FIRST(&name_registry); nve; nve = TAILQ_NEXT(nve, nve_link), i++)
 		i++;
-	ATENABLE(nve_lock_pri,NVE_LOCK);
 	return(i);
 }
 
@@ -1095,7 +1087,6 @@ getNbpTable(p, s, c)
 	else
 		nve = TAILQ_FIRST(&name_registry);
 
-	ATDISABLE(nve_lock_pri,NVE_LOCK);
 	for ( ; nve && c ; nve = TAILQ_NEXT(nve, nve_link), p++,i++) {
 		if (i>= s) {
 			p->nbpe_object = nve->object;
@@ -1103,7 +1094,6 @@ getNbpTable(p, s, c)
 			c--;
 		}
 	}
-	ATENABLE(nve_lock_pri,NVE_LOCK);
 	if (nve) {
 		next = nve;
 		nextNo = i;
@@ -1343,7 +1333,6 @@ nve_entry_t *nbp_find_nve(nve)
 {
 	register nve_entry_t	*nve_entry;
 
-	ATDISABLE(nve_lock_pri,NVE_LOCK);
 	TAILQ_FOREACH(nve_entry, &name_registry, nve_link) {
 		if (nve->zone_hash &&
 		    ((nve->zone_hash != nve_entry->zone_hash) &&
@@ -1363,10 +1352,8 @@ nve_entry_t *nbp_find_nve(nve)
 			continue;
 
 		/* Found a match! */
-		ATENABLE(nve_lock_pri,NVE_LOCK);
 		return (nve_entry);
 	}
-	ATENABLE(nve_lock_pri,NVE_LOCK);
 
 	return (NULL);
 } /* nbp_find_nve */
@@ -1377,22 +1364,19 @@ static int nbp_enum_gen (nve_entry)
 	register int		new_enum = 0;
 	register nve_entry_t	*ne;
 
-	ATDISABLE(nve_lock_pri,NVE_LOCK);
 re_do:
 	TAILQ_FOREACH(ne, &name_registry, nve_link) {
 		if ((*(int *)&ne->address == *(int *)&nve_entry->address) &&
 			(ne->enumerator == new_enum)) {
-			if (new_enum == 255) {
-				ATENABLE(nve_lock_pri,NVE_LOCK);
+			if (new_enum == 255)
 				return(EADDRNOTAVAIL);
-			} else {
+			else {
 				new_enum++;
 				goto re_do;
 			}
 		}
 	}
 
-	ATENABLE(nve_lock_pri,NVE_LOCK);
 	nve_entry->enumerator = new_enum;
 	return (0);
 }
@@ -1445,9 +1429,7 @@ int nbp_new_nve_entry(nve_entry, ifID)
 	new_entry->tag = tag;
 	new_entry->pid =  proc_selfpid();
 
-	ATDISABLE(nve_lock_pri,NVE_LOCK);
 	TAILQ_INSERT_TAIL(&name_registry, new_entry, nve_link);
-	ATENABLE(nve_lock_pri,NVE_LOCK);
 	at_state.flags |= AT_ST_NBP_CHANGED;
 
 #ifdef NBP_DEBUG
diff --git a/bsd/netat/ddp_r_rtmp.c b/bsd/netat/ddp_r_rtmp.c
index 65120d847..8c2c2e8d3 100644
--- a/bsd/netat/ddp_r_rtmp.c
+++ b/bsd/netat/ddp_r_rtmp.c
@@ -98,7 +98,6 @@ extern int elap_online3();
 
 extern pktsIn, pktsOut, pktsDropped, pktsHome;
 extern short ErrorRTMPoverflow, ErrorZIPoverflow;
-extern atlock_t ddpinp_lock;
 extern lck_mtx_t * atalk_mutex;
 
 /*
@@ -683,9 +682,11 @@ static void rtmp_update(ifID, rtmp, tuple_nb)
 					("rtmp_update: Shorter route found %d-%d, update\n",
 					 NewRoute.NetStart, NewRoute.NetStop));
 
+#ifdef AURP_SUPPORT
 			if (ddp_AURPsendx && (aurp_ifID->ifFlags & AT_IFF_AURP))
 				ddp_AURPsendx(AURPCODE_RTUPDATE,
 					      (void *)&NewRoute, AURPEV_NetDistChange);
+#endif
 			}
 		}
 		else { /* no entry found */
@@ -706,10 +707,11 @@ static void rtmp_update(ifID, rtmp, tuple_nb)
 					      NewRoute.NextIRNode, NewRoute.NetDist, NewRoute.NetPort,
 					      NewRoute.EntryState) == (RT_entry *)NULL)
 					ErrorRTMPoverflow = 1;
-
+#ifdef AURP_SUPPORT
 				else if (ddp_AURPsendx && (aurp_ifID->ifFlags & AT_IFF_AURP))
 					ddp_AURPsendx(AURPCODE_RTUPDATE,
 						      (void *)&NewRoute, AURPEV_NetAdded);
+#endif
 			}		
 		}
 
@@ -744,7 +746,6 @@ void rtmp_timeout(ifID)
 register at_ifaddr_t        *ifID;
 {
 		register u_char state;
-		register unsigned int s;
 		short i;
 		RT_entry *en = &RT_table[0];
 
@@ -760,7 +761,6 @@ register at_ifaddr_t        *ifID;
 		if (ifID->ifRouterState > NO_ROUTER)
 			ifID->ifRouterState--;
 
-		ATDISABLE(s, ddpinp_lock);
 		for (i = 0 ; i < RT_maxentry; i++,en++) {
 
 			/* we want to age "learned" nets, not directly connected ones */
@@ -784,10 +784,11 @@ register at_ifaddr_t        *ifID;
 					dPrintf(D_M_RTMP, D_L_INFO,
 						("rtmp_timeout: Bad State for %d-%d (e#%d): remove\n",
 							en->NetStart, en->NetStop, i));
-
+#ifdef AURP_SUPPORT
 				if (ddp_AURPsendx && (aurp_ifID->ifFlags & AT_IFF_AURP))
 					ddp_AURPsendx(AURPCODE_RTUPDATE,
 						(void *)en, AURPEV_NetDeleted);
+#endif
 	
 					/* then clear the bit in the table concerning this entry.
 					If the zone Count reaches zero, remove the entry */
@@ -803,7 +804,6 @@ register at_ifaddr_t        *ifID;
 				}
 			}
 		}
-		ATENABLE(s, ddpinp_lock);
 		timeout(rtmp_timeout, (caddr_t) ifID, 20*SYS_HZ);
 		
 		atalk_unlock();
@@ -992,7 +992,6 @@ static int rtmp_send_table(ifID, DestNet, DestNode, split_hz, socket,
 	short size,status ;
 	register at_ddp_t	*ddp;
 	register short EntNb = 0, sent_tuple = 0;
-	register unsigned int s;
 
 	if (ifID->ifRoutingState < PORT_ONLINE) {
 		dPrintf(D_M_RTMP, D_L_INFO,
@@ -1017,7 +1016,6 @@ static int rtmp_send_table(ifID, DestNet, DestNode, split_hz, socket,
 	ddp = (at_ddp_t *)(gbuf_rptr(m));
 	Buff_ptr = (char *)((char *)ddp + DDP_X_HDR_SIZE + 10); 
 
-	ATDISABLE(s, ddpinp_lock);
 	while (EntNb < RT_maxentry) {
 
 		if (Entry->NetStop && ((Entry->EntryState & 0x0F) >= RTE_STATE_SUSPECT)) {
@@ -1050,7 +1048,6 @@ static int rtmp_send_table(ifID, DestNet, DestNode, split_hz, socket,
 		if (size > (DDP_DATA_SIZE-20)) {
 			DDPLEN_ASSIGN(ddp, size + DDP_X_HDR_SIZE + 10);
 			gbuf_winc(m,size);
-			ATENABLE(s, ddpinp_lock);
 			if (status = ddp_router_output(m, ifID, AT_ADDR,
 				NET_VALUE(DestNet),DestNode, 0)){
 			  dPrintf(D_M_RTMP, D_L_WARNING,
@@ -1072,13 +1069,11 @@ static int rtmp_send_table(ifID, DestNet, DestNode, split_hz, socket,
 				 sent_tuple, ifID->ifPort));
 			sent_tuple = 0;
 			size = 0;
-			ATDISABLE(s, ddpinp_lock);
 		}
 
 		Entry++;
 		EntNb++;
 	}
-	ATENABLE(s, ddpinp_lock);
 
 	/*
 	 * If we have some remaining entries to send, send them now.
@@ -1636,10 +1631,9 @@ void rtmp_purge(ifID)
 	at_ifaddr_t *ifID;
 {
 	u_char state;
-	int i, s;
+	int i;
 	RT_entry *en = &RT_table[0];
 
-	ATDISABLE(s, ddpinp_lock);
 	for (i=0; i < RT_maxentry; i++) {
 		state = en->EntryState & 0x0F;
 		if ((state > RTE_STATE_UNUSED) && (state != RTE_STATE_PERMANENT)
@@ -1649,5 +1643,4 @@ void rtmp_purge(ifID)
 		}
 		en++;
 	}
-	ATENABLE(s, ddpinp_lock);
 }
diff --git a/bsd/netat/ddp_r_zip.c b/bsd/netat/ddp_r_zip.c
index e81a691d3..5f715350a 100644
--- a/bsd/netat/ddp_r_zip.c
+++ b/bsd/netat/ddp_r_zip.c
@@ -446,7 +446,7 @@ static void zip_send_reply_to_query(mreceived, ifID)
 
 	/* access the number of nets requested in the Query */
 	network_count  = *((char *)(ddp_received->data) + 1);
-	NetAsked = (u_short *)(ddp_received->data+ 2);
+	NetAsked = (u_short *)(ddp_received->data + 2);
 
 	/* check the validity of the Query packet */
 
@@ -484,7 +484,7 @@ newPacket:
 	reply_length = 2;	/* 1st byte is ZIP reply code, 2nd is network count */
 
 	for (i = 0 ; i < network_count ; i ++, NetAsked++) {
-	  Entry = rt_blookup(*NetAsked);
+	  Entry = rt_blookup(ntohs(*NetAsked));
 
 	  if (Entry != NULL && ((Entry->EntryState & 0x0F) >= RTE_STATE_SUSPECT) &&
 	      RT_ALL_ZONES_KNOWN(Entry)) { /* this net is well known... */
@@ -547,7 +547,7 @@ newPacket:
 			 * and build a separate packet for each extended network requested
 			 */
 
-	    	zip_send_ext_reply_to_query(mreceived, ifID, Entry, *NetAsked);
+	    	zip_send_ext_reply_to_query(mreceived, ifID, Entry, ntohs(*NetAsked));
 
 	    }
 	  }
@@ -585,7 +585,7 @@ void zip_router_input (m, ifID)
 	register at_ddp_t	*ddp;
 	register at_atp_t	*atp;
 	register at_zip_t	*zip;
-	register u_long	 user_bytes;
+	u_char	 user_bytes[4];
 	register u_short user_byte;
 	
 	/* variables for ZipNotify processing */
@@ -768,8 +768,8 @@ void zip_router_input (m, ifID)
 
 		/* Get the user bytes in network order */
 
-		user_bytes = UAL_VALUE(atp->user_bytes);
-		user_byte = user_bytes >> 24; /* Get the zeroth byte */
+		*((u_long*)user_bytes) = UAL_VALUE(atp->user_bytes);
+		user_byte = user_bytes[0]; /* Get the zeroth byte */
 
 		dPrintf(D_M_ZIP, D_L_INPUT,
 			("zip_input: received a ZIP_ATP command=%d\n", user_byte));
@@ -1158,7 +1158,7 @@ int zip_type_packet (m)
 	register at_atp_t	*atp;
 	register at_ddp_t	*ddp;
 	register at_zip_t	*zip;
-	register u_long	user_bytes;
+	u_char	user_bytes[4];
 	register int	user_byte;
 
 	ddp = (at_ddp_t *)gbuf_rptr(m);
@@ -1177,8 +1177,8 @@ int zip_type_packet (m)
 			else
 				atp = (at_atp_t *)(gbuf_rptr(gbuf_cont(m)));
 			/* Get the user bytes in network order */
-			user_bytes = UAL_VALUE(atp->user_bytes);
-			user_byte = user_bytes >> 24; /* Get the zeroth byte */
+			*((u_long*)user_bytes) = UAL_VALUE(atp->user_bytes);
+			user_byte = user_bytes[0]; /* Get the zeroth byte */
 			if ((user_byte == ZIP_GETMYZONE) ||
 			    (user_byte == ZIP_GETZONELIST) ||
 			    (user_byte == ZIP_GETLOCALZONES))
@@ -1264,8 +1264,7 @@ int zip_handle_getmyzone(ifID, m)
         r_atp->bitmap = 0;
         UAS_UAS(r_atp->tid, atp->tid);
         ulongtmp = 1;
-        ulongtmp = htonl(ulongtmp);
-	UAL_ASSIGN(r_atp->user_bytes, ulongtmp); /* no of zones */
+		UAL_ASSIGN_HTON(r_atp->user_bytes, ulongtmp); /* no of zones */
 
         /* fill up atp data part */
         bcopy((caddr_t) &ifID->ifZoneName, (caddr_t) r_atp->data, ifID->ifZoneName.len+1);
@@ -1479,7 +1478,7 @@ zip_reply_received(m, ifID, reply_type)
 
 	/* access the number of nets provided in the ZIP Reply */
 
-	network_count  = *(u_char *)(gbuf_rptr(m) + DDP_X_HDR_SIZE + 1);
+	network_count  = ntohs(*(u_char *)(gbuf_rptr(m) + DDP_X_HDR_SIZE + 1));
 
 	PacketPtr = (char *)(gbuf_rptr(m) + DDP_X_HDR_SIZE + 2);
 
@@ -1491,7 +1490,7 @@ zip_reply_received(m, ifID, reply_type)
 
 	while (payload_len > 0 && network_count >0) {
 
-		Network = *(at_net_al *)PacketPtr;
+		Network = ntohs(*(at_net_al *)PacketPtr);
 		PacketPtr += 2;
 		zname = (at_nvestr_t *)PacketPtr;
 		if (payload_len)
@@ -1621,8 +1620,7 @@ static void zip_reply_to_getmyzone (ifID, m)
         r_atp->bitmap = 0;
         UAS_UAS(r_atp->tid, atp->tid);
         ulongtmp = 1;
-        ulongtmp = htonl(ulongtmp);
-	UAL_ASSIGN(r_atp->user_bytes, ulongtmp); /* no of zones */
+		UAL_ASSIGN_HTON(r_atp->user_bytes, ulongtmp); /* no of zones */
 
 	data_ptr = (char *)r_atp->data;
 
@@ -1719,7 +1717,7 @@ zip_reply_to_getzonelist (ifID, m)
 
 			/* get the start index from the ATP request */
 
-		StartPoint = (UAL_VALUE(atp->user_bytes) & 0xffff) -1;
+		StartPoint = (UAL_VALUE_NTOH(atp->user_bytes) & 0xffff) -1;
 
 		/* find the next zone to send */
 
@@ -1760,7 +1758,7 @@ zip_reply_to_getzonelist (ifID, m)
 				ulongtmp += 0x01000000;
 
         
-		UAL_ASSIGN(r_atp->user_bytes, ulongtmp); /* # of zones and flag*/
+		UAL_ASSIGN_HTON(r_atp->user_bytes, ulongtmp); /* # of zones and flag*/
 
         size = DDP_X_HDR_SIZE + ATP_HDR_SIZE + PacketLen;
         gbuf_winc(rm,size);
@@ -1831,7 +1829,7 @@ int zip_reply_to_getlocalzones (ifID, m)
 
 	/* get the start index from the ATP request */
 
-	Index_wanted = (UAL_VALUE(atp->user_bytes) & 0xffff) -1;
+	Index_wanted = (UAL_VALUE_NTOH(atp->user_bytes) & 0xffff) -1;
 
 	dPrintf(D_M_ZIP_LOW, D_L_INFO, 
 		("zip_r_GLZ: for station %d:%d Index_wanted = %d\n",
@@ -1959,7 +1957,7 @@ FullPacket:
         r_atp->bitmap = 0;
         UAS_UAS(r_atp->tid, atp->tid);
         ulongtmp =  ((last_flag << 24) & 0xFF000000) + ZonesInPacket; /* # of zones and flag*/
-	UAL_ASSIGN(r_atp->user_bytes, ulongtmp);
+		UAL_ASSIGN_HTON(r_atp->user_bytes, ulongtmp);
         size = DDP_X_HDR_SIZE + ATP_HDR_SIZE + packet_len;
         gbuf_winc(rm,size);
         DDPLEN_ASSIGN(r_ddp, size);
diff --git a/bsd/netat/ddp_rtmptable.c b/bsd/netat/ddp_rtmptable.c
index 81eb400ef..9141b98db 100644
--- a/bsd/netat/ddp_rtmptable.c
+++ b/bsd/netat/ddp_rtmptable.c
@@ -92,7 +92,6 @@ char errstr[512];		/* used to display meaningfull router errors*/
 extern at_ifaddr_t *ifID_table[];
 extern at_ifaddr_t *ifID_home;
 extern snmpStats_t	snmpStats;
-extern atlock_t	ddpinp_lock;
 
 short ErrorRTMPoverflow = 0;	/* flag if RTMP table is too small for this net */
 short ErrorZIPoverflow  = 0;	/* flag if ZIP table is too small for this net */
@@ -152,12 +151,10 @@ at_net_al NetNumber;
 
 	RT_entry *ptree = &RT_table_start;
 	at_net_al LowEnd;
-	register unsigned int s;
 /*
 	dPrintf(D_M_RTMP_LOW, D_L_ROUTING, ("%s : Lookup for Net=%d\n",
 		 "rt_blookup", NetNumber));
 */	
-	ATDISABLE(s, ddpinp_lock);
 	while (ptree) {
 
 		if (NetNumber > ptree->NetStop) {
@@ -182,7 +179,6 @@ at_net_al NetNumber;
 			ptree = ptree->left;
 			continue;
 		   }
-		   ATENABLE(s, ddpinp_lock);
 		   
 		   /* we're in the range (either extended or not)
 		    * return the entry found.
@@ -196,7 +192,6 @@ at_net_al NetNumber;
 		   return (ptree);
 		}	
 	}
-	ATENABLE(s, ddpinp_lock);
 
 	dPrintf(D_M_RTMP_LOW, D_L_ROUTING, ("%s : %04d : NOT FOUND\n",
 		 "rt_blookup", NetNumber));
@@ -502,23 +497,19 @@ int zt_add_zonename(zname)
 at_nvestr_t *zname;
 {
 	register short res,i;
-	register unsigned int s;
 
 	if (res = zt_find_zname(zname))
 		return(res);
 
-	ATDISABLE(s, ddpinp_lock);
 	for (i = 0; i < ZT_maxentry ; i++) {
 		if (ZT_table[i].ZoneCount == 0 && ZT_table[i].Zone.len == 0) {/* free entry */
 			ZT_table[i].Zone = *zname;
 			dPrintf(D_M_RTMP, D_L_VERBOSE, ("zt_add_zonename: zone #%d %s len=%d\n",
 				i, ZT_table[i].Zone.str, ZT_table[i].Zone.len));
 			at_state.flags |= AT_ST_ZT_CHANGED;
-			ATENABLE(s, ddpinp_lock);
 			return(i+1);
 		}
 	}
-	ATENABLE(s, ddpinp_lock);
 	/* table full... */
 	return (ZT_MAXEDOUT);
 }
@@ -713,14 +704,10 @@ RT_entry *ent;
 	register u_char *zmap;
 	register u_short i,j;
 	register int	   zone_count = 0 ;
-	register unsigned int s;
 
-	ATDISABLE(s, ddpinp_lock);
 
-	if (!RT_ALL_ZONES_KNOWN(ent)) {
-			ATENABLE(s, ddpinp_lock);
+	if (!RT_ALL_ZONES_KNOWN(ent))
 			return (0);
-	}
 	zmap = ent->ZoneBitMap;
 
 	for (i = 0 ; i < ZT_BYTES ; i++) {
@@ -733,7 +720,6 @@ RT_entry *ent;
 		zmap++;
 	}
 
-	ATENABLE(s, ddpinp_lock);
 	return (zone_count);
 }
 
@@ -745,13 +731,11 @@ at_nvestr_t *zname;
 {
 	register short i, j, found;
 	register char c1, c2;
-	register unsigned int s;
 
 
 	if (!zname->len)
 		return(0);
 
-	ATDISABLE(s, ddpinp_lock);
 	for (i = 0 ; i < ZT_maxentry ; i++) {
 			if (!ZT_table[i].ZoneCount || zname->len != ZT_table[i].Zone.len)
 					continue;
@@ -775,13 +759,10 @@ at_nvestr_t *zname;
 				}
 			}
 
-			if (found) {
-				ATENABLE(s, ddpinp_lock);
+			if (found)
 				return (i+1);
-			}
 	}
 
-	ATENABLE(s, ddpinp_lock);
 	return(0);
 }
 
@@ -794,14 +775,11 @@ void zt_set_zmap(znum, zmap)
      char *zmap;
 {
 	register u_short num = znum -1;
-	register unsigned int s;
 
-	ATDISABLE(s, ddpinp_lock);
 	if (!(zmap[num >> 3] & 0x80 >> (num % 8))) {
 		zmap[num >> 3] |= 0x80 >> (num % 8);
 		ZT_table[num].ZoneCount++;
 	}
-	ATENABLE(s, ddpinp_lock);
 }
 
 
@@ -813,14 +791,11 @@ void zt_clr_zmap(znum, zmap)
      char *zmap;
 {
 	register u_short num = znum -1;
-	register unsigned int s;
 
-	ATDISABLE(s, ddpinp_lock);
 	if (zmap[num >> 3] & 0x80 >> (num % 8)) {
 		zmap[num >> 3] ^= 0x80 >> (num % 8);
 		ZT_table[num].ZoneCount--;
 	}
-	ATENABLE(s, ddpinp_lock);
 }
 
 
diff --git a/bsd/netat/ddp_sip.c b/bsd/netat/ddp_sip.c
index 07cb4a9b7..27a9e695d 100644
--- a/bsd/netat/ddp_sip.c
+++ b/bsd/netat/ddp_sip.c
@@ -120,7 +120,7 @@ void sip_input(mp, ifID)
 	/* assuming that the whole packet is in one contiguous buffer */
 	atp = (at_atp_t	*)ddp->data;
 	
-	switch(UAL_VALUE(atp->user_bytes)) {
+	switch(UAL_VALUE_NTOH(atp->user_bytes)) {
 	case SIP_SYSINFO_CMD :
 		/* Sending a response with "AppleTalk driver version" (u_short)
 		 * followed by 14 zeros will pacify the interpoll.
@@ -138,11 +138,11 @@ void sip_input(mp, ifID)
 		else
 			resp = (u_char *)gbuf_rptr(tmp);
 		bzero(resp, 16);
-		*(u_short *)resp = SIP_DRIVER_VERSION;
+		*(u_short *)resp = htons(SIP_DRIVER_VERSION);
 
 		ubytes.response = SIP_GOOD_RESPONSE;
 		ubytes.unused = 0;
-		ubytes.responder_version = SIP_RESPONDER_VERSION;
+		ubytes.responder_version = htons(SIP_RESPONDER_VERSION);
 		break;
 	case SIP_DATALINK_CMD :
 		/* In this case, the magic spell is to send 2 zeroes after
@@ -159,23 +159,23 @@ void sip_input(mp, ifID)
 		else
 			resp = (u_char *)gbuf_rptr(tmp);
 		bzero(resp, 16);
-		*(u_short *)resp = SIP_DRIVER_VERSION;
+		*(u_short *)resp = htons(SIP_DRIVER_VERSION);
 
 		ubytes.response = SIP_GOOD_RESPONSE;
 		ubytes.unused = 0;
-		ubytes.responder_version = SIP_RESPONDER_VERSION;
+		ubytes.responder_version = htons(SIP_RESPONDER_VERSION);
 		break;
 	default :
 		/* bad request, send a bad command response back */
 		ubytes.response = SIP_BAD_RESPONSE;
 		ubytes.unused = 0;
-		ubytes.responder_version = SIP_RESPONDER_VERSION;
+		ubytes.responder_version = htons(SIP_RESPONDER_VERSION);
 	}
 
 	NET_NET(ddp->dst_net, ddp->src_net);
 	ddp->dst_node = ddp->src_node;
 	ddp->dst_socket = ddp->src_socket;
-	bcopy((caddr_t) &ubytes, (caddr_t) atp->user_bytes, sizeof(ubytes));
+	UAL_ASSIGN_HTON(atp->user_bytes, &ubytes);
 	atp->cmd = ATP_CMD_TRESP;
 	atp->eom = 1;
 	atp->sts = 0;
diff --git a/bsd/netat/ddp_usrreq.c b/bsd/netat/ddp_usrreq.c
index b4862ef98..9ad2b99f2 100644
--- a/bsd/netat/ddp_usrreq.c
+++ b/bsd/netat/ddp_usrreq.c
@@ -86,7 +86,7 @@ int ddp_pru_control(struct socket *so, u_long cmd, caddr_t data,
 int	ddp_pru_attach(struct socket *so, int proto,
 		   struct proc *p)
 {
-	int s, error = 0;
+	int error = 0;
 	at_ddp_t *ddp = NULL;
 	struct atpcb *pcb = (struct atpcb *)((so)->so_pcb);
 
@@ -94,9 +94,7 @@ int	ddp_pru_attach(struct socket *so, int proto,
 	if (error != 0)
 		return error;
 
-	s = splnet();
 	error = at_pcballoc(so, &ddp_head);
-	splx(s);
 	if (error)
 		return error;
 	pcb = (struct atpcb *)((so)->so_pcb);
@@ -111,7 +109,7 @@ int	ddp_pru_attach(struct socket *so, int proto,
 int  ddp_pru_disconnect(struct socket *so)
 {
 
-	int s, error = 0;
+	int error = 0;
 	at_ddp_t *ddp = NULL;
 	struct atpcb *pcb = (struct atpcb *)((so)->so_pcb);
 
@@ -122,9 +120,7 @@ int  ddp_pru_disconnect(struct socket *so)
 		return ENOTCONN;
 
 	soisdisconnected(so);
-	s = splnet();
 	at_pcbdetach(pcb);
-	splx(s);
 
 	return error;
 }
@@ -132,31 +128,25 @@ int  ddp_pru_disconnect(struct socket *so)
 
 int  ddp_pru_abort(struct socket *so)
 {
-	int s;
 	struct atpcb *pcb = (struct atpcb *)((so)->so_pcb);
 
 	if (pcb == NULL) 
 		return (EINVAL);
 
 	soisdisconnected(so);
-	s = splnet();
 	at_pcbdetach(pcb);
-	splx(s);
 
 	return 0;
 }
 
 int  ddp_pru_detach(struct socket *so)
 {
-	int s;
 	struct atpcb *pcb = (struct atpcb *)((so)->so_pcb);
 
 	if (pcb == NULL) 
 		return (EINVAL);
 
-	s = splnet();
 	at_pcbdetach(pcb);
-	splx(s);
 	return 0;
 }
 					  
@@ -229,8 +219,8 @@ int	ddp_pru_send(struct socket *so, int flags, struct mbuf *m,
 		}
 	}
 	if (ddp) {
-		ddp->length = m->m_pkthdr.len;
-		UAS_ASSIGN(ddp->checksum, 
+		DDPLEN_ASSIGN(ddp, m->m_pkthdr.len);
+		UAS_ASSIGN_HTON(ddp->checksum, 
 			   (pcb->ddp_flags & DDPFLG_CHKSUM)? 1: 0);
 		ddp->type = (pcb->ddptype)? pcb->ddptype: DEFAULT_OT_DDPTYPE;
 #ifdef NOT_YET
@@ -259,8 +249,8 @@ int	ddp_pru_send(struct socket *so, int flags, struct mbuf *m,
 			NET_ASSIGN(ddp->src_net, ifID->ifThisNode.s_net);
 			ddp->src_node = ifID->ifThisNode.s_node;
 			ddp->src_socket = pcb->lport;
-			if (UAS_VALUE(ddp->checksum))
-				UAS_ASSIGN(ddp->checksum, ddp_checksum(m, 4));
+			if (UAS_VALUE_NTOH(ddp->checksum))
+				UAS_ASSIGN_HTON(ddp->checksum, ddp_checksum(m, 4));
 			ddp_input(n, ifID);
 		}
 	}
@@ -270,7 +260,6 @@ int	ddp_pru_send(struct socket *so, int flags, struct mbuf *m,
 int   ddp_pru_sockaddr(struct socket *so, 
 		   struct sockaddr **nam)
 {
-        int s;
 	struct atpcb *pcb;
 	struct sockaddr_at *sat;
 
@@ -279,9 +268,7 @@ int   ddp_pru_sockaddr(struct socket *so,
 		return(ENOMEM);
 	bzero((caddr_t)sat, sizeof(*sat));
 
-	s = splnet();
 	if ((pcb = sotoatpcb(so)) == NULL) {
-		splx(s);
 		FREE(sat, M_SONAME);
 		return(EINVAL);
 	}
@@ -290,7 +277,6 @@ int   ddp_pru_sockaddr(struct socket *so,
 	sat->sat_len = sizeof(*sat);
 	sat->sat_port = pcb->lport;
 	sat->sat_addr = pcb->laddr;
-	splx(s);
 
 	*nam = (struct sockaddr *)sat;
 	return(0);
@@ -300,7 +286,6 @@ int   ddp_pru_sockaddr(struct socket *so,
 int  ddp_pru_peeraddr(struct socket *so, 
 		  struct sockaddr **nam)
 {
-        int s;
 	struct atpcb *pcb;
 	struct sockaddr_at *sat;
 
@@ -309,9 +294,7 @@ int  ddp_pru_peeraddr(struct socket *so,
 		return (ENOMEM);
 	bzero((caddr_t)sat, sizeof(*sat));
 
-	s = splnet();
 	if ((pcb = sotoatpcb(so)) == NULL) {
-		splx(s);
 		FREE(sat, M_SONAME);
 		return(EINVAL);
 	}
@@ -320,7 +303,6 @@ int  ddp_pru_peeraddr(struct socket *so,
 	sat->sat_len = sizeof(*sat);
 	sat->sat_port = pcb->rport;
 	sat->sat_addr = pcb->raddr;
-	splx(s);
 
 	*nam = (struct sockaddr *)sat;
 	return(0);
diff --git a/bsd/netat/drv_dep.c b/bsd/netat/drv_dep.c
index 5fb70325a..12ac367e8 100644
--- a/bsd/netat/drv_dep.c
+++ b/bsd/netat/drv_dep.c
@@ -82,38 +82,12 @@ struct ifqueue atalkintrq; 	/* appletalk and aarp packet input queue */
 
 short appletalk_inited = 0;
 
-extern atlock_t 
-	ddpall_lock, ddpinp_lock, arpinp_lock, refall_lock, nve_lock,
-  	aspall_lock, asptmo_lock, atpall_lock, atptmo_lock, atpgen_lock;
 
-extern int (*sys_ATsocket )(), (*sys_ATgetmsg)(), (*sys_ATputmsg)();
-extern int (*sys_ATPsndreq)(), (*sys_ATPsndrsp)();
-extern int (*sys_ATPgetreq)(), (*sys_ATPgetrsp)();
 
 void atalk_load()
 {
-	extern int _ATsocket(), _ATgetmsg(), _ATputmsg();
-	extern int _ATPsndreq(), _ATPsndrsp(), _ATPgetreq(), _ATPgetrsp();
 	extern lck_mtx_t *domain_proto_mtx;
 
-	sys_ATsocket  = _ATsocket;
-	sys_ATgetmsg  = _ATgetmsg;
-	sys_ATputmsg  = _ATputmsg;
-	sys_ATPsndreq = _ATPsndreq;
-	sys_ATPsndrsp = _ATPsndrsp;
-	sys_ATPgetreq = _ATPgetreq;
-	sys_ATPgetrsp = _ATPgetrsp;
-
-	ATLOCKINIT(ddpall_lock);
-	ATLOCKINIT(ddpinp_lock);
-	ATLOCKINIT(arpinp_lock);
-	ATLOCKINIT(refall_lock);
-	ATLOCKINIT(aspall_lock);
-	ATLOCKINIT(asptmo_lock);
-	ATLOCKINIT(atpall_lock);
-	ATLOCKINIT(atptmo_lock);
-	ATLOCKINIT(atpgen_lock);
-	ATLOCKINIT(nve_lock);
 
 	atp_init();
 	atp_link();
@@ -134,14 +108,6 @@ void atalk_unload()  /* not currently used */
 	extern gbuf_t *scb_resource_m;
 	extern gbuf_t *atp_resource_m;
 
-	sys_ATsocket  = 0;
-	sys_ATgetmsg  = 0;
-	sys_ATputmsg  = 0;
-	sys_ATPsndreq = 0;
-	sys_ATPsndrsp = 0;
-	sys_ATPgetreq = 0;
-	sys_ATPgetrsp = 0;
-
 	atp_unlink();
 
 #ifdef NOT_YET
@@ -173,7 +139,7 @@ void appletalk_hack_start()
 int pat_output(patp, mlist, dst_addr, type)
 	at_ifaddr_t *patp;
 	struct mbuf *mlist;			/* packet chain */
-	unsigned char *dst_addr;
+	unsigned char *dst_addr;	/* for atalk addr - net # must be in network byte order */
 	int 	type;
 {
 	struct mbuf *m, *m1;
diff --git a/bsd/netat/nbp.h b/bsd/netat/nbp.h
index 656303e17..9623dc513 100644
--- a/bsd/netat/nbp.h
+++ b/bsd/netat/nbp.h
@@ -97,8 +97,16 @@
 #define	NBP_HDR_SIZE	2
 
 typedef struct at_nbp {
-        unsigned      	control : 4,
-        	      	tuple_count : 4;
+#if BYTE_ORDER == BIG_ENDIAN
+        unsigned      	
+        	control : 4,
+        	tuple_count : 4;
+#endif
+#if BYTE_ORDER == LITTLE_ENDIAN
+		unsigned
+			tuple_count : 4,
+			control : 4;
+#endif
 	u_char		at_nbp_id;
 	at_nbptuple_t	tuple[NBP_TUPLE_MAX];
 } at_nbp_t;
diff --git a/bsd/netat/sys_dep.c b/bsd/netat/sys_dep.c
index 4f63e7351..ece349050 100644
--- a/bsd/netat/sys_dep.c
+++ b/bsd/netat/sys_dep.c
@@ -57,13 +57,6 @@
 #include <netat/at_pcb.h>
 #include <netat/debug.h>
 
-int (*sys_ATsocket)() = 0;
-int (*sys_ATgetmsg)() = 0;
-int (*sys_ATputmsg)() = 0;
-int (*sys_ATPsndreq)() = 0;
-int (*sys_ATPsndrsp)() = 0;
-int (*sys_ATPgetreq)() = 0;
-int (*sys_ATPgetrsp)() = 0;
 
 extern at_state_t at_state;	/* global state of AT network */
 extern at_ifaddr_t *ifID_home;	/* default interface */
@@ -77,6 +70,12 @@ extern lck_mtx_t * atalk_mutex;
 #define f_offset f_fglob->fg_offset
 #define f_data f_fglob->fg_data
 
+extern int _ATsocket(int, int *, void *);
+extern int _ATgetmsg(int, strbuf_t *, strbuf_t *, int *, int *, void *);
+extern int _ATputmsg();
+extern int _ATPsndreq(), _ATPsndrsp(), _ATPgetreq(), _ATPgetrsp();
+
+
 int ATsocket(proc, uap, retval)
 	struct proc *proc;
 	struct ATsocket_args *uap;
@@ -84,13 +83,13 @@ int ATsocket(proc, uap, retval)
 {
 	int err;
 	atalk_lock();
-	if (sys_ATsocket) {
+	if (_ATsocket) {
 		/* required check for all AppleTalk system calls */
 		if (!(at_state.flags & AT_ST_STARTED) || !ifID_home) {
 			*retval = -1;
 			err = ENOTREADY;
 		} else {
-			*retval = (*sys_ATsocket)(uap->proto, &err, proc);
+			*retval = _ATsocket((int)uap->proto, (int *)&err, (void *)proc);
 		}
 	} else {
 		*retval = -1;
@@ -108,14 +107,14 @@ int ATgetmsg(proc, uap, retval)
 	int err;
 
 	atalk_lock();
-	if (sys_ATgetmsg) {
+	if (_ATgetmsg) {
 		/* required check for all AppleTalk system calls */
 		if (!(at_state.flags & AT_ST_STARTED) || !ifID_home) {
 			*retval = -1;
 			err = ENOTREADY;
 		} else {
 			*retval = 
-			  (*sys_ATgetmsg)(uap->fd, uap->ctlptr, uap->datptr, 
+			  (*_ATgetmsg)(uap->fd, uap->ctlptr, uap->datptr, 
 					  uap->flags, &err, proc);
 		}
 	} else {
@@ -134,14 +133,14 @@ int ATputmsg(proc, uap, retval)
 	int err;
 
 	atalk_lock();
-	if (sys_ATputmsg) {
+	if (_ATputmsg) {
 		/* required check for all AppleTalk system calls */
 		if (!(at_state.flags & AT_ST_STARTED) || !ifID_home) {
 			*retval = -1;
 			err = ENOTREADY;
 		} else {
 			*retval = 
-			  (*sys_ATputmsg)(uap->fd, uap->ctlptr, uap->datptr, 
+			  _ATputmsg(uap->fd, uap->ctlptr, uap->datptr, 
 					  uap->flags, &err, proc);
 		}
 	} else {
@@ -160,14 +159,14 @@ int ATPsndreq(proc, uap, retval)
 	int err;
 
 	atalk_lock();
-	if (sys_ATPsndreq) {
+	if (_ATPsndreq) {
 		/* required check for all AppleTalk system calls */
 		if (!(at_state.flags & AT_ST_STARTED) || !ifID_home) {
 			*retval = -1;
 			err = ENOTREADY;
 		} else {
 			*retval = 
-			  (*sys_ATPsndreq)(uap->fd, uap->buf, uap->len, 
+			  _ATPsndreq(uap->fd, uap->buf, uap->len, 
 					   uap->nowait, &err, proc);
 		}
 	} else {
@@ -186,14 +185,14 @@ int ATPsndrsp(proc, uap, retval)
 	int err;
 
 	atalk_lock();
-	if (sys_ATPsndrsp) {
+	if (_ATPsndrsp) {
 		/* required check for all AppleTalk system calls */
 		if (!(at_state.flags & AT_ST_STARTED) || !ifID_home) {
 			*retval = -1;
 			err = ENOTREADY;
 		} else { 
 			*retval = 
-			  (*sys_ATPsndrsp)(uap->fd, uap->respbuff, 
+			  _ATPsndrsp(uap->fd, uap->respbuff, 
 					   uap->resplen, uap->datalen, &err, proc);
 		}
 	} else {
@@ -212,14 +211,14 @@ int ATPgetreq(proc, uap, retval)
 	int err;
 
 	atalk_lock();
-	if (sys_ATPgetreq) {
+	if (_ATPgetreq) {
 		/* required check for all AppleTalk system calls */
 		if (!(at_state.flags & AT_ST_STARTED) || !ifID_home) {
 			*retval = -1;
 			err = ENOTREADY;
 		} else {
 			*retval = 
-			  (*sys_ATPgetreq)(uap->fd, uap->buf, uap->buflen, 
+			  _ATPgetreq(uap->fd, uap->buf, uap->buflen, 
 					   &err, proc);
 		}
 	} else {
@@ -238,14 +237,14 @@ int ATPgetrsp(proc, uap, retval)
 	int err = 0;
 
 	atalk_lock();
-	if (sys_ATPgetrsp) {
+	if (_ATPgetrsp) {
 		/* required check for all AppleTalk system calls */
 		if (!(at_state.flags & AT_ST_STARTED) || !ifID_home) {
 			*retval = -1;
 			err = ENOTREADY;
 		} else {
 			*retval = 
-			  (*sys_ATPgetrsp)(uap->fd, uap->bdsp, &err, proc);
+			  _ATPgetrsp(uap->fd, uap->bdsp, &err, proc);
 		}
 	} else {
 		*retval = -1;
diff --git a/bsd/netat/sys_glue.c b/bsd/netat/sys_glue.c
index edb15b3fe..597835330 100644
--- a/bsd/netat/sys_glue.c
+++ b/bsd/netat/sys_glue.c
@@ -97,7 +97,6 @@ SYSCTL_STRUCT(_net_appletalk, OID_AUTO, ddpstats, CTLFLAG_RD,
 static void ioccmd_t_32_to_64( ioccmd_t *from_p, user_ioccmd_t *to_p );
 static void ioccmd_t_64_to_32( user_ioccmd_t *from_p, ioccmd_t *to_p );
 
-atlock_t refall_lock;
 
 caddr_t	atp_free_cluster_list = 0;
 
@@ -294,7 +293,7 @@ int _ATrw(fp, rw, uio, p)
      struct uio *uio;
      struct proc *p;
 {
-    int s, err, len, clen = 0, res;
+    int err, len, clen = 0, res;
     gref_t *gref;
     gbuf_t *m, *mhead, *mprev;
 
@@ -306,7 +305,6 @@ int _ATrw(fp, rw, uio, p)
     if ((len = uio_resid(uio)) == 0)
     	return 0;
 
-    ATDISABLE(s, gref->lock);
 
     if (rw == UIO_READ) {
 	KERNEL_DEBUG(DBG_ADSP_ATRW, 0, gref, len, gref->rdhead, 0);
@@ -314,23 +312,18 @@ int _ATrw(fp, rw, uio, p)
 		gref->sevents |= POLLMSG;
 		err = msleep(&gref->event, atalk_mutex, PSOCK | PCATCH, "AT read", 0);
 		gref->sevents &= ~POLLMSG;
-		if (err != 0) {
-			ATENABLE(s, gref->lock);
+		if (err != 0)
 			return err;
-		}
 		KERNEL_DEBUG(DBG_ADSP_ATRW, 1, gref, gref->rdhead, mhead, gbuf_next(mhead));
 	}
 
-	if (gref->errno) {
-		ATENABLE(s, gref->lock);
+	if (gref->errno)
 		return EPIPE;
-	}
 	if ((gref->rdhead = gbuf_next(mhead)) == 0)
 		gref->rdtail = 0;
 
 	KERNEL_DEBUG(DBG_ADSP_ATRW, 2, gref, gref->rdhead, mhead, gbuf_next(mhead));
 
-	ATENABLE(s, gref->lock);
 
 //##### LD TEST 08/05
 //	simple_lock(&gref->lock);
@@ -362,12 +355,10 @@ int _ATrw(fp, rw, uio, p)
 			gbuf_cont(mprev) = 0;
 		else
 			mhead = 0;
-		ATDISABLE(s, gref->lock);
 		if (gref->rdhead == 0)
 			gref->rdtail = m;
 		gbuf_next(m) = gref->rdhead;
 		gref->rdhead = m;
-		ATENABLE(s, gref->lock);
 	}
 	if (mhead)
 		gbuf_freem(mhead);
@@ -380,14 +371,11 @@ int _ATrw(fp, rw, uio, p)
 			gref->sevents |= POLLSYNC;
 			err = msleep(&gref->event, atalk_mutex, PSOCK | PCATCH, "AT write", 0);
 			gref->sevents &= ~POLLSYNC;
-			if (err != 0) {
-				ATENABLE(s, gref->lock);
+			if (err != 0)
 				return err;
-			}
 		}
 	}
 
-	ATENABLE(s, gref->lock);
 
 	/* allocate a buffer to copy in the write data */
 	if ((m = gbuf_alloc(AT_WR_OFFSET+len, PRI_MED)) == 0)
@@ -448,7 +436,7 @@ int _ATwrite(fp, uio, cred, flags, p)
 /* bms:  update to be callable from kernel */
 int at_ioctl(gref_t *gref, u_long cmd, caddr_t arg, int fromKernel)
 {
-    int s, err = 0, len;
+    int err = 0, len;
     u_int size;
     gbuf_t *m, *mdata;
     ioc_t *ioc;
@@ -528,7 +516,6 @@ int at_ioctl(gref_t *gref, u_long cmd, caddr_t arg, int fromKernel)
 	gref_wput(gref, m);
 
     /* wait for the ioc ack */
-    ATDISABLE(s, gref->lock);
     while ((m = gref->ichead) == 0) {
         gref->sevents |= POLLPRI;
 #ifdef APPLETALK_DEBUG
@@ -537,7 +524,6 @@ int at_ioctl(gref_t *gref, u_long cmd, caddr_t arg, int fromKernel)
 		err = msleep(&gref->iocevent, atalk_mutex, PSOCK | PCATCH, "AT ioctl", 0);
 		gref->sevents &= ~POLLPRI;
 		if (err != 0) {
-			ATENABLE(s, gref->lock);
 #ifdef APPLETALK_DEBUG
 			kprintf("at_ioctl: EINTR\n");
 #endif
@@ -551,7 +537,6 @@ int at_ioctl(gref_t *gref, u_long cmd, caddr_t arg, int fromKernel)
 
 	gref->ichead = gbuf_next(m);
 
-	ATENABLE(s, gref->lock);
 
 #ifdef APPLETALK_DEBUG
 	kprintf("at_ioctl: woke up from ioc sleep gref = 0x%x\n", 
@@ -638,7 +623,7 @@ int _ATselect(fp, which, wql, proc)
 	void * wql;
 	struct proc *proc;
 {
-	int s, err, rc = 0;
+	int err, rc = 0;
 	gref_t *gref;
 
 	/* Radar 4128949: Drop the proc_fd lock here to avoid lock inversion issues with the other AT calls
@@ -656,7 +641,6 @@ int _ATselect(fp, which, wql, proc)
 	if (err != 0)
 		rc = 1;
 	else {
-	     ATDISABLE(s, gref->lock);
 	     if (which == FREAD) {
 		  if (gref->rdhead || (gref->readable && (*gref->readable)(gref)))
 		       rc = 1;
@@ -676,7 +660,6 @@ int _ATselect(fp, which, wql, proc)
 		  } else
 		       rc = 1;
 	     }
-	     ATENABLE(s, gref->lock);
 	}
 
 	return rc;
@@ -694,9 +677,7 @@ void atalk_putnext(gref, m)
 	gref_t *gref;
 	gbuf_t *m;
 {
-	int s;
 
-	ATDISABLE(s, gref->lock);
 
 	/* *** potential leak? *** */
 	gbuf_next(m) = 0;
@@ -741,7 +722,6 @@ void atalk_putnext(gref, m)
 		    }
 	} /* switch gbuf_type(m) */
 
-	ATENABLE(s, gref->lock);
 } /* atalk_putnext */
 
 void atalk_enablew(gref)
@@ -754,9 +734,7 @@ void atalk_enablew(gref)
 void atalk_flush(gref)
 	gref_t *gref;
 {
-	int s;
 
-	ATDISABLE(s, gref->lock);
 	if (gref->rdhead) {
 		gbuf_freel(gref->rdhead);
 		gref->rdhead = 0;
@@ -765,7 +743,6 @@ void atalk_flush(gref)
 		gbuf_freel(gref->ichead);
 		gref->ichead = 0;
 	}
-	ATENABLE(s, gref->lock);
 }
 
 /*
@@ -776,8 +753,6 @@ void atalk_notify(gref, errno)
 	register gref_t *gref;
 	int errno;
 {
-	int s;
-	ATDISABLE(s, gref->lock);
 
 	if (gref->atpcb_socket) {
 	    /* For DDP --
@@ -808,35 +783,29 @@ void atalk_notify(gref, errno)
 		}
 	    }
 	}
-	ATENABLE(s, gref->lock);
 } /* atalk_notify */
 
 void atalk_notify_sel(gref)
 	gref_t *gref;
 {
-	int s;
 
-	ATDISABLE(s, gref->lock);
 	if (gref->sevents & POLLIN) {
 		gref->sevents &= ~POLLIN;
 		selwakeup(&gref->si);
 	}
-	ATENABLE(s, gref->lock);
 }
 
 int atalk_peek(gref, event)
 	gref_t *gref;
 	unsigned char *event;
 {
-	int s, rc;
+	int rc;
 
-	ATDISABLE(s, gref->lock);
 	if (gref->rdhead) {
 		*event = *gbuf_rptr(gref->rdhead);
 		rc = 0;
 	} else
 		rc = -1;
-	ATENABLE(s, gref->lock);
 
 	return rc;
 }
@@ -882,15 +851,13 @@ int gref_alloc(grefp)
 	gref_t **grefp;
 {
 	extern gbuf_t *atp_resource_m;
-	int i, s;
+	int i;
 	gbuf_t *m;
 	gref_t *gref, *gref_array;
 
 	*grefp = (gref_t *)NULL;
 
-	ATDISABLE(s, refall_lock);
 	if (gref_free_list == 0) {
-		ATENABLE(s, refall_lock);
 #ifdef APPLETALK_DEBUG
 		kprintf("gref_alloc: gbufalloc size=%d\n", GREF_PER_BLK*sizeof(gref_t));
 #endif
@@ -900,7 +867,6 @@ int gref_alloc(grefp)
 		gref_array = (gref_t *)gbuf_rptr(m);
 		for (i=0; i < GREF_PER_BLK-1; i++)
 			gref_array[i].atpcb_next = (gref_t *)&gref_array[i+1];
-		ATDISABLE(s, refall_lock);
 		gbuf_cont(m) = atp_resource_m;
 		atp_resource_m = m;
 		gref_array[i].atpcb_next = gref_free_list;
@@ -909,10 +875,6 @@ int gref_alloc(grefp)
 
 	gref = gref_free_list;
 	gref_free_list = gref->atpcb_next;
-	ATENABLE(s, refall_lock);
-	ATLOCKINIT(gref->lock);
-//### LD Test 08/05/98
-//	simple_lock_init(&gref->lock);
 	ATEVENTINIT(gref->event);
 	ATEVENTINIT(gref->iocevent);
 
@@ -926,7 +888,7 @@ int gref_alloc(grefp)
 /* bms:  make gref_close callable from kernel */
 int gref_close(gref_t *gref)
 {
-	int s, rc;
+	int rc;
 
 	switch (gref->proto) {
 
@@ -954,11 +916,9 @@ int gref_close(gref_t *gref)
 		selthreadclear(&gref->si);
 
 		/* from original gref_free() */
-		ATDISABLE(s, refall_lock);
 		bzero((char *)gref, sizeof(gref_t));
 		gref->atpcb_next = gref_free_list;
 		gref_free_list = gref;
-		ATENABLE(s, refall_lock);
 	}
 
 	return rc;
@@ -995,7 +955,7 @@ struct mbuf *m_clattach(extbuf, extfree, extsize, extarg, wait)
         m->m_ext.ext_size = extsize;
         m->m_ext.ext_arg = extarg;
         m->m_ext.ext_refs.forward = 
-	  m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
+	  	m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
         m->m_data = extbuf;
         m->m_flags |= M_EXT;
 
diff --git a/bsd/netat/sysglue.h b/bsd/netat/sysglue.h
index 2c84a83f4..579f73702 100644
--- a/bsd/netat/sysglue.h
+++ b/bsd/netat/sysglue.h
@@ -64,21 +64,13 @@ typedef struct {
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with ioccmd_t
  */
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 typedef struct {
 	int  		ic_cmd;
 	int  		ic_timout;
 	int  		ic_len;
-	user_addr_t	ic_dp;
+	user_addr_t	ic_dp __attribute__((aligned(8)));
 } user_ioccmd_t;
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif // KERNEL_PRIVATE
 #endif // KERNEL
 
@@ -164,10 +156,6 @@ typedef struct {
 typedef int atevent_t;
 
 typedef int atlock_t;
-typedef int *atomic_p; 
-#define ATLOCKINIT(a)  (a = (atlock_t) EVENT_NULL)
-#define ATDISABLE(l, a)
-#define ATENABLE(l, a)
 #define ATEVENTINIT(a)  (a = (atevent_t) EVENT_NULL)
 #define DDP_OUTPUT(m) ddp_putmsg(0,m)
 #define StaticProc static
diff --git a/bsd/netinet/icmp_var.h b/bsd/netinet/icmp_var.h
index 274778ca4..5249697e6 100644
--- a/bsd/netinet/icmp_var.h
+++ b/bsd/netinet/icmp_var.h
@@ -116,4 +116,4 @@ extern int badport_bandlim(int);
 #define BANDLIM_RST_OPENPORT 4   /* No connection, listener */
 #define BANDLIM_MAX 4
 #endif KERNEL_PRIVATE
-#endif _NETINET_ICMP_VAR_H_
+#endif /* _NETINET_ICMP_VAR_H_ */
diff --git a/bsd/netinet/if_ether.h b/bsd/netinet/if_ether.h
index 188398729..d0bb1057d 100644
--- a/bsd/netinet/if_ether.h
+++ b/bsd/netinet/if_ether.h
@@ -150,4 +150,4 @@ int	arpresolve(struct ifnet *, struct rtentry *, struct mbuf *,
 void	arp_ifinit(struct ifnet *, struct ifaddr *);
 #endif KERNEL_PRIVATE
 
-#endif _NETINET_IF_ETHER_H_
+#endif /* _NETINET_IF_ETHER_H_ */
diff --git a/bsd/netinet/in.c b/bsd/netinet/in.c
index 255978981..6686c77ee 100644
--- a/bsd/netinet/in.c
+++ b/bsd/netinet/in.c
@@ -714,7 +714,7 @@ in_control(
         */
 		if (ifp->if_flags & IFF_MULTICAST) {
 			struct in_addr addr;
-			struct in_multi *inm;
+			struct in_multi *inm = NULL;
 
 			TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) 
 				if (ifa->ifa_addr->sa_family == AF_INET)
diff --git a/bsd/netinet/in.h b/bsd/netinet/in.h
index b6bef2588..8e71c72f8 100644
--- a/bsd/netinet/in.h
+++ b/bsd/netinet/in.h
@@ -601,4 +601,4 @@ u_long	in_netof(struct in_addr);
 const char	*inet_ntop(int, const void *, char *, size_t); /* in libkern */
 #endif /* KERNEL */
 
-#endif _NETINET_IN_H_
+#endif /* _NETINET_IN_H_ */
diff --git a/bsd/netinet/in_arp.c b/bsd/netinet/in_arp.c
index bcfcfbf59..ed713b2b0 100644
--- a/bsd/netinet/in_arp.c
+++ b/bsd/netinet/in_arp.c
@@ -168,11 +168,11 @@ static void
 arptimer(
 	__unused void *ignored_arg)
 {
-	struct llinfo_arp *la = llinfo_arp.lh_first;
-	struct llinfo_arp *ola;
+	struct llinfo_arp *la, *ola;
 	struct timeval timenow;
 
 	lck_mtx_lock(rt_mtx);
+	la = llinfo_arp.lh_first;
 	getmicrotime(&timenow);
 	while ((ola = la) != 0) {
 		struct rtentry *rt = la->la_rt;
diff --git a/bsd/netinet/ip.h b/bsd/netinet/ip.h
index 0bcda2140..cc5131a0a 100644
--- a/bsd/netinet/ip.h
+++ b/bsd/netinet/ip.h
@@ -64,7 +64,7 @@
 #ifndef _NETINET_IP_H_
 #define _NETINET_IP_H_
 #include <sys/appleapiopts.h>
-#include <sys/types.h>>		/* XXX temporary hack to get u_ types */
+#include <sys/types.h>		/* XXX temporary hack to get u_ types */
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
 
diff --git a/bsd/netinet/ip_divert.c b/bsd/netinet/ip_divert.c
index 8e1e7f3df..50fb1fd12 100644
--- a/bsd/netinet/ip_divert.c
+++ b/bsd/netinet/ip_divert.c
@@ -170,7 +170,6 @@ div_init(void)
 	 * allocate the lock attribute for divert pcb mutexes
 	 */
 	pcbinfo->mtx_attr = lck_attr_alloc_init();
-	lck_attr_setdefault(pcbinfo->mtx_attr);
 
 	if ((pcbinfo->mtx = lck_rw_alloc_init(pcbinfo->mtx_grp, pcbinfo->mtx_attr)) == NULL)
 		return;	/* pretty much dead if this fails... */
@@ -640,12 +639,9 @@ __private_extern__ int
 div_lock(struct socket *so, int refcount, int lr)
  {
 	int lr_saved;
-#ifdef __ppc__
-	if (lr == 0) {
-			__asm__ volatile("mflr %0" : "=r" (lr_saved));
-	}
+	if (lr == 0) 
+		lr_saved = (unsigned int) __builtin_return_address(0);
 	else lr_saved = lr;
-#endif
 	
 #ifdef MORE_DICVLOCK_DEBUG
 	printf("div_lock: so=%x sopcb=%x lock=%x ref=%x lr=%x\n",
@@ -668,7 +664,8 @@ div_lock(struct socket *so, int refcount, int lr)
 	
 	if (refcount)
 		so->so_usecount++;
-	so->reserved3 = (void *)lr_saved;
+	so->lock_lr[so->next_lock_lr] = (u_int32_t *)lr_saved;
+	so->next_lock_lr = (so->next_lock_lr+1) % SO_LCKDBG_MAX;
 
 	return (0);
 }
@@ -679,12 +676,11 @@ div_unlock(struct socket *so, int refcount, int lr)
 	int lr_saved;
 	lck_mtx_t * mutex_held;
 	struct inpcb *inp = sotoinpcb(so);	
-#ifdef __ppc__
-	if (lr == 0) {
-		__asm__ volatile("mflr %0" : "=r" (lr_saved));
-	}
+
+	if (lr == 0) 
+		lr_saved = (unsigned int) __builtin_return_address(0);
 	else lr_saved = lr;
-#endif
+
 	
 #ifdef MORE_DICVLOCK_DEBUG
 	printf("div_unlock: so=%x sopcb=%x lock=%x ref=%x lr=%x\n",
@@ -713,8 +709,9 @@ div_unlock(struct socket *so, int refcount, int lr)
 		return (0);
 	}
 	lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
+	so->unlock_lr[so->next_unlock_lr] = (u_int *)lr_saved;
+	so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX;
 	lck_mtx_unlock(mutex_held);
-	so->reserved4 = (void *)lr_saved;
 	return (0);
 }
 
diff --git a/bsd/netinet/ip_dummynet.c b/bsd/netinet/ip_dummynet.c
index 1d1921608..6b95dd5af 100644
--- a/bsd/netinet/ip_dummynet.c
+++ b/bsd/netinet/ip_dummynet.c
@@ -111,6 +111,10 @@
  */
 static dn_key curr_time = 0 ; /* current simulation time */
 
+/* this is for the timer that fires to call dummynet() - we only enable the timer when
+	there are packets to process, otherwise it's disabled */
+static int timer_enabled = 0;	
+
 static int dn_hash_size = 64 ;	/* default hash size */
 
 /* statistics on number of queue searches and search steps */
@@ -475,9 +479,6 @@ transmit_event(struct dn_pipe *pipe)
 				break ;
 			}
 			case DN_TO_IP_IN :
-				ip = mtod(m, struct ip *);
-				ip->ip_len = htons(ip->ip_len);
-				ip->ip_off = htons(ip->ip_off);
 				proto_inject(PF_INET, m);
 				break ;
 		
@@ -525,8 +526,15 @@ transmit_event(struct dn_pipe *pipe)
  * before being able to transmit a packet. The credit is taken from
  * either a pipe (WF2Q) or a flow_queue (per-flow queueing)
  */
+ 
+/* hz is 100, which gives a granularity of 10ms in the old timer. 
+ * The timer has been changed to fire every 1ms, so the use of
+ * hz has been modified here. All instances of hz have been left
+ * in place but adjusted by a factor of 10 so that hz is functionally 
+ * equal to 1000.
+ */
 #define SET_TICKS(_m, q, p)	\
-    ((_m)->m_pkthdr.len*8*hz - (q)->numbytes + p->bandwidth - 1 ) / \
+    ((_m)->m_pkthdr.len*8*(hz*10) - (q)->numbytes + p->bandwidth - 1 ) / \
 	    p->bandwidth ;
 
 /*
@@ -586,7 +594,7 @@ ready_event(struct dn_flow_queue *q)
     q->numbytes += ( curr_time - q->sched_time ) * p->bandwidth;
     while ( (pkt = q->head) != NULL ) {
 	int len = pkt->m_pkthdr.len;
-	int len_scaled = p->bandwidth ? len*8*hz : 0 ;
+	int len_scaled = p->bandwidth ? len*8*(hz*10) : 0 ;
 	if (len_scaled > q->numbytes )
 	    break ;
 	q->numbytes -= len_scaled ;
@@ -656,7 +664,7 @@ ready_event_wfq(struct dn_pipe *p)
 	    struct mbuf *pkt = q->head;
 	    struct dn_flow_set *fs = q->fs;
 	    u_int64_t len = pkt->m_pkthdr.len;
-	    int len_scaled = p->bandwidth ? len*8*hz : 0 ;
+	    int len_scaled = p->bandwidth ? len*8*(hz*10) : 0 ;
 
 	    heap_extract(sch, NULL); /* remove queue from heap */
 	    p->numbytes -= len_scaled ;
@@ -743,7 +751,7 @@ ready_event_wfq(struct dn_pipe *p)
 }
 
 /*
- * This is called once per tick, or HZ times per second. It is used to
+ * This is called every 1ms. It is used to
  * increment the current tick counter and schedule expired events.
  */
 static void
@@ -754,6 +762,8 @@ dummynet(void * __unused unused)
     struct dn_heap *heaps[3];
     int i;
     struct dn_pipe *pe ;
+    struct timespec ts;
+    struct timeval	tv;
 
     heaps[0] = &ready_heap ;		/* fixed-rate queues */
     heaps[1] = &wfq_ready_heap ;	/* wfq queues */
@@ -761,25 +771,31 @@ dummynet(void * __unused unused)
     
 	lck_mtx_lock(dn_mutex);
 	
-    curr_time++ ;
+        /* make all time measurements in milliseconds (ms) -  
+         * here we convert secs and usecs to msecs (just divide the 
+	 * usecs and take the closest whole number).
+         */
+        microuptime(&tv);
+        curr_time = (tv.tv_sec * 1000) + (tv.tv_usec / 1000);
+	
     for (i=0; i < 3 ; i++) {
 	h = heaps[i];
 	while (h->elements > 0 && DN_KEY_LEQ(h->p[0].key, curr_time) ) {
-	    if (h->p[0].key > curr_time)
+		if (h->p[0].key > curr_time)
 		printf("dummynet: warning, heap %d is %d ticks late\n",
-		    i, (int)(curr_time - h->p[0].key));
-	    p = h->p[0].object ; /* store a copy before heap_extract */
-	    heap_extract(h, NULL); /* need to extract before processing */
-	    if (i == 0)
+			i, (int)(curr_time - h->p[0].key));
+		p = h->p[0].object ; /* store a copy before heap_extract */
+		heap_extract(h, NULL); /* need to extract before processing */
+		if (i == 0)
 		ready_event(p) ;
-	    else if (i == 1) {
+		else if (i == 1) {
 		struct dn_pipe *pipe = p;
 		if (pipe->if_name[0] != '\0')
-		    printf("dummynet: bad ready_event_wfq for pipe %s\n",
+			printf("dummynet: bad ready_event_wfq for pipe %s\n",
 			pipe->if_name);
 		else
-		    ready_event_wfq(p) ;
-	    } else
+			ready_event_wfq(p) ;
+		} else
 		transmit_event(p);
 	}
     }
@@ -794,9 +810,22 @@ dummynet(void * __unused unused)
 	    pe->sum -= q->fs->weight ;
 	}
 	
-	lck_mtx_unlock(dn_mutex);
+	/* check the heaps to see if there's still stuff in there, and 
+	 * only set the timer if there are packets to process 
+	 */
+	timer_enabled = 0;
+	for (i=0; i < 3 ; i++) {
+		h = heaps[i];
+		if (h->elements > 0) { // set the timer
+			ts.tv_sec = 0;
+			ts.tv_nsec = 1 * 1000000;	// 1ms
+			timer_enabled = 1;
+			bsd_timeout(dummynet, NULL, &ts);
+			break;
+		}
+	}
 	
-    timeout(dummynet, NULL, 1);
+    lck_mtx_unlock(dn_mutex);
 }
  
 /*
@@ -1127,6 +1156,8 @@ dummynet_io(struct mbuf *m, int pipe_nr, int dir, struct ip_fw_args *fwa)
     u_int64_t len = m->m_pkthdr.len ;
     struct dn_flow_queue *q = NULL ;
     int is_pipe;
+    struct timespec ts;
+    struct timeval	tv;
     
 #if IPFW2
     ipfw_insn *cmd = fwa->rule->cmd + fwa->rule->act_ofs;
@@ -1141,6 +1172,13 @@ dummynet_io(struct mbuf *m, int pipe_nr, int dir, struct ip_fw_args *fwa)
     pipe_nr &= 0xffff ;
 
  	lck_mtx_lock(dn_mutex);
+
+	/* make all time measurements in milliseconds (ms) - 
+         * here we convert secs and usecs to msecs (just divide the 
+         * usecs and take the closest whole number).
+	 */
+        microuptime(&tv);
+	curr_time = (tv.tv_sec * 1000) + (tv.tv_usec / 1000);
 	
    /*
      * This is a dummynet rule, so we expect an O_PIPE or O_QUEUE rule.
@@ -1293,6 +1331,14 @@ dummynet_io(struct mbuf *m, int pipe_nr, int dir, struct ip_fw_args *fwa)
 	}
     }
 done:
+	/* start the timer and set global if not already set */
+	if (!timer_enabled) {
+		ts.tv_sec = 0;
+		ts.tv_nsec = 1 * 1000000;	// 1ms
+		timer_enabled = 1;
+		bsd_timeout(dummynet, NULL, &ts);
+    }
+
 	lck_mtx_unlock(dn_mutex);
     return 0;
 
@@ -1592,7 +1638,7 @@ config_pipe(struct dn_pipe *p)
      * delay = ms, must be translated into ticks.
      * qsize = slots/bytes
      */
-    p->delay = ( p->delay * hz ) / 1000 ;
+    p->delay = ( p->delay * (hz*10) ) / 1000 ;
     /* We need either a pipe number or a flow_set number */
     if (p->pipe_nr == 0 && pfs->fs_nr == 0)
 	return EINVAL ;
@@ -1944,7 +1990,7 @@ dummynet_get(struct sockopt *sopt)
 	 * After each flow_set, copy the queue descriptor it owns.
 	 */
 	bcopy(p, bp, sizeof( *p ) );
-	pipe_bp->delay = (pipe_bp->delay * 1000) / hz ;
+	pipe_bp->delay = (pipe_bp->delay * 1000) / (hz*10) ; 
 	/*
 	 * XXX the following is a hack based on ->next being the
 	 * first field in dn_pipe and dn_flow_set. The correct
@@ -2031,7 +2077,6 @@ ip_dn_init(void)
 	dn_mutex_grp_attr = lck_grp_attr_alloc_init();
 	dn_mutex_grp = lck_grp_alloc_init("dn", dn_mutex_grp_attr);
 	dn_mutex_attr = lck_attr_alloc_init();
-	lck_attr_setdefault(dn_mutex_attr);
 
 	if ((dn_mutex = lck_mtx_alloc_init(dn_mutex_grp, dn_mutex_attr)) == NULL) {
 		printf("ip_dn_init: can't alloc dn_mutex\n");
@@ -2051,6 +2096,4 @@ ip_dn_init(void)
     ip_dn_ctl_ptr = ip_dn_ctl;
     ip_dn_io_ptr = dummynet_io;
     ip_dn_ruledel_ptr = dn_rule_delete;
-    
-    timeout(dummynet, NULL, 1);
 }
diff --git a/bsd/netinet/ip_fw.h b/bsd/netinet/ip_fw.h
index e1cafd4cf..10e65293a 100644
--- a/bsd/netinet/ip_fw.h
+++ b/bsd/netinet/ip_fw.h
@@ -322,4 +322,4 @@ extern struct ipfw_flow_id last_pkt ;
 #endif KERNEL_PRIVATE
 
 #endif !IPFW2
-#endif _IP_FW_H
+#endif /* _IP_FW_H */
diff --git a/bsd/netinet/ip_fw2.c b/bsd/netinet/ip_fw2.c
index 9801fb7d3..84f8f0c1a 100644
--- a/bsd/netinet/ip_fw2.c
+++ b/bsd/netinet/ip_fw2.c
@@ -2942,8 +2942,10 @@ ipfw_ctl(struct sockopt *sopt)
 			int	i, len = 0;
 			struct ip_old_fw	*buf2, *rule_vers0;
 			
+			lck_mtx_lock(ipfw_mutex);
 			buf2 = _MALLOC(static_count * sizeof(struct ip_old_fw), M_TEMP, M_WAITOK);
 			if (buf2 == 0) {
+				lck_mtx_unlock(ipfw_mutex);
 				error = ENOBUFS;
 			}
 			
@@ -2959,6 +2961,7 @@ ipfw_ctl(struct sockopt *sopt)
 					len += sizeof(*rule_vers0);
 					rule_vers0++;
 				}
+				lck_mtx_unlock(ipfw_mutex);
 				error = sooptcopyout(sopt, buf2, len);
 				_FREE(buf2, M_TEMP);
 			}
@@ -2968,11 +2971,13 @@ ipfw_ctl(struct sockopt *sopt)
 			struct ipfw_dyn_rule_compat	*dyn_rule_vers1, *dyn_last = NULL;
 			ipfw_dyn_rule 	*p;
 
+			lck_mtx_lock(ipfw_mutex);
 			buf_size = static_count * sizeof(struct ip_fw_compat) +
 						dyn_count * sizeof(struct ipfw_dyn_rule_compat);
 						
 			buf2 = _MALLOC(buf_size, M_TEMP, M_WAITOK);
 			if (buf2 == 0) {
+				lck_mtx_unlock(ipfw_mutex);
 				error = ENOBUFS;
 			}
 			
@@ -3017,6 +3022,7 @@ ipfw_ctl(struct sockopt *sopt)
 						dyn_last->next = NULL;
 					}
 				}
+				lck_mtx_unlock(ipfw_mutex);
 				
 				error = sooptcopyout(sopt, buf2, len);
 				_FREE(buf2, M_TEMP);
@@ -3263,7 +3269,6 @@ ipfw_init(void)
 	ipfw_mutex_grp_attr = lck_grp_attr_alloc_init();
 	ipfw_mutex_grp = lck_grp_alloc_init("ipfw", ipfw_mutex_grp_attr);
 	ipfw_mutex_attr = lck_attr_alloc_init();
-	lck_attr_setdefault(ipfw_mutex_attr);
 
 	if ((ipfw_mutex = lck_mtx_alloc_init(ipfw_mutex_grp, ipfw_mutex_attr)) == NULL) {
 		printf("ipfw_init: can't alloc ipfw_mutex\n");
diff --git a/bsd/netinet/ip_input.c b/bsd/netinet/ip_input.c
index d84dac759..b71b165d4 100644
--- a/bsd/netinet/ip_input.c
+++ b/bsd/netinet/ip_input.c
@@ -354,14 +354,11 @@ ip_init()
 		ipf_init();
 
 		ip_mutex_grp_attr  = lck_grp_attr_alloc_init();
-		lck_grp_attr_setdefault(ip_mutex_grp_attr);
 
 		ip_mutex_grp = lck_grp_alloc_init("ip", ip_mutex_grp_attr);
 
 		ip_mutex_attr = lck_attr_alloc_init();
 
-		lck_attr_setdefault(ip_mutex_attr);
-
 		if ((ip_mutex = lck_mtx_alloc_init(ip_mutex_grp, ip_mutex_attr)) == NULL) {
 			printf("ip_init: can't alloc ip_mutex\n");
 			return;
diff --git a/bsd/netinet/ip_mroute.h b/bsd/netinet/ip_mroute.h
index 3d1a18fee..684a08acb 100644
--- a/bsd/netinet/ip_mroute.h
+++ b/bsd/netinet/ip_mroute.h
@@ -296,4 +296,4 @@ extern int	(*mrt_ioctl)(int, caddr_t, struct proc *);
 #endif
 
 #endif KERNEL_PRIVATE
-#endif _NETINET_IP_MROUTE_H_
+#endif /* _NETINET_IP_MROUTE_H_ */
diff --git a/bsd/netinet/ip_output.c b/bsd/netinet/ip_output.c
index e3c2b862d..e95d54b91 100644
--- a/bsd/netinet/ip_output.c
+++ b/bsd/netinet/ip_output.c
@@ -483,6 +483,11 @@ loopit:
 				
 				lck_mtx_unlock(ip_mutex);
 				ipf_ref();
+				
+				/* 4135317 - always pass network byte order to filter */
+				HTONS(ip->ip_len);
+				HTONS(ip->ip_off);
+				
 				TAILQ_FOREACH(filter, &ipv4_filters, ipf_link) {
 					if (seen == 0) {
 						if ((struct ipfilter *)inject_filter_ref == filter)
@@ -501,6 +506,11 @@ loopit:
 						}
 					}
 				}
+				
+				/* set back to host byte order */
+				NTOHS(ip->ip_len);
+				NTOHS(ip->ip_off);
+				
 				lck_mtx_lock(ip_mutex);
 				ipf_unref();
 				didfilter = 1;
@@ -614,6 +624,11 @@ injectit:
 		
 		lck_mtx_unlock(ip_mutex);
 		ipf_ref();
+		
+		/* 4135317 - always pass network byte order to filter */
+		HTONS(ip->ip_len);
+		HTONS(ip->ip_off);
+		
 		TAILQ_FOREACH(filter, &ipv4_filters, ipf_link) {
 			if (seen == 0) {
 				if ((struct ipfilter *)inject_filter_ref == filter)
@@ -632,6 +647,11 @@ injectit:
 				}
 			}
 		}
+		
+		/* set back to host byte order */
+		NTOHS(ip->ip_len);
+		NTOHS(ip->ip_off);
+		
 		ipf_unref();
 		lck_mtx_lock(ip_mutex);
 	}
@@ -809,6 +829,11 @@ injectit:
 		
 		lck_mtx_unlock(ip_mutex);
 		ipf_ref();
+		
+		/* 4135317 - always pass network byte order to filter */
+		HTONS(ip->ip_len);
+		HTONS(ip->ip_off);
+		
 		TAILQ_FOREACH(filter, &ipv4_filters, ipf_link) {
 			if (filter->ipf_filter.ipf_output) {
 				errno_t result;
@@ -824,6 +849,11 @@ injectit:
 				}
 			}
 		}
+		
+		/* set back to host byte order */
+		NTOHS(ip->ip_len);
+		NTOHS(ip->ip_off);
+		
 		ipf_unref();
 		lck_mtx_lock(ip_mutex);
 	}
diff --git a/bsd/netinet/ip_var.h b/bsd/netinet/ip_var.h
index 197064300..53702d8a2 100644
--- a/bsd/netinet/ip_var.h
+++ b/bsd/netinet/ip_var.h
@@ -242,5 +242,5 @@ void	ip_rsvp_force_done(struct socket *);
 
 void	in_delayed_cksum(struct mbuf *m);
 
-#endif KERNEL_PRIVATE
-#endif !_NETINET_IP_VAR_H_
+#endif /* KERNEL_PRIVATE */
+#endif /* !_NETINET_IP_VAR_H_ */
diff --git a/bsd/netinet/kpi_ipfilter.c b/bsd/netinet/kpi_ipfilter.c
index 270b87aba..bb8f2bf9d 100644
--- a/bsd/netinet/kpi_ipfilter.c
+++ b/bsd/netinet/kpi_ipfilter.c
@@ -307,6 +307,11 @@ ipf_injectv4_out(
 		m_freem(m);
 		return ENETUNREACH;
 	}
+	
+	/* Put ip_len and ip_off in host byte order, ip_output expects that */
+	NTOHS(ip->ip_len);
+	NTOHS(ip->ip_off);
+	
 	/* Send  */
 	error = ip_output(m, NULL, &ro, IP_ALLOWBROADCAST | IP_RAWOUTPUT, imo);
 	
@@ -455,7 +460,6 @@ ipf_init(void)
 		error = ENOMEM;
 		goto done;
 	}
-	lck_grp_attr_setdefault(grp_attributes);
 	
 	lck_grp = lck_grp_alloc_init("IP Filter", grp_attributes);
 	if (lck_grp == 0) {
@@ -470,7 +474,6 @@ ipf_init(void)
 		error = ENOMEM;
 		goto done;
 	}
-	lck_attr_setdefault(lck_attributes);
 	
 	kipf_lock = lck_mtx_alloc_init(lck_grp, lck_attributes);
 	if (kipf_lock == 0) {
diff --git a/bsd/netinet/raw_ip.c b/bsd/netinet/raw_ip.c
index fd2ae4fe6..6219b9250 100644
--- a/bsd/netinet/raw_ip.c
+++ b/bsd/netinet/raw_ip.c
@@ -150,7 +150,6 @@ rip_init()
 	 * allocate lock group attribute and group for udp pcb mutexes
 	 */
 	pcbinfo->mtx_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setdefault(pcbinfo->mtx_grp_attr);
 
 	pcbinfo->mtx_grp = lck_grp_alloc_init("ripcb", pcbinfo->mtx_grp_attr);
 		
@@ -158,7 +157,6 @@ rip_init()
 	 * allocate the lock attribute for udp pcb mutexes
 	 */
 	pcbinfo->mtx_attr = lck_attr_alloc_init();
-	lck_attr_setdefault(pcbinfo->mtx_attr);
 
 	if ((pcbinfo->mtx = lck_rw_alloc_init(pcbinfo->mtx_grp, pcbinfo->mtx_attr)) == NULL)
 		return;	/* pretty much dead if this fails... */
@@ -742,22 +740,25 @@ rip_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
 	return rip_output(m, so, dst);
 }
 
+/* note: rip_unlock is called from different protos  instead of the generic socket_unlock,
+ * it will handle the socket dealloc on last reference 
+ * */
 int
 rip_unlock(struct socket *so, int refcount, int debug)
 {
 	int lr_saved;
 	struct inpcb *inp = sotoinpcb(so);
-#ifdef __ppc__
-	if (debug == 0) {
-		__asm__ volatile("mflr %0" : "=r" (lr_saved));
-	}
+
+	if (debug == 0) 
+		lr_saved = (unsigned int) __builtin_return_address(0);
 	else lr_saved = debug;
-#endif
+
 	if (refcount) {
 		if (so->so_usecount <= 0)
 			panic("rip_unlock: bad refoucnt so=%x val=%x\n", so, so->so_usecount);
 		so->so_usecount--;
 		if (so->so_usecount == 0 && (inp->inp_wantcnt == WNT_STOPUSING)) {
+			/* cleanup after last reference */
 			lck_mtx_unlock(so->so_proto->pr_domain->dom_mtx);
 			lck_rw_lock_exclusive(ripcbinfo.mtx);
 			in_pcbdispose(inp);
@@ -765,6 +766,8 @@ rip_unlock(struct socket *so, int refcount, int debug)
 			return(0);
 		}
 	}
+	so->unlock_lr[so->next_unlock_lr] = (u_int *)lr_saved;
+	so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX;
 	lck_mtx_unlock(so->so_proto->pr_domain->dom_mtx);
 	return(0);
 }
diff --git a/bsd/netinet/tcp.h b/bsd/netinet/tcp.h
index d7fc1d20c..4a2460b16 100644
--- a/bsd/netinet/tcp.h
+++ b/bsd/netinet/tcp.h
@@ -114,7 +114,7 @@ struct tcphdr {
 #define TCPOPT_WINDOW		3
 #define TCPOLEN_WINDOW		3
 #define TCPOPT_SACK_PERMITTED	4		/* Experimental */
-#define TCPOLEN_SACK_PERMITTED	2
+#define    TCPOLEN_SACK_PERMITTED	2
 #define TCPOPT_SACK		5		/* Experimental */
 #define TCPOLEN_SACK		8		/* len of sack block */
 #define TCPOPT_TIMESTAMP	8
@@ -201,7 +201,6 @@ struct tcphdr {
 #define TCP_NOPUSH		0x04	/* don't push last block of write */
 #define TCP_NOOPT		0x08	/* don't use TCP options */
 #define TCP_KEEPALIVE	0x10	/* idle time used when SO_KEEPALIVE is enabled */
-
 #endif /* _POSIX_C_SOURCE */
 
 #endif
diff --git a/bsd/netinet/tcp_input.c b/bsd/netinet/tcp_input.c
index a766f96e7..a6ea7bf86 100644
--- a/bsd/netinet/tcp_input.c
+++ b/bsd/netinet/tcp_input.c
@@ -1232,6 +1232,7 @@ findpcb:
 	if (tp->t_state != TCPS_LISTEN && optp)
 		tcp_dooptions(tp, optp, optlen, th, &to);
 
+
 	if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) {
 		if (to.to_flags & TOF_SCALE) {
 			tp->t_flags |= TF_RCVD_SCALE;
@@ -1790,11 +1791,17 @@ trimthenstep6:
 	 *    TIME_WAIT STATE:
 	 *	Drop the segment - see Stevens, vol. 2, p. 964 and
 	 *      RFC 1337.
+	 *
+	 *      Radar 4776325: Allows for the case where we ACKed the FIN but
+	 *      	       there is already a RST in flight from the peer.
+	 *      	       In that case, accept the RST for non-established
+	 *      	       state if it's one off from last_ack_sent.
 	 */
 	if (thflags & TH_RST) {
 		if ((SEQ_GEQ(th->th_seq, tp->last_ack_sent) &&
 		    SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) ||
-		    (tp->rcv_wnd == 0 && tp->last_ack_sent == th->th_seq)) {
+		    (tp->rcv_wnd == 0 && 
+		     	(tp->last_ack_sent == th->th_seq)) || ((tp->last_ack_sent - 1) == th->th_seq)) {
 			switch (tp->t_state) {
 
 			case TCPS_SYN_RECEIVED:
diff --git a/bsd/netinet/tcp_subr.c b/bsd/netinet/tcp_subr.c
index 2f61e0e1e..093cd3cbc 100644
--- a/bsd/netinet/tcp_subr.c
+++ b/bsd/netinet/tcp_subr.c
@@ -323,14 +323,12 @@ tcp_init()
 	 * allocate lock group attribute and group for tcp pcb mutexes
 	 */
      pcbinfo->mtx_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setdefault(pcbinfo->mtx_grp_attr);
 	pcbinfo->mtx_grp = lck_grp_alloc_init("tcppcb", pcbinfo->mtx_grp_attr);
 		
 	/*
 	 * allocate the lock attribute for tcp pcb mutexes
 	 */
 	pcbinfo->mtx_attr = lck_attr_alloc_init();
-	lck_attr_setdefault(pcbinfo->mtx_attr);
 
 	if ((pcbinfo->mtx = lck_rw_alloc_init(pcbinfo->mtx_grp, pcbinfo->mtx_attr)) == NULL) {
 		printf("tcp_init: mutex not alloced!\n");
@@ -1675,12 +1673,9 @@ tcp_lock(so, refcount, lr)
 	int lr;
 {
 	int lr_saved;
-#ifdef __ppc__
-	if (lr == 0) {
-		__asm__ volatile("mflr %0" : "=r" (lr_saved));
-	}
+	if (lr == 0) 
+		lr_saved = (unsigned int) __builtin_return_address(0);
 	else lr_saved = lr;
-#endif
 
 	if (so->so_pcb) {
 		lck_mtx_lock(((struct inpcb *)so->so_pcb)->inpcb_mtx);
@@ -1696,7 +1691,8 @@ tcp_lock(so, refcount, lr)
 
 	if (refcount)
 		so->so_usecount++;
-	so->reserved3 = (void *)lr_saved;
+	so->lock_lr[so->next_lock_lr] = (u_int32_t *)lr_saved;
+	so->next_lock_lr = (so->next_lock_lr+1) % SO_LCKDBG_MAX;
 	return (0);
 }
 
@@ -1707,12 +1703,9 @@ tcp_unlock(so, refcount, lr)
 	int lr;
 {
 	int lr_saved;
-#ifdef __ppc__
-	if (lr == 0) {
-		__asm__ volatile("mflr %0" : "=r" (lr_saved));
-	}
+	if (lr == 0) 
+		lr_saved = (unsigned int) __builtin_return_address(0);
 	else lr_saved = lr;
-#endif
 
 #ifdef MORE_TCPLOCK_DEBUG
 	printf("tcp_unlock: so=%x sopcb=%x lock=%x ref=%x lr=%x\n", 
@@ -1723,15 +1716,14 @@ tcp_unlock(so, refcount, lr)
 
 	if (so->so_usecount < 0)
 		panic("tcp_unlock: so=%x usecount=%x\n", so, so->so_usecount);	
-	if (so->so_pcb == NULL) {
+	if (so->so_pcb == NULL) 
 		panic("tcp_unlock: so=%x NO PCB usecount=%x lr=%x\n", so, so->so_usecount, lr_saved);
-		lck_mtx_unlock(so->so_proto->pr_domain->dom_mtx);
-	}
 	else {
 		lck_mtx_assert(((struct inpcb *)so->so_pcb)->inpcb_mtx, LCK_MTX_ASSERT_OWNED);
+		so->unlock_lr[so->next_unlock_lr] = (u_int *)lr_saved;
+		so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX;
 		lck_mtx_unlock(((struct inpcb *)so->so_pcb)->inpcb_mtx);
 	}
-	so->reserved4 = (void *)lr_saved;
 	return (0);
 }
 
diff --git a/bsd/netinet/udp_usrreq.c b/bsd/netinet/udp_usrreq.c
index c0376e448..8d3f88735 100644
--- a/bsd/netinet/udp_usrreq.c
+++ b/bsd/netinet/udp_usrreq.c
@@ -205,12 +205,10 @@ udp_init()
 	 * allocate lock group attribute and group for udp pcb mutexes
 	 */
 	pcbinfo->mtx_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setdefault(pcbinfo->mtx_grp_attr);
 
 	pcbinfo->mtx_grp = lck_grp_alloc_init("udppcb", pcbinfo->mtx_grp_attr);
 		
 	pcbinfo->mtx_attr = lck_attr_alloc_init();
-	lck_attr_setdefault(pcbinfo->mtx_attr);
 
 	if ((pcbinfo->mtx = lck_rw_alloc_init(pcbinfo->mtx_grp, pcbinfo->mtx_attr)) == NULL)
 		return;	/* pretty much dead if this fails... */
@@ -1228,27 +1226,22 @@ udp_lock(so, refcount, debug)
 	int refcount, debug;
 {
 	int lr_saved;
-#ifdef __ppc__
-	if (debug == 0) {
-		__asm__ volatile("mflr %0" : "=r" (lr_saved));
-	}
+	if (debug == 0) 
+		lr_saved = (unsigned int) __builtin_return_address(0);
 	else lr_saved = debug;
-#endif
 
 	if (so->so_pcb) {
 		lck_mtx_assert(((struct inpcb *)so->so_pcb)->inpcb_mtx, LCK_MTX_ASSERT_NOTOWNED);
 		lck_mtx_lock(((struct inpcb *)so->so_pcb)->inpcb_mtx);
 	}
-	else {
+	else 
 		panic("udp_lock: so=%x NO PCB! lr=%x\n", so, lr_saved);
-		lck_mtx_assert(so->so_proto->pr_domain->dom_mtx, LCK_MTX_ASSERT_NOTOWNED);
-		lck_mtx_lock(so->so_proto->pr_domain->dom_mtx);
-	}
 
 	if (refcount) 
 		so->so_usecount++;
 
-	so->reserved3= lr_saved;
+	so->lock_lr[so->next_lock_lr] = (void *)lr_saved;
+	so->next_lock_lr = (so->next_lock_lr+1) % SO_LCKDBG_MAX;
 	return (0);
 }
 
@@ -1261,12 +1254,11 @@ udp_unlock(so, refcount, debug)
 	int lr_saved;
 	struct inpcb *inp = sotoinpcb(so);
     	struct inpcbinfo *pcbinfo	= &udbinfo;
-#ifdef __ppc__
-	if (debug == 0) {
-		__asm__ volatile("mflr %0" : "=r" (lr_saved));
-	}
+
+	if (debug == 0) 
+		lr_saved = (unsigned int) __builtin_return_address(0);
 	else lr_saved = debug;
-#endif
+
 	if (refcount) {
 		so->so_usecount--;
 #if 0
@@ -1279,18 +1271,16 @@ udp_unlock(so, refcount, debug)
 		}
 #endif
 	}
-	if (so->so_pcb == NULL) {
+	if (so->so_pcb == NULL) 
 		panic("udp_unlock: so=%x NO PCB! lr=%x\n", so, lr_saved);
-		lck_mtx_assert(so->so_proto->pr_domain->dom_mtx, LCK_MTX_ASSERT_OWNED);
-		lck_mtx_unlock(so->so_proto->pr_domain->dom_mtx);
-	}
 	else {
 		lck_mtx_assert(((struct inpcb *)so->so_pcb)->inpcb_mtx, LCK_MTX_ASSERT_OWNED);
+		so->unlock_lr[so->next_unlock_lr] = (void *)lr_saved;
+		so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX;
 		lck_mtx_unlock(((struct inpcb *)so->so_pcb)->inpcb_mtx);
 	}
 
 
-	so->reserved4 = lr_saved;
 	return (0);
 }
 
diff --git a/bsd/netinet/udp_var.h b/bsd/netinet/udp_var.h
index 2edc73e82..77854b776 100644
--- a/bsd/netinet/udp_var.h
+++ b/bsd/netinet/udp_var.h
@@ -151,5 +151,5 @@ lck_mtx_t *	udp_getlock (struct socket *, int);
 void *	udp_getlock (struct socket *, int);
 #endif
 
-#endif KERNEL_PRIVATE
-#endif _NETINET_UDP_VAR_H_
+#endif /* KERNEL_PRIVATE */
+#endif /* _NETINET_UDP_VAR_H_ */
diff --git a/bsd/netinet6/ah.h b/bsd/netinet6/ah.h
index 3e7f8dcf8..e2a75ebd8 100644
--- a/bsd/netinet6/ah.h
+++ b/bsd/netinet6/ah.h
@@ -88,4 +88,4 @@ extern int ah4_calccksum(struct mbuf *, caddr_t, size_t,
 	const struct ah_algorithm *, struct secasvar *);
 #endif KERNEL_PRIVATE
 
-#endif _NETINET6_AH_H_
+#endif /* _NETINET6_AH_H_ */
diff --git a/bsd/netinet6/esp.h b/bsd/netinet6/esp.h
index 1d9d0c0ba..6802e81a3 100644
--- a/bsd/netinet6/esp.h
+++ b/bsd/netinet6/esp.h
@@ -102,6 +102,6 @@ extern size_t esp_hdrsiz(struct ipsecrequest *);
 extern int esp_schedule(const struct esp_algorithm *, struct secasvar *);
 extern int esp_auth(struct mbuf *, size_t, size_t,
 	struct secasvar *, u_char *);
-#endif KERNEL_PRIVATE
+#endif /* KERNEL_PRIVATE */
 
-#endif _NETINET6_ESP_H_
+#endif /* _NETINET6_ESP_H_ */
diff --git a/bsd/netinet6/in6.h b/bsd/netinet6/in6.h
index 47f18fed4..989ea25a4 100644
--- a/bsd/netinet6/in6.h
+++ b/bsd/netinet6/in6.h
@@ -182,7 +182,8 @@ extern const struct in6_addr in6mask32;
 extern const struct in6_addr in6mask64;
 extern const struct in6_addr in6mask96;
 extern const struct in6_addr in6mask128;
-#endif KERNEL_PRIVATE
+#endif /* KERNEL_PRIVATE */
+
 
 /*
  * Macros started with IPV6_ADDR is KAME local
@@ -413,7 +414,7 @@ struct route_in6 {
 #define IPV6_RECVRETOPTS	6  /* bool; receive IP6 opts for response */
 #define IPV6_RECVDSTADDR	7  /* bool; receive IP6 dst addr w/dgram */
 #define IPV6_RETOPTS		8  /* ip6_opts; set/get IP6 options */
-#endif 0
+#endif /* 0 */
 #define IPV6_SOCKOPT_RESERVED1	3  /* reserved for future use */
 #endif /* _POSIX_C_SOURCE */
 #define IPV6_UNICAST_HOPS	4  /* int; IP6 hops */
@@ -441,12 +442,12 @@ struct route_in6 {
 #ifndef _POSIX_C_SOURCE
 #ifndef KERNEL
 #define IPV6_BINDV6ONLY		IPV6_V6ONLY
-#endif KERNEL
+#endif /* KERNEL */
 
 
 #if 1 /*IPSEC*/
 #define IPV6_IPSEC_POLICY	28 /* struct; get/set security policy */
-#endif 1
+#endif /* 1 */
 #define IPV6_FAITH		29 /* bool; accept FAITH'ed connections */
 
 #if 1 /*IPV6FIREWALL*/
@@ -455,7 +456,7 @@ struct route_in6 {
 #define IPV6_FW_FLUSH		32 /* flush firewall rule chain */
 #define IPV6_FW_ZERO		33 /* clear single/all firewall counter(s) */
 #define IPV6_FW_GET		34 /* get entire firewall rule chain */
-#endif 1
+#endif /* 1 */
 
 /* to define items, should talk with KAME guys first, for *BSD compatibility */
 
@@ -626,7 +627,7 @@ void	in6_sin_2_v4mapsin6_in_sock(struct sockaddr **nam);
 #define	satosin6(sa)	((struct sockaddr_in6 *)(sa))
 #define	sin6tosa(sin6)	((struct sockaddr *)(sin6))
 #define	ifatoia6(ifa)	((struct in6_ifaddr *)(ifa))
-#endif KERNEL_PRIVATE
+#endif /* KERNEL_PRIVATE */
 
 #ifndef KERNEL
 __BEGIN_DECLS
@@ -670,6 +671,6 @@ extern int inet6_rth_reverse(const void *, void *);
 extern int inet6_rth_segments(const void *);
 extern struct in6_addr *inet6_rth_getaddr(const void *, int);
 __END_DECLS
-#endif !KERNEL
+#endif /* !KERNEL */
 #endif /* _POSIX_C_SOURCE */
 #endif /* !_NETINET6_IN6_H_ */
diff --git a/bsd/netinet6/in6_var.h b/bsd/netinet6/in6_var.h
index 787393d33..0ef5ef924 100644
--- a/bsd/netinet6/in6_var.h
+++ b/bsd/netinet6/in6_var.h
@@ -648,6 +648,6 @@ int in6_embedscope __P((struct in6_addr *, const struct sockaddr_in6 *,
 int in6_recoverscope __P((struct sockaddr_in6 *, const struct in6_addr *,
 	struct ifnet *));
 void in6_clearscope __P((struct in6_addr *));
-#endif KERNEL_PRIVATE
+#endif /* KERNEL_PRIVATE */
 
-#endif _NETINET6_IN6_VAR_H_
+#endif /* _NETINET6_IN6_VAR_H_ */
diff --git a/bsd/netinet6/ip6_input.c b/bsd/netinet6/ip6_input.c
index ccfce46ed..64221cf42 100644
--- a/bsd/netinet6/ip6_input.c
+++ b/bsd/netinet6/ip6_input.c
@@ -211,11 +211,9 @@ ip6_init()
 	}
 
 	ip6_mutex_grp_attr  = lck_grp_attr_alloc_init();
-	lck_grp_attr_setdefault(ip6_mutex_grp_attr);
 
 	ip6_mutex_grp = lck_grp_alloc_init("ip6", ip6_mutex_grp_attr);
 	ip6_mutex_attr = lck_attr_alloc_init();
-	lck_attr_setdefault(ip6_mutex_attr);
 
 	if ((ip6_mutex = lck_mtx_alloc_init(ip6_mutex_grp, ip6_mutex_attr)) == NULL) {
 		printf("ip6_init: can't alloc ip6_mutex\n");
diff --git a/bsd/netinet6/ipcomp.h b/bsd/netinet6/ipcomp.h
index 41ab61b85..8d17dc7de 100644
--- a/bsd/netinet6/ipcomp.h
+++ b/bsd/netinet6/ipcomp.h
@@ -64,7 +64,7 @@ struct ipsecrequest;
 extern const struct ipcomp_algorithm *ipcomp_algorithm_lookup(int);
 extern void ipcomp4_input(struct mbuf *, int);
 extern int ipcomp4_output(struct mbuf *, struct ipsecrequest *);
-#endif KERNEL_PRIVATE
-#endif KERNEL
+#endif /* KERNEL_PRIVATE */
+#endif /* KERNEL */
 
-#endif _NETINET6_IPCOMP_H_
+#endif /* _NETINET6_IPCOMP_H_ */
diff --git a/bsd/netinet6/ipsec.h b/bsd/netinet6/ipsec.h
index 2a4f6dcef..154ff39a1 100644
--- a/bsd/netinet6/ipsec.h
+++ b/bsd/netinet6/ipsec.h
@@ -345,4 +345,4 @@ extern char *ipsec_dump_policy(caddr_t, char *);
 extern const char *ipsec_strerror(void);
 #endif KERNEL
 
-#endif _NETINET6_IPSEC_H_
+#endif /* _NETINET6_IPSEC_H_ */
diff --git a/bsd/netinet6/pim6_var.h b/bsd/netinet6/pim6_var.h
index 1cb8ec648..c90b9bdde 100644
--- a/bsd/netinet6/pim6_var.h
+++ b/bsd/netinet6/pim6_var.h
@@ -52,7 +52,6 @@ struct pim6stat {
 	u_quad_t pim6s_snd_registers;	/* sent registers		*/
 };
 
-extern struct pim6stat pim6stat;
 
 
 /*
diff --git a/bsd/netkey/key.c b/bsd/netkey/key.c
index 2c635da4c..aee29ba54 100644
--- a/bsd/netkey/key.c
+++ b/bsd/netkey/key.c
@@ -526,7 +526,6 @@ key_init(void)
 	sadb_mutex_grp_attr = lck_grp_attr_alloc_init();
 	sadb_mutex_grp = lck_grp_alloc_init("sadb", sadb_mutex_grp_attr);
 	sadb_mutex_attr = lck_attr_alloc_init();
-	lck_attr_setdefault(sadb_mutex_attr);
 
 	if ((sadb_mutex = lck_mtx_alloc_init(sadb_mutex_grp, sadb_mutex_attr)) == NULL) {
 		printf("key_init: can't alloc sadb_mutex\n");
diff --git a/bsd/nfs/nfs.h b/bsd/nfs/nfs.h
index b23acc3aa..79e58917f 100644
--- a/bsd/nfs/nfs.h
+++ b/bsd/nfs/nfs.h
@@ -100,6 +100,7 @@
 #ifndef NFS_MAXDIRATTRTIMO
 #define	NFS_MAXDIRATTRTIMO 60
 #endif
+#define	NFS_IOSIZE	(256 * 1024)	/* suggested I/O size */
 #define	NFS_WSIZE	16384		/* Def. write data size <= 16K */
 #define	NFS_RSIZE	16384		/* Def. read data size <= 16K */
 #define	NFS_DGRAM_WSIZE	8192		/* UDP Def. write data size <= 8K */
@@ -221,17 +222,13 @@ struct nfs_args3 {
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with nfs_args
  */
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_nfs_args {
 	int		version;	/* args structure version number */
-	user_addr_t	addr;		/* file server address */
+	user_addr_t	addr __attribute((aligned(8)));		/* file server address */
 	int		addrlen;	/* length of address */
 	int		sotype;		/* Socket type */
 	int		proto;		/* and Protocol */
-	user_addr_t	fh;		/* File handle to be mounted */
+	user_addr_t	fh __attribute((aligned(8)));		/* File handle to be mounted */
 	int		fhsize;		/* Size, in bytes, of fh */
 	int		flags;		/* flags */
 	int		wsize;		/* write size in bytes */
@@ -243,7 +240,7 @@ struct user_nfs_args {
 	int		readahead;	/* # of blocks to readahead */
 	int		leaseterm;	/* obsolete: Term (sec) of lease */
 	int		deadthresh;	/* obsolete: Retrans threshold */
-	user_addr_t	hostname;	/* server's name */
+	user_addr_t	hostname __attribute((aligned(8)));	/* server's name */
 	/* NFS_ARGSVERSION 3 ends here */
 	int		acregmin;	/* reg file min attr cache timeout */
 	int		acregmax;	/* reg file max attr cache timeout */
@@ -252,11 +249,11 @@ struct user_nfs_args {
 };
 struct user_nfs_args3 {
 	int		version;	/* args structure version number */
-	user_addr_t	addr;		/* file server address */
+	user_addr_t	addr __attribute((aligned(8)));		/* file server address */
 	int		addrlen;	/* length of address */
 	int		sotype;		/* Socket type */
 	int		proto;		/* and Protocol */
-	user_addr_t	fh;		/* File handle to be mounted */
+	user_addr_t	fh __attribute((aligned(8)));		/* File handle to be mounted */
 	int		fhsize;		/* Size, in bytes, of fh */
 	int		flags;		/* flags */
 	int		wsize;		/* write size in bytes */
@@ -268,13 +265,9 @@ struct user_nfs_args3 {
 	int		readahead;	/* # of blocks to readahead */
 	int		leaseterm;	/* obsolete: Term (sec) of lease */
 	int		deadthresh;	/* obsolete: Retrans threshold */
-	user_addr_t	hostname;	/* server's name */
+	user_addr_t	hostname __attribute((aligned(8)));	/* server's name */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif // KERNEL
 
 /*
@@ -346,20 +339,12 @@ struct nfsd_args {
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with nfsd_args
  */
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_nfsd_args {
 	int	        sock;		/* Socket to serve */
-	user_addr_t	name;		/* Client addr for connection based sockets */
+	user_addr_t	name __attribute((aligned(8)));		/* Client addr for connection based sockets */
 	int	        namelen;	/* Length of name */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif // KERNEL
 
 struct nfsd_srvargs {
@@ -439,10 +424,6 @@ struct nfs_export_args {
 #ifdef KERNEL
 /* LP64 version of export_args */
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_nfs_export_args {
 	uint32_t		nxa_fsid;	/* export FS ID */
 	uint32_t		nxa_expid;	/* export ID */
@@ -453,10 +434,6 @@ struct user_nfs_export_args {
 	user_addr_t		nxa_nets;	/* array of net args */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif /* KERNEL */
 
 /* nfs export arg flags */
diff --git a/bsd/nfs/nfs_bio.c b/bsd/nfs/nfs_bio.c
index 014eb06e0..80fa7fb2f 100644
--- a/bsd/nfs/nfs_bio.c
+++ b/bsd/nfs/nfs_bio.c
@@ -160,7 +160,6 @@ void
 nfs_nbinit(void)
 {
 	nfs_buf_lck_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(nfs_buf_lck_grp_attr);
 	nfs_buf_lck_grp = lck_grp_alloc_init("nfs_buf", nfs_buf_lck_grp_attr);
 
 	nfs_buf_lck_attr = lck_attr_alloc_init();
@@ -260,13 +259,11 @@ nfs_buf_freeup(int timer)
 	while ((fbp = TAILQ_FIRST(&nfsbuffreeup))) {
 		TAILQ_REMOVE(&nfsbuffreeup, fbp, nb_free);
 		/* nuke any creds */
-		if (fbp->nb_rcred != NOCRED) {
-			kauth_cred_rele(fbp->nb_rcred);
-			fbp->nb_rcred = NOCRED;
+		if (IS_VALID_CRED(fbp->nb_rcred)) {
+			kauth_cred_unref(&fbp->nb_rcred);
 		}
-		if (fbp->nb_wcred != NOCRED) {
-			kauth_cred_rele(fbp->nb_wcred);
-			fbp->nb_wcred = NOCRED;
+		if (IS_VALID_CRED(fbp->nb_wcred)) {
+			kauth_cred_unref(&fbp->nb_wcred);
 		}
 		/* if buf was NB_META, dump buffer */
 		if (ISSET(fbp->nb_flags, NB_META) && fbp->nb_data)
@@ -636,9 +633,9 @@ nfs_buf_get(
 	struct nfsbuf **bpp)
 {
 	struct nfsnode *np = VTONFS(vp);
+	struct nfsmount *nmp = VFSTONFS(vnode_mount(vp));
 	struct nfsbuf *bp;
 	int biosize, bufsize;
-	kauth_cred_t cred;
 	int slpflag = PCATCH;
 	int operation = (flags & NBLK_OPMASK);
 	int error = 0;
@@ -648,10 +645,14 @@ nfs_buf_get(
 	*bpp = NULL;
 
 	bufsize = size;
-	if (bufsize > MAXBSIZE)
-		panic("nfs_buf_get: buffer larger than MAXBSIZE requested");
+	if (bufsize > NFS_MAXBSIZE)
+		panic("nfs_buf_get: buffer larger than NFS_MAXBSIZE requested");
 
-	biosize = vfs_statfs(vnode_mount(vp))->f_iosize;
+	if (!nmp) {
+		FSDBG_BOT(541, vp, blkno, 0, ENXIO);
+		return (ENXIO);
+	}
+	biosize = nmp->nm_biosize;
 
 	if (UBCINVALID(vp) || !UBCINFOEXISTS(vp)) {
 		operation = NBLK_META;
@@ -784,15 +785,11 @@ loop:
 		}
 		LIST_REMOVE(bp, nb_hash);
 		/* nuke any creds we're holding */
-		cred = bp->nb_rcred;
-		if (cred != NOCRED) {
-			bp->nb_rcred = NOCRED; 
-			kauth_cred_rele(cred);
+		if (IS_VALID_CRED(bp->nb_rcred)) {
+			kauth_cred_unref(&bp->nb_rcred);
 		}
-		cred = bp->nb_wcred;
-		if (cred != NOCRED) {
-			bp->nb_wcred = NOCRED; 
-			kauth_cred_rele(cred);
+		if (IS_VALID_CRED(bp->nb_wcred)) {
+			kauth_cred_unref(&bp->nb_wcred);
 		}
 		/* if buf will no longer be NB_META, dump old buffer */
 		if (operation == NBLK_META) {
@@ -982,7 +979,9 @@ nfs_buf_release(struct nfsbuf *bp, int freeup)
 				panic("ubc_upl_unmap failed");
 			bp->nb_data = NULL;
 		}
-		if (bp->nb_flags & (NB_ERROR | NB_INVAL | NB_NOCACHE)) {
+		/* abort pages if error, invalid, or non-needcommit nocache */
+		if ((bp->nb_flags & (NB_ERROR | NB_INVAL)) ||
+		    ((bp->nb_flags & NB_NOCACHE) && !(bp->nb_flags & (NB_NEEDCOMMIT | NB_DELWRI)))) {
 			if (bp->nb_flags & (NB_READ | NB_INVAL | NB_NOCACHE))
 				upl_flags = UPL_ABORT_DUMP_PAGES;
 			else
@@ -1012,10 +1011,9 @@ pagelist_cleanup_done:
 		/* was this the last buffer in the file? */
 		if (NBOFF(bp) + bp->nb_bufsize > (off_t)(VTONFS(vp)->n_size)) {
 			/* if so, invalidate all pages of last buffer past EOF */
-			int biosize = vfs_statfs(vnode_mount(vp))->f_iosize;
 			off_t start, end;
 			start = trunc_page_64(VTONFS(vp)->n_size) + PAGE_SIZE_64;
-			end = trunc_page_64(NBOFF(bp) + biosize);
+			end = trunc_page_64(NBOFF(bp) + bp->nb_bufsize);
 			if (end > start) {
 				if (!(rv = ubc_sync_range(vp, start, end, UBC_INVALIDATE)))
 					printf("nfs_buf_release(): ubc_sync_range failed!\n");
@@ -1040,8 +1038,9 @@ pagelist_cleanup_done:
 		wakeup_buffer = 1;
 	}
 
-	/* If it's not cacheable, or an error, mark it invalid. */
-	if (ISSET(bp->nb_flags, (NB_NOCACHE|NB_ERROR)))
+	/* If it's non-needcommit nocache, or an error, mark it invalid. */
+	if (ISSET(bp->nb_flags, NB_ERROR) ||
+	    (ISSET(bp->nb_flags, NB_NOCACHE) && !ISSET(bp->nb_flags, (NB_NEEDCOMMIT | NB_DELWRI))))
 		SET(bp->nb_flags, NB_INVAL);
 
 	if ((bp->nb_bufsize <= 0) || ISSET(bp->nb_flags, NB_INVAL)) {
@@ -1098,7 +1097,7 @@ pagelist_cleanup_done:
 	NFSBUFCNTCHK(1);
 
 	/* Unlock the buffer. */
-	CLR(bp->nb_flags, (NB_ASYNC | NB_NOCACHE | NB_STABLE | NB_IOD));
+	CLR(bp->nb_flags, (NB_ASYNC | NB_STABLE | NB_IOD));
 	CLR(bp->nb_lflags, NBL_BUSY);
 
 	FSDBG_BOT(548, bp, NBOFF(bp), bp->nb_flags, bp->nb_data);
@@ -1450,9 +1449,10 @@ nfs_bioread(
 		return (EINVAL);
 	}
 
+	biosize = nmp->nm_biosize;
 	if ((nmp->nm_flag & NFSMNT_NFSV3) && !(nmp->nm_state & NFSSTA_GOTFSINFO))
 		nfs_fsinfo(nmp, vp, cred, p);
-	biosize = vfs_statfs(vnode_mount(vp))->f_iosize;
+
 	vtype = vnode_vtype(vp);
 	/*
 	 * For nfs, cache consistency can only be maintained approximately.
@@ -1705,7 +1705,7 @@ again:
 				/* so write the buffer out and try again */
 				CLR(bp->nb_flags, (NB_DONE | NB_ERROR | NB_INVAL));
 				SET(bp->nb_flags, NB_ASYNC);
-				if (bp->nb_wcred == NOCRED)  {
+				if (!IS_VALID_CRED(bp->nb_wcred))  {
 					kauth_cred_ref(cred);
 					bp->nb_wcred = cred;
 				}
@@ -2014,9 +2014,11 @@ nfs_write(ap)
 		FSDBG_BOT(515, vp, uio->uio_offset, uio_uio_resid(uio), np->n_error);
 		return (np->n_error);
 	}
-	if ((nmp->nm_flag & NFSMNT_NFSV3) &&
-	    !(nmp->nm_state & NFSSTA_GOTFSINFO))
-		(void)nfs_fsinfo(nmp, vp, cred, p);
+
+	biosize = nmp->nm_biosize;
+	if ((nmp->nm_flag & NFSMNT_NFSV3) && !(nmp->nm_state & NFSSTA_GOTFSINFO))
+		nfs_fsinfo(nmp, vp, cred, p);
+
 	if (ioflag & (IO_APPEND | IO_SYNC)) {
 		if (np->n_flag & NMODIFIED) {
 			NATTRINVALIDATE(np);
@@ -2049,8 +2051,6 @@ nfs_write(ap)
 		return (0);
 	}
 
-	biosize = vfs_statfs(vnode_mount(vp))->f_iosize;
-
 	if (vnode_isnocache(vp)) {
 		if (!(np->n_flag & NNOCACHE)) {
 			if (NVALIDBUFS(np)) {
@@ -2091,9 +2091,9 @@ again:
 		NFS_BUF_MAP(bp);
 
 		if (np->n_flag & NNOCACHE)
-			SET(bp->nb_flags, (NB_NOCACHE|NB_STABLE));
+			SET(bp->nb_flags, NB_NOCACHE);
 
-		if (bp->nb_wcred == NOCRED) {
+		if (!IS_VALID_CRED(bp->nb_wcred)) {
 			kauth_cred_ref(cred);
 			bp->nb_wcred = cred;
 		}
@@ -2233,7 +2233,7 @@ again:
 				char *d;
 				int i;
 				if (np->n_flag & NNOCACHE)
-					SET(eofbp->nb_flags, (NB_NOCACHE|NB_STABLE));
+					SET(eofbp->nb_flags, NB_NOCACHE);
 				NFS_BUF_MAP(eofbp);
 				FSDBG(516, eofbp, eofoff, biosize - eofoff, 0xe0fff01e);
 				d = eofbp->nb_data;
@@ -2498,9 +2498,14 @@ again:
 
 	} while (uio_uio_resid(uio) > 0 && n > 0);
 
+	if (np->n_flag & NNOCACHE) {
+		/* make sure all the buffers are flushed out */
+		error = nfs_flush(vp, MNT_WAIT, cred, p, 0);
+	}
+
 	np->n_flag &= ~NWRBUSY;
-	FSDBG_BOT(515, vp, uio->uio_offset, uio_uio_resid(uio), 0);
-	return (0);
+	FSDBG_BOT(515, vp, uio->uio_offset, uio_uio_resid(uio), error);
+	return (error);
 }
 
 /*
@@ -2585,7 +2590,7 @@ nfs_vinvalbuf_internal(
 				}
 				bp->nb_dirty &= (1 << (round_page_32(end)/PAGE_SIZE)) - 1;
 				/* also make sure we'll have a credential to do the write */
-				if (mustwrite && (bp->nb_wcred == NOCRED) && (cred == NOCRED)) {
+				if (mustwrite && !IS_VALID_CRED(bp->nb_wcred) && !IS_VALID_CRED(cred)) {
 					printf("nfs_vinvalbuf: found dirty buffer with no write creds\n");
 					mustwrite = 0;
 				}
@@ -2598,7 +2603,7 @@ nfs_vinvalbuf_internal(
 					/* (NB_NOCACHE indicates buffer should be discarded) */
 					CLR(bp->nb_flags, (NB_DONE | NB_ERROR | NB_INVAL | NB_ASYNC));
 					SET(bp->nb_flags, NB_STABLE | NB_NOCACHE);
-					if (bp->nb_wcred == NOCRED) {
+					if (!IS_VALID_CRED(bp->nb_wcred)) {
 						kauth_cred_ref(cred);
 						bp->nb_wcred = cred;
 					}
@@ -2837,13 +2842,13 @@ again:
 		}
 
 		if (ISSET(bp->nb_flags, NB_READ)) {
-			if (bp->nb_rcred == NOCRED && cred != NOCRED) {
+			if (!IS_VALID_CRED(bp->nb_rcred) && IS_VALID_CRED(cred)) {
 				kauth_cred_ref(cred);
 				bp->nb_rcred = cred;
 			}
 		} else {
 			SET(bp->nb_flags, NB_WRITEINPROG);
-			if (bp->nb_wcred == NOCRED && cred != NOCRED) {
+			if (!IS_VALID_CRED(bp->nb_wcred) && IS_VALID_CRED(cred)) {
 				kauth_cred_ref(cred);
 				bp->nb_wcred = cred;
 			}
@@ -3089,10 +3094,10 @@ nfs_doio(struct nfsbuf *bp, kauth_cred_t cr, proc_t p)
 
 		/* compare page mask to nb_dirty; if there are other dirty pages */
 		/* then write FILESYNC; otherwise, write UNSTABLE if async and */
-		/* not needcommit/nocache/call; otherwise write FILESYNC */
+		/* not needcommit/stable; otherwise write FILESYNC */
 		if (bp->nb_dirty & ~pagemask)
 		    iomode = NFSV3WRITE_FILESYNC;
-		else if ((bp->nb_flags & (NB_ASYNC | NB_NEEDCOMMIT | NB_NOCACHE | NB_STABLE)) == NB_ASYNC)
+		else if ((bp->nb_flags & (NB_ASYNC | NB_NEEDCOMMIT | NB_STABLE)) == NB_ASYNC)
 		    iomode = NFSV3WRITE_UNSTABLE;
 		else
 		    iomode = NFSV3WRITE_FILESYNC;
@@ -3140,7 +3145,7 @@ nfs_doio(struct nfsbuf *bp, kauth_cred_t cr, proc_t p)
 		 * NB_NEEDCOMMIT flags.
 		 */
 		if (error == EINTR || (!error && bp->nb_flags & NB_NEEDCOMMIT)) {
-		    CLR(bp->nb_flags, NB_INVAL | NB_NOCACHE);
+		    CLR(bp->nb_flags, NB_INVAL);
 		    if (!ISSET(bp->nb_flags, NB_DELWRI)) {
 			SET(bp->nb_flags, NB_DELWRI);
 			OSAddAtomic(1, (SInt32*)&nfs_nbdwrite);
diff --git a/bsd/nfs/nfs_node.c b/bsd/nfs/nfs_node.c
index 7ae725953..5c04a0f60 100644
--- a/bsd/nfs/nfs_node.c
+++ b/bsd/nfs/nfs_node.c
@@ -99,7 +99,6 @@ nfs_nhinit(void)
 	nfsnodehashtbl = hashinit(desiredvnodes, M_NFSNODE, &nfsnodehash);
 
 	nfs_node_hash_lck_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(nfs_node_hash_lck_grp_attr);
 	nfs_node_hash_lck_grp = lck_grp_alloc_init("nfs_node_hash", nfs_node_hash_lck_grp_attr);
 
 	nfs_node_hash_lck_attr = lck_attr_alloc_init();
@@ -317,7 +316,6 @@ nfs_inactive(ap)
 {
 	register struct nfsnode *np;
 	register struct sillyrename *sp;
-	kauth_cred_t cred;
 
 	np = VTONFS(ap->a_vp);
 	if (vnode_vtype(ap->a_vp) != VDIR) {
@@ -348,10 +346,8 @@ nfs_inactive(ap)
 		LIST_REMOVE(np, n_hash);
 		np->n_flag &= ~NHASHED;
 		lck_mtx_unlock(nfs_node_hash_mutex);
-		cred = sp->s_cred;
-		if (cred != NOCRED) {
-			sp->s_cred = NOCRED;
-			kauth_cred_rele(cred);
+		if (IS_VALID_CRED(sp->s_cred)) {
+			kauth_cred_unref(&sp->s_cred);
 		}
 		vnode_rele(sp->s_dvp);
 		FREE_ZONE((caddr_t)sp, sizeof (struct sillyrename), M_NFSREQ);
diff --git a/bsd/nfs/nfs_serv.c b/bsd/nfs/nfs_serv.c
index 22df9dfbb..04c0aee42 100644
--- a/bsd/nfs/nfs_serv.c
+++ b/bsd/nfs/nfs_serv.c
@@ -1171,11 +1171,7 @@ nfsrv_write(nfsd, slp, procp, mrq)
 			*tl++ = txdr_unsigned(stable);
 		else
 			*tl++ = txdr_unsigned(NFSV3WRITE_FILESYNC);
-		/*
-		 * Actually, there is no need to txdr these fields,
-		 * but it may make the values more human readable,
-		 * for debugging purposes.
-		 */
+		/* write verifier */
 		*tl++ = txdr_unsigned(boottime_sec());
 		*tl = txdr_unsigned(0);
 	} else {
@@ -1475,11 +1471,7 @@ loop1:
 			    nfsm_build(tl, u_long *, 4 * NFSX_UNSIGNED);
 			    *tl++ = txdr_unsigned(nfsd->nd_len);
 			    *tl++ = txdr_unsigned(swp->nd_stable);
-			    /*
-			     * Actually, there is no need to txdr these fields,
-			     * but it may make the values more human readable,
-			     * for debugging purposes.
-			     */
+			    /* write verifier */
 			    *tl++ = txdr_unsigned(boottime_sec());
 			    *tl = txdr_unsigned(0);
 			} else {
@@ -1605,7 +1597,10 @@ nfsrvw_sort(list, num)
 /*
  * copy credentials making sure that the result can be compared with bcmp().
  *
- * XXX ILLEGAL
+ * NOTE:	This function is only intended to operate on a real input
+ *		credential and a template output credential; the template
+ *		ouptut credential is intended to then be used as an argument
+ *		to kauth_cred_create() - AND NEVER REFERENCED OTHERWISE.
  */
 void
 nfsrv_setcred(kauth_cred_t incred, kauth_cred_t outcred)
@@ -1613,7 +1608,6 @@ nfsrv_setcred(kauth_cred_t incred, kauth_cred_t outcred)
 	int i;
 
 	bzero((caddr_t)outcred, sizeof (*outcred));
-	outcred->cr_ref = 1;
 	outcred->cr_uid = kauth_cred_getuid(incred);
 	outcred->cr_ngroups = incred->cr_ngroups;
 	for (i = 0; i < incred->cr_ngroups; i++)
@@ -1981,8 +1975,6 @@ nfsrv_mknod(nfsd, slp, procp, mrq)
 
 	context.vc_proc = procp;
 	context.vc_ucred = nfsd->nd_cr;
-	hacked_context.vc_proc = procp;
-	hacked_context.vc_ucred = proc_ucred(procp);
 
 	/*
 	 * Save the original credential UID in case they are
@@ -2103,6 +2095,9 @@ nfsrv_mknod(nfsd, slp, procp, mrq)
 			vnode_put(vp);
 			vp = NULL;
 		}
+		hacked_context.vc_proc = procp;
+		hacked_context.vc_ucred = kauth_cred_proc_ref(procp);
+
 		nd.ni_cnd.cn_nameiop = LOOKUP;
 		nd.ni_cnd.cn_flags &= ~LOCKPARENT;
 		nd.ni_cnd.cn_context = &hacked_context;
@@ -2114,6 +2109,7 @@ nfsrv_mknod(nfsd, slp, procp, mrq)
 			if (nd.ni_cnd.cn_flags & ISSYMLINK)
 			        error = EINVAL;
 		}
+		kauth_cred_unref(&hacked_context.vc_ucred);
 	}
 out1:
 	if (xacl != NULL)
@@ -2361,9 +2357,9 @@ retry:
 
 	/* reset credential if it was remapped */
 	if (nfsd->nd_cr != saved_cred) {
-		kauth_cred_rele(nfsd->nd_cr);
+		kauth_cred_ref(saved_cred);
+		kauth_cred_unref(&nfsd->nd_cr);
 		nfsd->nd_cr = saved_cred;
-		kauth_cred_ref(nfsd->nd_cr);
 	}
 
 	tond.ni_cnd.cn_nameiop = RENAME;
@@ -2727,7 +2723,7 @@ out:
 	if (topath)
 		FREE_ZONE(topath, MAXPATHLEN, M_NAMEI);
 	if (saved_cred)
-		kauth_cred_rele(saved_cred);
+		kauth_cred_unref(&saved_cred);
 	return (0);
 
 nfsmout:
@@ -2766,7 +2762,7 @@ nfsmout:
 	if (topath)
 		FREE_ZONE(topath, MAXPATHLEN, M_NAMEI);
 	if (saved_cred)
-		kauth_cred_rele(saved_cred);
+		kauth_cred_unref(&saved_cred);
 	return (error);
 }
 
@@ -3733,6 +3729,7 @@ nfsrv_readdirplus(nfsd, slp, procp, mrq)
 	vnode_t vp, nvp;
 	struct flrep fl;
 	struct nfs_filehandle dnfh, *nfhp = (struct nfs_filehandle *)&fl.fl_fhsize;
+	u_long fhsize;
 	struct nfs_export *nx;
 	struct nfs_export_options *nxo;
 	uio_t auio;
@@ -3944,7 +3941,8 @@ again:
 			 */
 			fp = (struct nfs_fattr *)&fl.fl_fattr;
 			nfsm_srvfillattr(vap, fp);
-			fl.fl_fhsize = txdr_unsigned(nfhp->nfh_len);
+			fhsize = nfhp->nfh_len;
+			fl.fl_fhsize = txdr_unsigned(fhsize);
 			fl.fl_fhok = nfs_true;
 			fl.fl_postopok = nfs_true;
 			if (vnopflag & VNODE_READDIR_SEEKOFF32)
@@ -3989,7 +3987,7 @@ again:
 			/*
 			 * Now copy the flrep structure out.
 			 */
-			xfer = sizeof(struct flrep) - sizeof(fl.fl_nfh) + fl.fl_fhsize;
+			xfer = sizeof(struct flrep) - sizeof(fl.fl_nfh) + fhsize;
 			cp = (caddr_t)&fl;
 			while (xfer > 0) {
 				nfsm_clget;
diff --git a/bsd/nfs/nfs_socket.c b/bsd/nfs/nfs_socket.c
index f0ca1838e..1f4fda19e 100644
--- a/bsd/nfs/nfs_socket.c
+++ b/bsd/nfs/nfs_socket.c
@@ -316,7 +316,6 @@ nfs_bind_resv_nopriv(struct nfsmount *nmp)
 	if (nfs_bind_resv_thread_state < NFS_BIND_RESV_THREAD_STATE_RUNNING) {
 		if (nfs_bind_resv_thread_state < NFS_BIND_RESV_THREAD_STATE_INITTED) {
 			nfs_bind_resv_lck_grp_attr = lck_grp_attr_alloc_init();
-			lck_grp_attr_setstat(nfs_bind_resv_lck_grp_attr);
 			nfs_bind_resv_lck_grp = lck_grp_alloc_init("nfs_bind_resv", nfs_bind_resv_lck_grp_attr);
 			nfs_bind_resv_lck_attr = lck_attr_alloc_init();
 			nfs_bind_resv_mutex = lck_mtx_alloc_init(nfs_bind_resv_lck_grp, nfs_bind_resv_lck_attr);
@@ -2746,8 +2745,8 @@ nfs_getreq(nd, nfsd, has_header)
 	nd->nd_dpos = dpos;
 	return (0);
 nfsmout:
-	if (nd->nd_cr)
-		kauth_cred_rele(nd->nd_cr);
+	if (IS_VALID_CRED(nd->nd_cr))
+		kauth_cred_unref(&nd->nd_cr);
 	return (error);
 }
 
diff --git a/bsd/nfs/nfs_srvcache.c b/bsd/nfs/nfs_srvcache.c
index ea1fee651..c89c0ca98 100644
--- a/bsd/nfs/nfs_srvcache.c
+++ b/bsd/nfs/nfs_srvcache.c
@@ -170,7 +170,6 @@ nfsrv_initcache()
 {
 	/* init nfs server request cache mutex */
 	nfsrv_reqcache_lck_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(nfsrv_reqcache_lck_grp_attr);
 	nfsrv_reqcache_lck_grp = lck_grp_alloc_init("nfsrv_reqcache", nfsrv_reqcache_lck_grp_attr);
 	nfsrv_reqcache_lck_attr = lck_attr_alloc_init();
 	nfsrv_reqcache_mutex = lck_mtx_alloc_init(nfsrv_reqcache_lck_grp, nfsrv_reqcache_lck_attr);
diff --git a/bsd/nfs/nfs_subs.c b/bsd/nfs/nfs_subs.c
index 7f0e9032a..3497ef8f9 100644
--- a/bsd/nfs/nfs_subs.c
+++ b/bsd/nfs/nfs_subs.c
@@ -1220,7 +1220,6 @@ nfs_init(struct vfsconf *vfsp)
 	}
 	/* init nfsiod mutex */
 	nfs_iod_lck_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(nfs_iod_lck_grp_attr);
 	nfs_iod_lck_grp = lck_grp_alloc_init("nfs_iod", nfs_iod_lck_grp_attr);
 	nfs_iod_lck_attr = lck_attr_alloc_init();
 	nfs_iod_mutex = lck_mtx_alloc_init(nfs_iod_lck_grp, nfs_iod_lck_attr);
@@ -1232,7 +1231,6 @@ nfs_init(struct vfsconf *vfsp)
 #ifndef NFS_NOSERVER
 	/* init nfsd mutex */
 	nfsd_lck_grp_attr = lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(nfsd_lck_grp_attr);
 	nfsd_lck_grp = lck_grp_alloc_init("nfsd", nfsd_lck_grp_attr);
 	nfsd_lck_attr = lck_attr_alloc_init();
 	nfsd_mutex = lck_mtx_alloc_init(nfsd_lck_grp, nfsd_lck_attr);
@@ -2160,7 +2158,7 @@ nfsrv_hang_addrlist(struct nfs_export *nx, struct user_nfs_export_args *unxa)
 			if (!cred)
 				return (ENOMEM);
 		} else {
-			cred = NULL;
+			cred = NOCRED;
 		}
 
 		if (nxna.nxna_addr.ss_len == 0) {
@@ -2204,7 +2202,8 @@ nfsrv_hang_addrlist(struct nfs_export *nx, struct user_nfs_export_args *unxa)
 					break;
 				}
 			if ((rnh = nx->nx_rtable[i]) == 0) {
-				kauth_cred_rele(cred);
+			        if (IS_VALID_CRED(cred))
+				        kauth_cred_unref(&cred);
 				_FREE(no, M_NETADDR);
 				return (ENOBUFS);
 			}
@@ -2231,7 +2230,8 @@ nfsrv_hang_addrlist(struct nfs_export *nx, struct user_nfs_export_args *unxa)
 						matched = 1;
 				}
 			}
-			kauth_cred_rele(cred);
+			if (IS_VALID_CRED(cred))
+			        kauth_cred_unref(&cred);
 			_FREE(no, M_NETADDR);
 			if (matched)
 				continue;
@@ -2262,8 +2262,8 @@ nfsrv_free_netopt(struct radix_node *rn, void *w)
 	struct nfs_netopt *nno = (struct nfs_netopt *)rn;
 
 	(*rnh->rnh_deladdr)(rn->rn_key, rn->rn_mask, rnh);
-	if (nno->no_opt.nxo_cred)
-		kauth_cred_rele(nno->no_opt.nxo_cred);
+	if (IS_VALID_CRED(nno->no_opt.nxo_cred))
+		kauth_cred_unref(&nno->no_opt.nxo_cred);
 	_FREE((caddr_t)rn, M_NETADDR);
 	*cnt -= 1;
 	return (0);
@@ -2318,9 +2318,8 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 				/* delete all netopts for this export */
 				nfsrv_free_addrlist(nx);
 				nx->nx_flags &= ~NX_DEFAULTEXPORT;
-				if (nx->nx_defopt.nxo_cred) {
-					kauth_cred_rele(nx->nx_defopt.nxo_cred);
-					nx->nx_defopt.nxo_cred = NULL;
+				if (IS_VALID_CRED(nx->nx_defopt.nxo_cred)) {
+					kauth_cred_unref(&nx->nx_defopt.nxo_cred);
 				}
 				FREE(nx->nx_path, M_TEMP);
 				FREE(nx, M_TEMP);
@@ -2542,9 +2541,9 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 			}
 
 			/* grab file handle */
-			nx->nx_fh.nfh_xh.nxh_version = NFS_FH_VERSION;
-			nx->nx_fh.nfh_xh.nxh_fsid = nx->nx_fs->nxfs_id;
-			nx->nx_fh.nfh_xh.nxh_expid = nx->nx_id;
+			nx->nx_fh.nfh_xh.nxh_version = htonl(NFS_FH_VERSION);
+			nx->nx_fh.nfh_xh.nxh_fsid = htonl(nx->nx_fs->nxfs_id);
+			nx->nx_fh.nfh_xh.nxh_expid = htonl(nx->nx_id);
 			nx->nx_fh.nfh_xh.nxh_flags = 0;
 			nx->nx_fh.nfh_xh.nxh_reserved = 0;
 			nx->nx_fh.nfh_len = NFS_MAX_FID_SIZE;
@@ -2574,9 +2573,8 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 				/* delete all netopts for this export */
 				nfsrv_free_addrlist(nx);
 				nx->nx_flags &= ~NX_DEFAULTEXPORT;
-				if (nx->nx_defopt.nxo_cred) {
-					kauth_cred_rele(nx->nx_defopt.nxo_cred);
-					nx->nx_defopt.nxo_cred = NULL;
+				if (IS_VALID_CRED(nx->nx_defopt.nxo_cred)) {
+					kauth_cred_unref(&nx->nx_defopt.nxo_cred);
 				}
 				FREE(nx->nx_path, M_TEMP);
 				FREE(nx, M_TEMP);
@@ -2586,9 +2584,8 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 			/* delete all netopts for this export */
 			nfsrv_free_addrlist(nx);
 			nx->nx_flags &= ~NX_DEFAULTEXPORT;
-			if (nx->nx_defopt.nxo_cred) {
-				kauth_cred_rele(nx->nx_defopt.nxo_cred);
-				nx->nx_defopt.nxo_cred = NULL;
+			if (IS_VALID_CRED(nx->nx_defopt.nxo_cred)) {
+				kauth_cred_unref(&nx->nx_defopt.nxo_cred);
 			}
 		}
 	}
@@ -2656,11 +2653,15 @@ static struct nfs_export *
 nfsrv_fhtoexport(struct nfs_filehandle *nfhp)
 {
 	struct nfs_export *nx;
-	nx = NFSEXPHASH(nfhp->nfh_xh.nxh_fsid, nfhp->nfh_xh.nxh_expid)->lh_first;
+	uint32_t fsid, expid;
+
+	fsid = ntohl(nfhp->nfh_xh.nxh_fsid);
+	expid = ntohl(nfhp->nfh_xh.nxh_expid);
+	nx = NFSEXPHASH(fsid, expid)->lh_first;
 	for (; nx; nx = LIST_NEXT(nx, nx_hash)) {
-		if (nx->nx_fs->nxfs_id != nfhp->nfh_xh.nxh_fsid)
+		if (nx->nx_fs->nxfs_id != fsid)
 			continue;
-		if (nx->nx_id != nfhp->nfh_xh.nxh_expid)
+		if (nx->nx_id != expid)
 			continue;
 		break;
 	}
@@ -2681,12 +2682,14 @@ nfsrv_fhtovp(
 {
 	int error;
 	struct mount *mp;
+	uint32_t v;
 
 	*vpp = NULL;
 	*nxp = NULL;
 	*nxop = NULL;
 
-	if (nfhp->nfh_xh.nxh_version != NFS_FH_VERSION) {
+	v = ntohl(nfhp->nfh_xh.nxh_version);
+	if (v != NFS_FH_VERSION) {
 		/* file handle format not supported */
 		return (ESTALE);
 	}
@@ -2694,7 +2697,8 @@ nfsrv_fhtovp(
 		return (EBADRPC);
 	if (nfhp->nfh_len < (int)sizeof(nfhp->nfh_xh))
 		return (ESTALE);
-	if (nfhp->nfh_xh.nxh_flags & NXHF_INVALIDFH)
+	v = ntohs(nfhp->nfh_xh.nxh_flags);
+	if (v & NXHF_INVALIDFH)
 		return (ESTALE);
 
 /* XXX Revisit when enabling WebNFS */
@@ -2741,9 +2745,9 @@ nfsrv_credcheck(
 	if (nxo && nxo->nxo_cred) {
 		if ((nxo->nxo_flags & NX_MAPALL) ||
 		    ((nxo->nxo_flags & NX_MAPROOT) && !suser(nfsd->nd_cr, NULL))) {
-			kauth_cred_rele(nfsd->nd_cr);
+			kauth_cred_ref(nxo->nxo_cred);
+			kauth_cred_unref(&nfsd->nd_cr);
 			nfsd->nd_cr = nxo->nxo_cred;
-			kauth_cred_ref(nfsd->nd_cr);
 		}
 	}
 	return (0);
@@ -2790,9 +2794,9 @@ nfsrv_vptofh(
 {
 	int error;
 
-	nfhp->nfh_xh.nxh_version = NFS_FH_VERSION;
-	nfhp->nfh_xh.nxh_fsid = nx->nx_fs->nxfs_id;
-	nfhp->nfh_xh.nxh_expid = nx->nx_id;
+	nfhp->nfh_xh.nxh_version = htonl(NFS_FH_VERSION);
+	nfhp->nfh_xh.nxh_fsid = htonl(nx->nx_fs->nxfs_id);
+	nfhp->nfh_xh.nxh_expid = htonl(nx->nx_id);
 	nfhp->nfh_xh.nxh_flags = 0;
 	nfhp->nfh_xh.nxh_reserved = 0;
 
@@ -2803,7 +2807,7 @@ nfsrv_vptofh(
 	if (dnfhp && nfsrv_fhmatch(dnfhp, &nx->nx_fh)) {
 		nfhp->nfh_len = v2 ? NFSX_V2FH : sizeof(nfhp->nfh_xh);
 		nfhp->nfh_xh.nxh_fidlen = 0;
-		nfhp->nfh_xh.nxh_flags = NXHF_INVALIDFH;
+		nfhp->nfh_xh.nxh_flags = htons(NXHF_INVALIDFH);
 		return (0);
 	}
 
diff --git a/bsd/nfs/nfs_syscalls.c b/bsd/nfs/nfs_syscalls.c
index 3d6015e64..dd1df6eb4 100644
--- a/bsd/nfs/nfs_syscalls.c
+++ b/bsd/nfs/nfs_syscalls.c
@@ -226,9 +226,9 @@ getfh(proc_t p, struct getfh_args *uap, __unused int *retval)
 	}
 
 	bzero(&nfh, sizeof(nfh));
-	nfh.nfh_xh.nxh_version = NFS_FH_VERSION;
-	nfh.nfh_xh.nxh_fsid = nxfs->nxfs_id;
-	nfh.nfh_xh.nxh_expid = nx->nx_id;
+	nfh.nfh_xh.nxh_version = htonl(NFS_FH_VERSION);
+	nfh.nfh_xh.nxh_fsid = htonl(nxfs->nxfs_id);
+	nfh.nfh_xh.nxh_expid = htonl(nx->nx_id);
 	nfh.nfh_xh.nxh_flags = 0;
 	nfh.nfh_xh.nxh_reserved = 0;
 	nfh.nfh_len = NFS_MAX_FID_SIZE;
@@ -271,41 +271,52 @@ fhopen( proc_t p,
 	struct fileproc *fp, *nfp;
 	int fmode, error, type;
 	int indx;
-	kauth_cred_t cred = proc_ucred(p);
 	struct vfs_context context;
 	kauth_action_t action;
 
 	context.vc_proc = p;
-	context.vc_ucred = cred;
+	context.vc_ucred = kauth_cred_proc_ref(p);
 
 	/*
 	 * Must be super user
 	 */
-	error = suser(cred, 0);
-	if (error)
+	error = suser(context.vc_ucred, 0);
+	if (error) {
+		kauth_cred_unref(&context.vc_ucred);
 		return (error);
+	}
 
 	fmode = FFLAGS(uap->flags);
 	/* why not allow a non-read/write open for our lockd? */
-	if (((fmode & (FREAD | FWRITE)) == 0) || (fmode & O_CREAT))
+	if (((fmode & (FREAD | FWRITE)) == 0) || (fmode & O_CREAT)) {
+		kauth_cred_unref(&context.vc_ucred);
 		return (EINVAL);
+	}
 
 	error = copyin(uap->u_fhp, &nfh.nfh_len, sizeof(nfh.nfh_len));
-	if (error)
+	if (error) {
+		kauth_cred_unref(&context.vc_ucred);
 		return (error);
+	}
 	if ((nfh.nfh_len < (int)sizeof(struct nfs_exphandle)) ||
-	    (nfh.nfh_len > (int)NFS_MAX_FH_SIZE))
+	    (nfh.nfh_len > (int)NFS_MAX_FH_SIZE)) {
+		kauth_cred_unref(&context.vc_ucred);
 		return (EINVAL);
+	}
 	error = copyin(uap->u_fhp, &nfh, sizeof(nfh.nfh_len) + nfh.nfh_len);
-	if (error)
+	if (error) {
+		kauth_cred_unref(&context.vc_ucred);
 		return (error);
+	}
 
 	lck_rw_lock_shared(&nfs_export_rwlock);
 	/* now give me my vnode, it gets returned to me with a reference */
 	error = nfsrv_fhtovp(&nfh, NULL, TRUE, &vp, &nx, &nxo);
 	lck_rw_done(&nfs_export_rwlock);
-	if (error)
+	if (error) {
+		kauth_cred_unref(&context.vc_ucred);
 		return (error);
+	}
 
 	/*
 	 * From now on we have to make sure not
@@ -348,7 +359,7 @@ fhopen( proc_t p,
 
 	// starting here... error paths should call vn_close/vnode_put
 	if ((error = falloc(p, &nfp, &indx)) != 0) {
-		vn_close(vp, fmode & FMASK, cred, p);
+		vn_close(vp, fmode & FMASK, context.vc_ucred, p);
 		goto bad;
 	}
 	fp = nfp;
@@ -373,6 +384,7 @@ fhopen( proc_t p,
 		if ((error = VNOP_ADVLOCK(vp, (caddr_t)fp->f_fglob, F_SETLK, &lf, type, &context))) {
 			vn_close(vp, fp->f_fglob->fg_flag, fp->f_fglob->fg_cred, p);
 			fp_free(p, indx, fp);
+			kauth_cred_unref(&context.vc_ucred);
 			return (error);
 		}
 		fp->f_fglob->fg_flag |= FHASLOCK;
@@ -386,10 +398,12 @@ fhopen( proc_t p,
 	proc_fdunlock(p);
 
 	*retval = indx;
+	kauth_cred_unref(&context.vc_ucred);
 	return (0);
 
 bad:
 	vnode_put(vp);
+	kauth_cred_unref(&context.vc_ucred);
 	return (error);
 }
 
@@ -522,7 +536,9 @@ nfssvc(proc_t p, struct nfssvc_args *uap, __unused int *retval)
 					break;
 			}
 			if (nuidp) {
-			    nfsrv_setcred(nuidp->nu_cr,nfsd->nfsd_nd->nd_cr);
+			    nfsrv_setcred(nuidp->nu_cr,&temp_cred);
+			    kauth_cred_unref(&nfsd->nfsd_nd->nd_cr);
+			    nfsd->nfsd_nd->nd_cr = kauth_cred_create(&temp_cred);
 			    nfsd->nfsd_nd->nd_flag |= ND_KERBFULL;
 			} else {
 			    /*
@@ -551,7 +567,7 @@ nfssvc(proc_t p, struct nfssvc_args *uap, __unused int *retval)
 					nu_lru);
 				    if (nuidp->nu_flag & NU_NAM)
 					mbuf_freem(nuidp->nu_nam);
-				    kauth_cred_rele(nuidp->nu_cr);
+				    kauth_cred_unref(&nuidp->nu_cr);
 				}
 				nuidp->nu_flag = 0;
 
@@ -591,7 +607,7 @@ nfssvc(proc_t p, struct nfssvc_args *uap, __unused int *retval)
 							MBUF_COPYALL, MBUF_WAITOK,
 							&nuidp->nu_nam);
 					if (error) {
-						kauth_cred_rele(nuidp->nu_cr);
+						kauth_cred_unref(&nuidp->nu_cr);
 						FREE_ZONE(nuidp, sizeof(struct nfsuid), M_NFSUID);
 						slp->ns_numuids--;
 						return (error);
@@ -603,8 +619,9 @@ nfssvc(proc_t p, struct nfssvc_args *uap, __unused int *retval)
 					nu_lru);
 				LIST_INSERT_HEAD(NUIDHASH(slp, nsd->nsd_uid),
 					nuidp, nu_hash);
-				nfsrv_setcred(nuidp->nu_cr,
-				    nfsd->nfsd_nd->nd_cr);
+				nfsrv_setcred(nuidp->nu_cr,&temp_cred);
+				kauth_cred_unref(&nfsd->nfsd_nd->nd_cr);
+				nfsd->nfsd_nd->nd_cr = kauth_cred_create(&temp_cred);
 				nfsd->nfsd_nd->nd_flag |= ND_KERBFULL;
 			    }
 			}
@@ -691,7 +708,7 @@ nfskerb_clientd(
 		    error = tsleep((caddr_t)&nmp->nm_authstr, PSOCK | PCATCH,
 			"nfskrbtimr", hz / 3);
 		    if (error == EINTR || error == ERESTART)
-			    dounmount(nmp->nm_mountp, 0, p);
+			    dounmount(nmp->nm_mountp, 0, NULL, p);
 	    }
 	}
 
@@ -702,7 +719,7 @@ nfskerb_clientd(
 		nnuidp = nuidp->nu_lru.tqe_next;
 		LIST_REMOVE(nuidp, nu_hash);
 		TAILQ_REMOVE(&nmp->nm_uidlruhead, nuidp, nu_lru);
-		kauth_cred_rele(nuidp->nu_cr);
+		kauth_cred_unref(&nuidp->nu_cr);
 		FREE_ZONE((caddr_t)nuidp, sizeof (struct nfsuid), M_NFSUID);
 	}
 	/*
@@ -945,8 +962,8 @@ nfssvc_nfsd(nsd, argp, p)
 			if (nd) {
 				if (nd->nd_nam2)
 					mbuf_freem(nd->nd_nam2);
-				if (nd->nd_cr)
-					kauth_cred_rele(nd->nd_cr);
+				if (IS_VALID_CRED(nd->nd_cr))
+					kauth_cred_unref(&nd->nd_cr);
 				FREE_ZONE((caddr_t)nd,
 						sizeof *nd, M_NFSRVDESC);
 				nd = NULL;
@@ -1093,8 +1110,8 @@ nfssvc_nfsd(nsd, argp, p)
 				lck_rw_done(&slp->ns_rwlock);
 			}
 			if (error == EINTR || error == ERESTART) {
-				if (nd->nd_cr)
-					kauth_cred_rele(nd->nd_cr);
+				if (IS_VALID_CRED(nd->nd_cr))
+					kauth_cred_unref(&nd->nd_cr);
 				FREE_ZONE((caddr_t)nd, sizeof *nd, M_NFSRVDESC);
 				nfsrv_slpderef(slp);
 				goto done;
@@ -1113,8 +1130,8 @@ nfssvc_nfsd(nsd, argp, p)
 				mbuf_freem(nd->nd_mrep);
 			if (nd->nd_nam2)
 				mbuf_freem(nd->nd_nam2);
-			if (nd->nd_cr)
-				kauth_cred_rele(nd->nd_cr);
+			if (IS_VALID_CRED(nd->nd_cr))
+				kauth_cred_unref(&nd->nd_cr);
 			FREE_ZONE((caddr_t)nd, sizeof *nd, M_NFSRVDESC);
 			nd = NULL;
 		    }
@@ -1624,7 +1641,7 @@ nfs_savenickauth(nmp, cred, len, key, mdp, dposp, mrep)
 				}
 				LIST_REMOVE(nuidp, nu_hash);
 				TAILQ_REMOVE(&nmp->nm_uidlruhead, nuidp, nu_lru);
-				kauth_cred_rele(nuidp->nu_cr);
+				kauth_cred_unref(&nuidp->nu_cr);
 			}
 			nuidp->nu_flag = 0;
 			kauth_cred_ref(cred);
@@ -1675,7 +1692,7 @@ nfsrv_slpfree(struct nfssvc_sock *slp)
 		TAILQ_REMOVE(&slp->ns_uidlruhead, nuidp, nu_lru);
 		if (nuidp->nu_flag & NU_NAM)
 			mbuf_freem(nuidp->nu_nam);
-		kauth_cred_rele(nuidp->nu_cr);
+		kauth_cred_unref(&nuidp->nu_cr);
 		FREE_ZONE((caddr_t)nuidp,
 				sizeof (struct nfsuid), M_NFSUID);
 	}
@@ -1683,8 +1700,8 @@ nfsrv_slpfree(struct nfssvc_sock *slp)
 	for (nwp = slp->ns_tq.lh_first; nwp; nwp = nnwp) {
 		nnwp = nwp->nd_tq.le_next;
 		LIST_REMOVE(nwp, nd_tq);
-		if (nwp->nd_cr)
-			kauth_cred_rele(nwp->nd_cr);
+		if (IS_VALID_CRED(nwp->nd_cr))
+			kauth_cred_unref(&nwp->nd_cr);
 		FREE_ZONE((caddr_t)nwp, sizeof *nwp, M_NFSRVDESC);
 	}
 	LIST_INIT(&slp->ns_tq);
diff --git a/bsd/nfs/nfs_vfsops.c b/bsd/nfs/nfs_vfsops.c
index 7c281f13b..847af5818 100644
--- a/bsd/nfs/nfs_vfsops.c
+++ b/bsd/nfs/nfs_vfsops.c
@@ -131,7 +131,7 @@ static int nfs_tprintf_delay = NFS_TPRINTF_DELAY;
 SYSCTL_INT(_vfs_generic_nfs_client, NFS_TPRINTF_DELAY,
     nextdowndelay, CTLFLAG_RW, &nfs_tprintf_delay, 0, "");
 
-static int	nfs_iosize(struct nfsmount *nmp);
+static int	nfs_biosize(struct nfsmount *);
 static int	mountnfs(struct user_nfs_args *,mount_t,mbuf_t,proc_t,vnode_t *);
 static int	nfs_mount(mount_t mp, vnode_t vp, user_addr_t data, vfs_context_t context);
 static int	nfs_start(mount_t mp, int flags, vfs_context_t context);
@@ -172,8 +172,8 @@ static int
 nfs_mount_diskless_private(struct nfs_dlmount *, const char *, int, vnode_t *, mount_t *);
 #endif /* NO_MOUNT_PRIVATE */
 
-static int nfs_iosize(nmp)
-	struct nfsmount* nmp;
+static int
+nfs_biosize(struct nfsmount *nmp)
 {
 	int iosize;
 
@@ -227,7 +227,7 @@ nfs_statfs(mount_t mp, struct vfsstatfs *sbp, vfs_context_t context)
 		nfs_fsinfo(nmp, vp, cred, p);
 	nfsm_reqhead(NFSX_FH(v3));
 	if (error) {
-		kauth_cred_rele(cred);
+		kauth_cred_unref(&cred);
 		vnode_put(vp);
 		return (error);
 	}
@@ -239,7 +239,7 @@ nfs_statfs(mount_t mp, struct vfsstatfs *sbp, vfs_context_t context)
 	nfsm_dissect(sfp, struct nfs_statfs *, NFSX_STATFS(v3));
 
 	sbp->f_flags = nmp->nm_flag;
-	sbp->f_iosize = nfs_iosize(nmp);
+	sbp->f_iosize = NFS_IOSIZE;
 	if (v3) {
 		/*
 		 * Adjust block size to get total block count to fit in a long.
@@ -284,7 +284,7 @@ nfs_statfs(mount_t mp, struct vfsstatfs *sbp, vfs_context_t context)
 		sbp->f_ffree = 0;
 	}
 	nfsm_reqdone;
- 	kauth_cred_rele(cred);
+ 	kauth_cred_unref(&cred);
 	vnode_put(vp);
 	return (error);
 }
@@ -1034,6 +1034,9 @@ mountnfs(
 	struct vfs_context context; /* XXX get from caller? */
 	u_int64_t xid;
 
+	/* up front because of reference */
+	context.vc_ucred = kauth_cred_proc_ref(p);
+
 	/*
 	 * Silently clear NFSMNT_NOCONN if it's a TCP mount, it makes
 	 * no sense in that context.
@@ -1045,12 +1048,14 @@ mountnfs(
 		nmp = VFSTONFS(mp);
 		/* update paths, file handles, etc, here	XXX */
 		mbuf_freem(nam);
+		kauth_cred_unref(&context.vc_ucred);
 		return (0);
 	} else {
 		MALLOC_ZONE(nmp, struct nfsmount *,
 				sizeof (struct nfsmount), M_NFSMNT, M_WAITOK);
 		if (!nmp) {
 			mbuf_freem(nam);
+			kauth_cred_unref(&context.vc_ucred);
 			return (ENOMEM);
 		}
 		bzero((caddr_t)nmp, sizeof (struct nfsmount));
@@ -1120,8 +1125,8 @@ mountnfs(
 	}
 	if (nmp->nm_wsize > maxio)
 		nmp->nm_wsize = maxio;
-	if (nmp->nm_wsize > MAXBSIZE)
-		nmp->nm_wsize = MAXBSIZE;
+	if (nmp->nm_wsize > NFS_MAXBSIZE)
+		nmp->nm_wsize = NFS_MAXBSIZE;
 
 	if ((argp->flags & NFSMNT_RSIZE) && argp->rsize > 0) {
 		nmp->nm_rsize = argp->rsize;
@@ -1132,8 +1137,8 @@ mountnfs(
 	}
 	if (nmp->nm_rsize > maxio)
 		nmp->nm_rsize = maxio;
-	if (nmp->nm_rsize > MAXBSIZE)
-		nmp->nm_rsize = MAXBSIZE;
+	if (nmp->nm_rsize > NFS_MAXBSIZE)
+		nmp->nm_rsize = NFS_MAXBSIZE;
 
 	if ((argp->flags & NFSMNT_READDIRSIZE) && argp->readdirsize > 0) {
 		nmp->nm_readdirsize = argp->readdirsize;
@@ -1187,7 +1192,7 @@ mountnfs(
 	 */
 	// LP64todo - fix CAST_DOWN of argp->fh
 	error = nfs_getattr_no_vnode(mp, CAST_DOWN(caddr_t, argp->fh), argp->fhsize,
-			proc_ucred(p), p, &nvattrs, &xid);
+			context.vc_ucred, p, &nvattrs, &xid);
 	if (error) {
 		/*
 		 * we got problems... we couldn't get the attributes
@@ -1229,8 +1234,9 @@ mountnfs(
 	 * the server about what its preferred I/O sizes are.
 	 */
 	if (nmp->nm_flag & NFSMNT_NFSV3)
-		nfs_fsinfo(nmp, *vpp, proc_ucred(p), p);
-	vfs_statfs(mp)->f_iosize = nfs_iosize(nmp);
+		nfs_fsinfo(nmp, *vpp, context.vc_ucred, p);
+	nmp->nm_biosize = nfs_biosize(nmp);
+	vfs_statfs(mp)->f_iosize = NFS_IOSIZE;
 
 	/*
 	 * V3 mounts give us a (relatively) reliable remote access(2)
@@ -1246,17 +1252,18 @@ mountnfs(
 	 * Do statfs to ensure static info gets set to reasonable values.
 	 */
 	context.vc_proc = p;
-	context.vc_ucred = proc_ucred(p);
 	nfs_statfs(mp, vfs_statfs(mp), &context);
 
 	if (nmp->nm_flag & NFSMNT_RESVPORT)
 		nfs_resv_mounts++;
 	nmp->nm_state |= NFSSTA_MOUNTED;
+	kauth_cred_unref(&context.vc_ucred);
 	return (0);
 bad:
 	nfs_disconnect(nmp);
 	FREE_ZONE((caddr_t)nmp, sizeof (struct nfsmount), M_NFSMNT);
 	mbuf_freem(nam);
+	kauth_cred_unref(&context.vc_ucred);
 	return (error);
 }
 
diff --git a/bsd/nfs/nfs_vnops.c b/bsd/nfs/nfs_vnops.c
index 5488aa9d6..c665337ca 100644
--- a/bsd/nfs/nfs_vnops.c
+++ b/bsd/nfs/nfs_vnops.c
@@ -1037,8 +1037,10 @@ nfs_setattr(ap)
 {
 	vnode_t vp = ap->a_vp;
 	struct nfsnode *np = VTONFS(vp);
+	struct nfsmount *nmp;
 	struct vnode_attr *vap = ap->a_vap;
 	int error = 0;
+	int biosize;
 	u_quad_t tsize;
 	kauth_cred_t cred;
 	proc_t p;
@@ -1046,6 +1048,10 @@ nfs_setattr(ap)
 #ifndef nolint
 	tsize = (u_quad_t)0;
 #endif
+	nmp = VFSTONFS(vnode_mount(vp));
+	if (!nmp)
+		return (ENXIO);
+	biosize = nmp->nm_biosize;
 
 	/* Setting of flags is not supported. */
 	if (VATTR_IS_ACTIVE(vap, va_flags))
@@ -1107,10 +1113,9 @@ nfs_setattr(ap)
 				}
 			} else if (np->n_size > vap->va_data_size) { /* shrinking? */
 				daddr64_t obn, bn;
-				int biosize, neweofoff, mustwrite;
+				int neweofoff, mustwrite;
 				struct nfsbuf *bp;
 
-				biosize = vfs_statfs(vnode_mount(vp))->f_iosize;
 				obn = (np->n_size - 1) / biosize;
 				bn = vap->va_data_size / biosize; 
 				for ( ; obn >= bn; obn--) {
@@ -1148,7 +1153,7 @@ nfs_setattr(ap)
 					/* (NB_NOCACHE indicates buffer should be discarded) */
 					CLR(bp->nb_flags, (NB_DONE | NB_ERROR | NB_INVAL | NB_ASYNC | NB_READ));
 					SET(bp->nb_flags, NB_STABLE | NB_NOCACHE);
-					if (bp->nb_wcred == NOCRED) {
+					if (!IS_VALID_CRED(bp->nb_wcred)) {
 						kauth_cred_ref(cred);
 						bp->nb_wcred = cred;
 					}
@@ -3549,9 +3554,7 @@ nfs_sillyrename(
 bad:
 	vnode_rele(sp->s_dvp);
 bad_norele:
-	tmpcred = sp->s_cred;
-	sp->s_cred = NOCRED;
-	kauth_cred_rele(tmpcred);
+	kauth_cred_unref(&sp->s_cred);
 	FREE_ZONE((caddr_t)sp, sizeof (struct sillyrename), M_NFSREQ);
 	return (error);
 }
@@ -3895,7 +3898,7 @@ nfs_flushcommits(vnode_t vp, proc_t p, int nowait)
 			 */
 			if (wcred_set == 0) {
 				wcred = bp->nb_wcred;
-				if (wcred == NOCRED)
+				if (!IS_VALID_CRED(wcred))
 					panic("nfs: needcommit w/out wcred");
 				wcred_set = 1;
 			} else if ((wcred_set == 1) && wcred != bp->nb_wcred) {
@@ -4408,7 +4411,7 @@ nfs_buf_write(struct nfsbuf *bp)
 		}
 		oldflags = bp->nb_flags;
 		FSDBG_BOT(553, bp, NBOFF(bp), bp->nb_flags, rv);
-		if (cr) {
+		if (IS_VALID_CRED(cr)) {
 			kauth_cred_ref(cr);
 		}
 		nfs_buf_release(bp, 1);
@@ -4426,8 +4429,8 @@ nfs_buf_write(struct nfsbuf *bp)
 			 */
 			nfs_vinvalbuf(vp, V_SAVE|V_IGNORE_WRITEERR, cr, p, 1);
 		}
-		if (cr)
-			kauth_cred_rele(cr);
+		if (IS_VALID_CRED(cr))
+			kauth_cred_unref(&cr);
 		return (rv);
 	} 
 
@@ -4729,7 +4732,7 @@ nfs_pagein(ap)
 	}
 
 	cred = ubc_getcred(vp);
-	if (cred == NOCRED)
+	if (!IS_VALID_CRED(cred))
 		cred = vfs_context_ucred(ap->a_context);
 	p = vfs_context_proc(ap->a_context);
 
@@ -4749,9 +4752,9 @@ nfs_pagein(ap)
 				UPL_ABORT_ERROR | UPL_ABORT_FREE_ON_EMPTY);
 		return (ENXIO);
 	}
+	biosize = nmp->nm_biosize;
 	if ((nmp->nm_flag & NFSMNT_NFSV3) && !(nmp->nm_state & NFSSTA_GOTFSINFO))
-		(void)nfs_fsinfo(nmp, vp, cred, p);
-	biosize = vfs_statfs(vnode_mount(vp))->f_iosize;
+		nfs_fsinfo(nmp, vp, cred, p);
 
 	plinfo = ubc_upl_pageinfo(pl);
 	ubc_upl_map(pl, &ioaddr);
@@ -4888,7 +4891,7 @@ nfs_pageout(ap)
 			ubc_upl_abort(pl, UPL_ABORT_DUMP_PAGES|UPL_ABORT_FREE_ON_EMPTY);
 		return (ENXIO);
 	}
-	biosize = vfs_statfs(vnode_mount(vp))->f_iosize;
+	biosize = nmp->nm_biosize;
 
 	/*
 	 * Check to see whether the buffer is incore.
@@ -4975,7 +4978,7 @@ nfs_pageout(ap)
 	}
 
 	cred = ubc_getcred(vp);
-	if (cred == NOCRED)
+	if (!IS_VALID_CRED(cred))
 		cred = vfs_context_ucred(ap->a_context);
 	p = vfs_context_proc(ap->a_context);
 
@@ -5143,12 +5146,11 @@ nfs_blktooff(ap)
 {
 	int biosize;
 	vnode_t vp = ap->a_vp;
-	mount_t mp = vnode_mount(vp);
+	struct nfsmount *nmp = VFSTONFS(vnode_mount(vp));
 
-	if (!mp)
+	if (!nmp)
 		return (ENXIO);
-
-	biosize = vfs_statfs(mp)->f_iosize;
+	biosize = nmp->nm_biosize;
 
 	*ap->a_offset = (off_t)(ap->a_lblkno * biosize);
 
@@ -5166,12 +5168,11 @@ nfs_offtoblk(ap)
 {
 	int biosize;
 	vnode_t vp = ap->a_vp;
-	mount_t mp = vnode_mount(vp);
+	struct nfsmount *nmp = VFSTONFS(vnode_mount(vp));
 
-	if (!mp)
+	if (!nmp)
 		return (ENXIO);
-
-	biosize = vfs_statfs(mp)->f_iosize;
+	biosize = nmp->nm_biosize;
 
 	*ap->a_lblkno = (daddr64_t)(ap->a_offset / biosize);
 
diff --git a/bsd/nfs/nfsmount.h b/bsd/nfs/nfsmount.h
index 0406d1b48..60db34d68 100644
--- a/bsd/nfs/nfsmount.h
+++ b/bsd/nfs/nfsmount.h
@@ -96,6 +96,7 @@ struct	nfsmount {
 	int	nm_timeouts;		/* Request timeouts */
 	int	nm_rsize;		/* Max size of read rpc */
 	int	nm_wsize;		/* Max size of write rpc */
+	int	nm_biosize;		/* buffer I/O size */
 	int	nm_readdirsize;		/* Size of a readdir rpc */
 	int	nm_readahead;		/* Num. of blocks to readahead */
 	int	nm_acregmin;		/* reg file min attr cache timeout */
diff --git a/bsd/nfs/nfsnode.h b/bsd/nfs/nfsnode.h
index 682d15bc3..a2da9b509 100644
--- a/bsd/nfs/nfsnode.h
+++ b/bsd/nfs/nfsnode.h
@@ -132,6 +132,8 @@ struct nfsbuf {
 	void *			nb_pagelist;	/* upl */
 };
 
+#define NFS_MAXBSIZE	(32 * PAGE_SIZE)	/* valid/dirty page masks limit buffer size */
+
 /*
  * These flags are kept in b_lflags... 
  * nfs_buf_mutex must be held before examining/updating
diff --git a/bsd/ppc/_types.h b/bsd/ppc/_types.h
index fe9fad4d1..9065c0b1d 100644
--- a/bsd/ppc/_types.h
+++ b/bsd/ppc/_types.h
@@ -39,7 +39,7 @@ typedef __signed char		__int8_t;
 typedef char			__int8_t;
 #endif	/* !__GNUC__ */
 typedef unsigned char		__uint8_t;
-typedef	unsigned short		__int16_t;
+typedef	short			__int16_t;
 typedef	unsigned short		__uint16_t;
 typedef int			__int32_t;
 typedef unsigned int		__uint32_t;
diff --git a/bsd/ppc/types.h b/bsd/ppc/types.h
index 0c542206b..69cb4c7d7 100644
--- a/bsd/ppc/types.h
+++ b/bsd/ppc/types.h
@@ -124,6 +124,9 @@ typedef int64_t		user_time_t;
 #define USER_ADDR_NULL	((user_addr_t) 0)
 #define CAST_USER_ADDR_T(a_ptr)   ((user_addr_t)((uintptr_t)(a_ptr)))
 
+/* This defines the size of syscall arguments after copying into the kernel: */
+typedef u_int64_t	syscall_arg_t;
+
 #ifndef __offsetof
 #define __offsetof(type, field) ((size_t)(&((type *)0)->field))
 #endif
diff --git a/bsd/sys/Makefile b/bsd/sys/Makefile
index 946ec4094..5e10aaed1 100644
--- a/bsd/sys/Makefile
+++ b/bsd/sys/Makefile
@@ -36,7 +36,7 @@ DATAFILES = \
 	ttydefaults.h ttydev.h  types.h ubc.h ucontext.h ucred.h uio.h un.h unistd.h unpcb.h \
 	user.h utfconv.h utsname.h vadvise.h vcmd.h  version.h \
 	vm.h vmmeter.h vmparam.h vnioctl.h vnode.h vnode_if.h vstat.h wait.h xattr.h \
-	_types.h _endian.h domain.h protosw.h
+	_types.h _endian.h domain.h protosw.h 
 
 # Only in the framework PrivateHeader area
 PRIVATE_DATAFILES = \
@@ -46,6 +46,7 @@ PRIVATE_DATAFILES = \
 	shm_internal.h \
 	ux_exception.h \
 	ktrace.h \
+	proc_info.h \
 	vnioctl.h
 
 # KERNELFILES will appear only in the kernel framework
@@ -93,7 +94,7 @@ INSTALL_MI_LIST	= ${DATAFILES}
 
 INSTALL_MI_DIR = sys
 
-EXPORT_MI_LIST	= ${KERNELFILES} ${PRIVATE_KERNELFILES} syscall.h ktrace.h linker_set.h
+EXPORT_MI_LIST	= ${KERNELFILES} ${PRIVATE_KERNELFILES} syscall.h ktrace.h linker_set.h bsdtask_info.h
 
 EXPORT_MI_DIR = sys
 
diff --git a/bsd/sys/_endian.h b/bsd/sys/_endian.h
index 3aa34f4c8..67c8fc0c3 100644
--- a/bsd/sys/_endian.h
+++ b/bsd/sys/_endian.h
@@ -95,25 +95,10 @@
 /*
  * Macros for network/external number representation conversion.
  */
-#if __DARWIN_BYTE_ORDER == __DARWIN_BIG_ENDIAN && !defined(lint)
-#define	ntohl(x)	(x)
-#define	ntohs(x)	(x)
-#define	htonl(x)	(x)
-#define	htons(x)	(x)
-
-#if	defined(KERNEL) || !defined(_POSIX_C_SOURCE)
-#define	NTOHL(x)	(x)
-#define	NTOHS(x)	(x)
-#define	HTONL(x)	(x)
-#define	HTONS(x)	(x)
-#endif /* defined(KERNEL) || !defined(_POSIX_C_SOURCE) */
-
-#else
-
 #if !defined(__ASSEMBLER__) 
 
 #include <stdint.h>
-#include <machine/byte_order.h>
+#include <libkern/OSByteOrder.h>
  
 __BEGIN_DECLS
 uint16_t	ntohs(uint16_t);
@@ -123,16 +108,11 @@ uint32_t	htonl(uint32_t);
 __END_DECLS
 #endif /* !defined(__ASSEMBLER__) */
 
-#define ntohs(x)	NXSwapBigShortToHost(x)
-#define htons(x)	NXSwapHostShortToBig(x)
+#define ntohs(x)	OSSwapBigToHostInt16(x)
+#define htons(x)	OSSwapHostToBigInt16(x)
 
-#if defined(__LP64__)
-#define ntohl(x)	NXSwapBigIntToHost(x)
-#define htonl(x)	NXSwapHostIntToBig(x)
-#else 
-#define ntohl(x)	NXSwapBigLongToHost(x)
-#define htonl(x)	NXSwapHostLongToBig(x)
-#endif /* defined(__LP64__) */
+#define ntohl(x)	OSSwapBigToHostInt32(x)
+#define htonl(x)	OSSwapHostToBigInt32(x)
 
 #if	defined(KERNEL) || !defined(_POSIX_C_SOURCE)
 #define	NTOHL(x)	(x) = ntohl((u_long)x)
@@ -140,5 +120,4 @@ __END_DECLS
 #define	HTONL(x)	(x) = htonl((u_long)x)
 #define	HTONS(x)	(x) = htons((u_short)x)
 #endif /* defined(KERNEL) || !defined(_POSIX_C_SOURCE) */
-#endif /* __DARWIN_BYTE_ORDER != __DARWIN_BIG_ENDIAN || defined(lint) */ 
 #endif /* !_SYS__ENDIAN_H_ */
diff --git a/bsd/sys/aio.h b/bsd/sys/aio.h
index 90fdbe6e2..c28521f84 100644
--- a/bsd/sys/aio.h
+++ b/bsd/sys/aio.h
@@ -53,24 +53,16 @@ struct aiocb {
 // LP64todo - should this move?
 #ifdef KERNEL
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_aiocb {
 	int					aio_fildes;		/* File descriptor */
 	off_t				aio_offset;		/* File offset */
-	user_addr_t			aio_buf;		/* Location of buffer */
+	user_addr_t			aio_buf __attribute((aligned(8)));		/* Location of buffer */
 	user_size_t			aio_nbytes;		/* Length of transfer */
 	int					aio_reqprio;	/* Request priority offset */
-	struct user_sigevent aio_sigevent;	/* Signal number and value */
+	struct user_sigevent aio_sigevent __attribute((aligned(8)));	/* Signal number and value */
 	int					aio_lio_opcode;	/* Operation to be performed */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif // KERNEL
 
 /*
diff --git a/bsd/sys/attr.h b/bsd/sys/attr.h
index 1bcb5212e..b960adf5a 100644
--- a/bsd/sys/attr.h
+++ b/bsd/sys/attr.h
@@ -384,27 +384,19 @@ struct fssearchblock {
  */
 // LP64todo - should this move?
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_fssearchblock {
 	user_addr_t         returnattrs;
 	user_addr_t         returnbuffer;
 	user_size_t         returnbuffersize;
 	user_ulong_t        maxmatches;
-	struct timeval		timelimit;
-	user_addr_t         searchparams1;
+	struct user_timeval	timelimit;
+	user_addr_t         searchparams1 __attribute((aligned(8)));
 	user_size_t         sizeofsearchparams1;
 	user_addr_t         searchparams2;
 	user_size_t         sizeofsearchparams2;
 	struct attrlist		searchattrs;
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 
 #endif // KERNEL
 
diff --git a/bsd/sys/bsdtask_info.h b/bsd/sys/bsdtask_info.h
new file mode 100644
index 000000000..91bd0027a
--- /dev/null
+++ b/bsd/sys/bsdtask_info.h
@@ -0,0 +1,114 @@
+
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _SYS_BSDTASK_INFO_H
+#define _SYS_BSDTASK_INFO_H
+
+struct proc_taskinfo_internal {          
+        uint64_t                pti_virtual_size;   /* virtual memory size (bytes) */
+        uint64_t                pti_resident_size;  /* resident memory size (bytes) */
+        uint64_t                pti_total_user;         /* total time */
+        uint64_t                pti_total_system;
+        uint64_t                pti_threads_user;       /* existing threads only */
+        uint64_t                pti_threads_system;
+        int32_t                 pti_policy;             /* default policy for new threads */
+        int32_t                 pti_faults;             /* number of page faults */
+        int32_t                 pti_pageins;    /* number of actual pageins */
+        int32_t                 pti_cow_faults; /* number of copy-on-write faults */
+        int32_t                 pti_messages_sent;      /* number of messages sent */
+        int32_t                 pti_messages_received; /* number of messages received */
+        int32_t                 pti_syscalls_mach;  /* number of mach system calls */
+        int32_t                 pti_syscalls_unix;  /* number of unix system calls */
+        int32_t                 pti_csw;            /* number of context switches */
+        int32_t                 pti_threadnum;          /* number of threads in the task */
+        int32_t                 pti_numrunning;         /* number of running threads */
+	int32_t                 pti_priority;           /* task priority*/
+};
+
+
+struct proc_threadinfo_internal {
+	uint64_t		pth_user_time;      /* user run time */
+	uint64_t		pth_system_time;    /* system run time */
+	int32_t			pth_cpu_usage;      /* scaled cpu usage percentage */
+	int32_t			pth_policy;		/* scheduling policy in effect */
+	int32_t			pth_run_state;      /* run state (see below) */
+	int32_t			pth_flags;          /* various flags (see below) */
+	int32_t			pth_sleep_time;     /* number of seconds that thread */
+	int32_t			pth_curpri;		/* cur priority*/
+	int32_t			pth_priority;		/*  priority*/
+	int32_t			pth_maxpriority;		/* max priority*/
+};
+
+
+
+struct proc_regioninfo_internal {
+	uint32_t		pri_protection;
+	uint32_t		pri_max_protection;
+	uint32_t		pri_inheritance;
+	uint32_t		pri_flags;		/* shared, external pager, is submap */
+	uint64_t		pri_offset;
+	uint32_t		pri_behavior;
+	uint32_t		pri_user_wired_count;
+	uint32_t		pri_user_tag;
+	uint32_t		pri_pages_resident;
+	uint32_t		pri_pages_shared_now_private;
+	uint32_t		pri_pages_swapped_out;
+	uint32_t		pri_pages_dirtied;
+	uint32_t		pri_ref_count;
+	uint32_t		pri_shadow_depth;
+	uint32_t		pri_share_mode;
+	uint32_t		pri_private_pages_resident;
+	uint32_t		pri_shared_pages_resident;
+	uint32_t		pri_obj_id;
+	uint64_t		pri_address;
+	uint64_t		pri_size;
+	uint32_t		pri_depth;
+};
+
+#ifdef  MACH_KERNEL_PRIVATE
+
+#define PROC_REGION_SUBMAP	1
+#define PROC_REGION_SHARED	2
+
+void  vm_map_region_top_walk(vm_map_entry_t entry, vm_region_top_info_t top);
+void vm_map_region_walk(vm_map_t map, vm_map_offset_t a, vm_map_entry_t entry, vm_object_offset_t offset, vm_object_size_t range, vm_region_extended_info_t extended);
+kern_return_t vnode_pager_get_object_vnode(memory_object_t mem_obj,uint32_t * vnodeaddr, uint32_t * vid);
+extern uint32_t vnode_vid(void *vp);
+
+#endif /* MACH_KERNEL_PRIVATE */
+
+extern int fill_procregioninfo(task_t t, uint64_t arg, struct proc_regioninfo_internal *pinfo, uint32_t *vp, uint32_t *vid);
+void fill_taskprocinfo(task_t task, struct proc_taskinfo_internal * ptinfo);
+int fill_taskthreadinfo(task_t task, uint64_t thaddr, struct proc_threadinfo_internal * ptinfo);
+int fill_taskthreadlist(task_t task, void * buffer, int thcount);
+int get_numthreads(task_t);
+
+
+#endif /*_SYS_BSDTASK_INFO_H */
+
diff --git a/bsd/sys/buf_internal.h b/bsd/sys/buf_internal.h
index 2afe6eb2c..728282146 100644
--- a/bsd/sys/buf_internal.h
+++ b/bsd/sys/buf_internal.h
@@ -80,7 +80,6 @@
 #include <sys/lock.h>
 
 
-extern lck_mtx_t    *buf_mtxp;
 #define NOLIST ((struct buf *)0x87654321)
 
 /*
@@ -195,6 +194,8 @@ struct buf {
 extern int niobuf;		/* The number of IO buffer headers for cluster IO */
 extern int nbuf;		/* The number of buffer headers */
 extern struct buf *buf;		/* The buffer headers. */
+extern int max_nbuf_headers;		/* The max number of buffer headers */
+extern int nbuf_hashelements;		/* The number of elements in bufhash */
 
 
 /*
diff --git a/bsd/sys/cdefs.h b/bsd/sys/cdefs.h
index 51c41845b..b18da3381 100644
--- a/bsd/sys/cdefs.h
+++ b/bsd/sys/cdefs.h
@@ -333,7 +333,7 @@
 #   define	__DARWIN_LDBL_COMPAT2(x) /* nothing */
 #   define	__DARWIN_LONG_DOUBLE_IS_DOUBLE	1
 #  endif
-#elif defined(__i386__) || defined(__ppc64__)
+#elif defined(__i386__) || defined(__ppc64__) || defined(__x86_64__)
 #  define	__DARWIN_LDBL_COMPAT(x)	/* nothing */
 #  define	__DARWIN_LDBL_COMPAT2(x) /* nothing */
 #  define	__DARWIN_LONG_DOUBLE_IS_DOUBLE	0
@@ -341,26 +341,4 @@
 #  error Unknown architecture
 #endif
 
-/*
- * Structure alignment control macros.  These specify how certain
- * shared structures should be aligned.  Some may need backward
- * compatible legacy (POWER) alignment, while others may need
- * forward compatible (NATURAL) alignment.
- */
-#if !defined(__DARWIN_ALIGN_POWER)
-#if defined(__ppc64__)
-#define __DARWIN_ALIGN_POWER 1
-#else
-#define __DARWIN_ALIGN_POWER 0
-#endif
-#endif /* __DARWIN_ALIGN_POWER */
-
-#if !defined(__DARWIN_ALIGN_NATURAL)
-#if defined(__ppc__) && defined(KERNEL)
-#define __DARWIN_ALIGN_NATURAL 1
-#else
-#define __DARWIN_ALIGN_NATURAL 0
-#endif
-#endif /* __DARWIN_ALIGN_NATURAL */
-
 #endif /* !_CDEFS_H_ */
diff --git a/bsd/sys/dirent.h b/bsd/sys/dirent.h
index fb6efcd74..af090db5a 100644
--- a/bsd/sys/dirent.h
+++ b/bsd/sys/dirent.h
@@ -85,9 +85,7 @@ typedef	__darwin_ino_t	ino_t;		/* inode number */
 
 #define __DARWIN_MAXNAMLEN	255
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 struct dirent {
 	ino_t d_ino;			/* file number of entry */
@@ -97,9 +95,7 @@ struct dirent {
 	char d_name[__DARWIN_MAXNAMLEN + 1];	/* name must be no longer than this */
 };
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 #ifdef KERNEL
 #include <sys/kernel_types.h>
diff --git a/bsd/sys/disk.h b/bsd/sys/disk.h
index 7d51c25eb..648ec9c9b 100644
--- a/bsd/sys/disk.h
+++ b/bsd/sys/disk.h
@@ -36,34 +36,36 @@
 /*
  * Definitions
  *
- * ioctl                            description
- * -------------------------------- --------------------------------------------
- * DKIOCEJECT                       eject media
- * DKIOCSYNCHRONIZECACHE            flush media
+ * ioctl                                 description
+ * ------------------------------------- ---------------------------------------
+ * DKIOCEJECT                            eject media
+ * DKIOCSYNCHRONIZECACHE                 flush media
  *
- * DKIOCFORMAT                      format media
- * DKIOCGETFORMATCAPACITIES         get media's formattable capacities
+ * DKIOCFORMAT                           format media
+ * DKIOCGETFORMATCAPACITIES              get media's formattable capacities
  *
- * DKIOCGETBLOCKSIZE                get media's block size
- * DKIOCGETBLOCKCOUNT               get media's block count
- * DKIOCGETFIRMWAREPATH             get media's firmware path
+ * DKIOCGETBLOCKSIZE                     get media's block size
+ * DKIOCGETBLOCKCOUNT                    get media's block count
+ * DKIOCGETFIRMWAREPATH                  get media's firmware path
  *
- * DKIOCISFORMATTED                 is media formatted?
- * DKIOCISWRITABLE                  is media writable?
+ * DKIOCISFORMATTED                      is media formatted?
+ * DKIOCISWRITABLE                       is media writable?
  *
- * DKIOCGETMAXBLOCKCOUNTREAD        get maximum block count for reads
- * DKIOCGETMAXBLOCKCOUNTWRITE       get maximum block count for writes
- * DKIOCGETMAXBYTECOUNTREAD         get maximum byte count for reads
- * DKIOCGETMAXBYTECOUNTWRITE        get maximum byte count for writes
- * DKIOCGETMAXSEGMENTCOUNTREAD      get maximum segment count for reads
- * DKIOCGETMAXSEGMENTCOUNTWRITE     get maximum segment count for writes
- * DKIOCGETMAXSEGMENTBYTECOUNTREAD  get maximum segment byte count for reads
- * DKIOCGETMAXSEGMENTBYTECOUNTWRITE get maximum segment byte count for writes
+ * DKIOCGETMAXBLOCKCOUNTREAD             get maximum block count for reads
+ * DKIOCGETMAXBLOCKCOUNTWRITE            get maximum block count for writes
+ * DKIOCGETMAXBYTECOUNTREAD              get maximum byte count for reads
+ * DKIOCGETMAXBYTECOUNTWRITE             get maximum byte count for writes
+ *
+ * DKIOCGETMAXSEGMENTCOUNTREAD           get maximum segment count for reads
+ * DKIOCGETMAXSEGMENTCOUNTWRITE          get maximum segment count for writes
+ * DKIOCGETMAXSEGMENTBYTECOUNTREAD       get maximum segment byte count for reads
+ * DKIOCGETMAXSEGMENTBYTECOUNTWRITE      get maximum segment byte count for writes
+ *
+ * DKIOCGETMINSEGMENTALIGNMENTBYTECOUNT  get minimum segment alignment in bytes
+ * DKIOCGETMAXSEGMENTADDRESSABLEBITCOUNT get maximum segment width in bits
  */
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 typedef struct
 {
@@ -87,38 +89,40 @@ typedef struct
     u_int8_t               reserved0064[8];        /* reserved, clear to zero */
 } dk_format_capacities_t;
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
+
+#define DKIOCEJECT                            _IO('d', 21)
+#define DKIOCSYNCHRONIZECACHE                 _IO('d', 22)
+
+#define DKIOCFORMAT                           _IOW('d', 26, dk_format_capacity_t)
+#define DKIOCGETFORMATCAPACITIES              _IOWR('d', 26, dk_format_capacities_t)
 
-#define DKIOCEJECT                       _IO('d', 21)
-#define DKIOCSYNCHRONIZECACHE            _IO('d', 22)
+#define DKIOCGETBLOCKSIZE                     _IOR('d', 24, u_int32_t)
+#define DKIOCGETBLOCKCOUNT                    _IOR('d', 25, u_int64_t)
+#define DKIOCGETFIRMWAREPATH                  _IOR('d', 28, dk_firmware_path_t)
 
-#define DKIOCFORMAT                      _IOW('d', 26, dk_format_capacity_t)
-#define DKIOCGETFORMATCAPACITIES         _IOWR('d', 26, dk_format_capacities_t)
+#define DKIOCISFORMATTED                      _IOR('d', 23, u_int32_t)
+#define DKIOCISWRITABLE                       _IOR('d', 29, u_int32_t)
 
-#define DKIOCGETBLOCKSIZE                _IOR('d', 24, u_int32_t)
-#define DKIOCGETBLOCKCOUNT               _IOR('d', 25, u_int64_t)
-#define DKIOCGETFIRMWAREPATH             _IOR('d', 28, dk_firmware_path_t)
+#define DKIOCGETMAXBLOCKCOUNTREAD             _IOR('d', 64, u_int64_t)
+#define DKIOCGETMAXBLOCKCOUNTWRITE            _IOR('d', 65, u_int64_t)
+#define DKIOCGETMAXBYTECOUNTREAD              _IOR('d', 70, u_int64_t)
+#define DKIOCGETMAXBYTECOUNTWRITE             _IOR('d', 71, u_int64_t)
 
-#define DKIOCISFORMATTED                 _IOR('d', 23, u_int32_t)
-#define DKIOCISWRITABLE                  _IOR('d', 29, u_int32_t)
+#define DKIOCGETMAXSEGMENTCOUNTREAD           _IOR('d', 66, u_int64_t)
+#define DKIOCGETMAXSEGMENTCOUNTWRITE          _IOR('d', 67, u_int64_t)
+#define DKIOCGETMAXSEGMENTBYTECOUNTREAD       _IOR('d', 68, u_int64_t)
+#define DKIOCGETMAXSEGMENTBYTECOUNTWRITE      _IOR('d', 69, u_int64_t)
 
-#define DKIOCGETMAXBLOCKCOUNTREAD        _IOR('d', 64, u_int64_t)
-#define DKIOCGETMAXBLOCKCOUNTWRITE       _IOR('d', 65, u_int64_t)
-#define DKIOCGETMAXBYTECOUNTREAD         _IOR('d', 70, u_int64_t)
-#define DKIOCGETMAXBYTECOUNTWRITE        _IOR('d', 71, u_int64_t)
-#define DKIOCGETMAXSEGMENTCOUNTREAD      _IOR('d', 66, u_int64_t)
-#define DKIOCGETMAXSEGMENTCOUNTWRITE     _IOR('d', 67, u_int64_t)
-#define DKIOCGETMAXSEGMENTBYTECOUNTREAD  _IOR('d', 68, u_int64_t)
-#define DKIOCGETMAXSEGMENTBYTECOUNTWRITE _IOR('d', 69, u_int64_t)
+#define DKIOCGETMINSEGMENTALIGNMENTBYTECOUNT  _IOR('d', 74, u_int64_t)
+#define DKIOCGETMAXSEGMENTADDRESSABLEBITCOUNT _IOR('d', 75, u_int64_t)
 
 #ifdef KERNEL
-#define DKIOCGETBLOCKCOUNT32             _IOR('d', 25, u_int32_t)
-#define DKIOCSETBLOCKSIZE                _IOW('d', 24, u_int32_t)
-#define DKIOCGETBSDUNIT                  _IOR('d', 27, u_int32_t)
-#define DKIOCISVIRTUAL                   _IOR('d', 72, u_int32_t)
-#define DKIOCGETBASE                     _IOR('d', 73, u_int64_t)
+#define DKIOCGETBLOCKCOUNT32                  _IOR('d', 25, u_int32_t)
+#define DKIOCSETBLOCKSIZE                     _IOW('d', 24, u_int32_t)
+#define DKIOCGETBSDUNIT                       _IOR('d', 27, u_int32_t)
+#define DKIOCISVIRTUAL                        _IOR('d', 72, u_int32_t)
+#define DKIOCGETBASE                          _IOR('d', 73, u_int64_t)
 #endif /* KERNEL */
 
 #endif	/* _SYS_DISK_H_ */
diff --git a/bsd/sys/domain.h b/bsd/sys/domain.h
index eea934b6f..7a683b1f6 100644
--- a/bsd/sys/domain.h
+++ b/bsd/sys/domain.h
@@ -84,9 +84,7 @@
 struct	mbuf;
 #define DOM_REENTRANT	0x01
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 struct	domain {
 	int	dom_family;		/* AF_xxx */
@@ -113,9 +111,7 @@ struct	domain {
 	u_long		reserved[2];
 };
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 #ifdef KERNEL
 extern struct	domain *domains;
diff --git a/bsd/sys/event.h b/bsd/sys/event.h
index ea5bc330f..7f98a7d00 100644
--- a/bsd/sys/event.h
+++ b/bsd/sys/event.h
@@ -73,9 +73,7 @@
 #define EVFILT_SYSCOUNT		9
 #define EVFILT_THREADMARKER	EVFILT_SYSCOUNT /* Internal use only */
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 struct kevent {
 	uintptr_t	ident;		/* identifier for this event */
@@ -103,9 +101,7 @@ struct user_kevent {
 
 #endif
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 #define EV_SET(kevp, a, b, c, d, e, f) do {	\
 	struct kevent *__kevp__ = (kevp);	\
diff --git a/bsd/sys/fcntl.h b/bsd/sys/fcntl.h
index 818590cb0..07499ad42 100644
--- a/bsd/sys/fcntl.h
+++ b/bsd/sys/fcntl.h
@@ -150,6 +150,10 @@ typedef __darwin_pid_t	pid_t;
 #define	FWASWRITTEN	0x10000		/* descriptor was written */
 #endif
 
+#ifndef _POSIX_C_SOURCE
+#define O_DIRECTORY    0x100000
+#endif
+
 /* defined by POSIX 1003.1; BSD default, so no bit required */
 #define	O_NOCTTY	0		/* don't assign controlling terminal */
 //#define	O_SYNC  /* ??? POSIX: Write according to synchronized I/O file integrity completion */
@@ -365,20 +369,12 @@ typedef struct fbootstraptransfer {
  * WARNING - keep in sync with fbootstraptransfer
  */
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 typedef struct user_fbootstraptransfer {
   off_t fbt_offset;             /* IN: offset to start read/write */
   user_size_t fbt_length;		/* IN: number of bytes to transfer */
   user_addr_t fbt_buffer;		/* IN: buffer to be read/written */
 } user_fbootstraptransfer_t;
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif // KERNEL
 
 /*
@@ -397,9 +393,7 @@ typedef struct user_fbootstraptransfer {
  * and a per filesystem type flag will be needed to interpret the
  * contiguous bytes count result from CMAP.
  */
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 struct log2phys {
 	unsigned int	l2p_flags;		/* unused so far */
@@ -407,9 +401,7 @@ struct log2phys {
 	off_t		l2p_devoffset;	/* bytes into device */
 };
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 #define	O_POPUP	   0x80000000   /* force window to popup on open */
 #define	O_ALERT	   0x20000000	/* small, clean popup window */
diff --git a/bsd/sys/file.h b/bsd/sys/file.h
index d1cb183b8..b65542175 100644
--- a/bsd/sys/file.h
+++ b/bsd/sys/file.h
@@ -76,9 +76,7 @@
 #include <sys/kernel_types.h>
 #endif
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 /* for the compat sake;  */
 struct extern_file {
@@ -93,9 +91,7 @@ struct extern_file {
 	caddr_t	f_data;		/* vnode or socket or SHM or semaphore */
 };
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 #ifdef KERNEL
 __BEGIN_DECLS
diff --git a/bsd/sys/file_internal.h b/bsd/sys/file_internal.h
index 814f0dba2..b9a29f9ef 100644
--- a/bsd/sys/file_internal.h
+++ b/bsd/sys/file_internal.h
@@ -199,8 +199,15 @@ struct kqueue;
 int fp_getfkq(struct proc *p, int fd, struct fileproc **resultfp, struct kqueue  **resultkq);
 struct psemnode;
 int fp_getfpsem(struct proc *p, int fd, struct fileproc **resultfp, struct psemnode  **resultpsem);
+struct pshmnode;
+int fp_getfpshm(struct proc *p, int fd, struct fileproc **resultfp, struct pshmnode  **resultpshm);
+struct pipe;
+int fp_getfpipe(struct proc *p, int fd, struct fileproc **resultfp, struct pipe  **resultpipe);
+struct atalk;
+int fp_getfatalk(struct proc *p, int fd, struct fileproc **resultfp, struct atalk  **resultatalk);
 struct vnode;
 int fp_getfvp(struct proc *p, int fd, struct fileproc **resultfp, struct vnode  **resultvp);
+int fp_getfvpandvid(struct proc *p, int fd, struct fileproc **resultfp, struct vnode  **resultvp, uint32_t * vidp);
 struct socket;
 int fp_getfsock(struct proc *p, int fd, struct fileproc **resultfp, struct socket  **results);
 int fp_lookup(struct proc *p, int fd, struct fileproc **resultfp, int locked);
diff --git a/bsd/sys/imgact.h b/bsd/sys/imgact.h
index 65aa3d9c3..cef71ad2c 100644
--- a/bsd/sys/imgact.h
+++ b/bsd/sys/imgact.h
@@ -85,7 +85,7 @@ struct image_params {
 	user_size_t 	ip_arch_size;		/* subfile length in ip_vp */
 	char		ip_interp_name[IMG_SHSIZE];	/* interpreter name */
 
-	/* Next two fields are for support of Classic... */
+	/* Next two fields are for support of architecture translation... */
 	char		*ip_p_comm;		/* optional alt p->p_comm */
 	char		*ip_tws_cache_name;	/* task working set cache */
 	struct vfs_context	*ip_vfs_context;	/* VFS context */
@@ -98,7 +98,11 @@ struct image_params {
  */
 #define	IMGPF_NONE	0x00000000		/* No flags */
 #define	IMGPF_INTERPRET	0x00000001		/* Interpreter invoked */
+#if defined (__i386__) || defined(__x86_64__)
+#define	IMGPF_POWERPC	0x00000002		/* ppc mode */
+#else
 #define	IMGPF_RESERVED1	0x00000002		/* reserved */
+#endif
 #define	IMGPF_WAS_64BIT	0x00000004		/* exec from a 64Bit binary */
 #define	IMGPF_IS_64BIT	0x00000008		/* exec to a 64Bit binary */
 
diff --git a/bsd/sys/ipcs.h b/bsd/sys/ipcs.h
index 38e99b62d..f30abbfe2 100644
--- a/bsd/sys/ipcs.h
+++ b/bsd/sys/ipcs.h
@@ -59,10 +59,6 @@ struct IPCS_command {
 #ifdef KERNEL_PRIVATE
 #include <machine/types.h>
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_IPCS_command {
 	int		ipcs_magic;	/* Magic number for struct layout */
 	int		ipcs_op;	/* Operation to perform */
@@ -71,10 +67,6 @@ struct user_IPCS_command {
 	user_addr_t	ipcs_data;	/* OP specific data */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif /* KERNEL_PRIVATE */
 
 /*
diff --git a/bsd/sys/kauth.h b/bsd/sys/kauth.h
index 151499920..aed48b901 100644
--- a/bsd/sys/kauth.h
+++ b/bsd/sys/kauth.h
@@ -185,7 +185,7 @@ extern void	kauth_cred_ref(kauth_cred_t _cred);
 extern void	kauth_cred_rele(kauth_cred_t _cred);
 extern kauth_cred_t kauth_cred_dup(kauth_cred_t cred);
 extern kauth_cred_t kauth_cred_copy_real(kauth_cred_t cred);
-extern void	kauth_cred_unref(kauth_cred_t _cred);
+extern void	kauth_cred_unref(kauth_cred_t *_cred);
 extern kauth_cred_t	kauth_cred_setuid(kauth_cred_t cred, uid_t uid);
 extern kauth_cred_t	kauth_cred_seteuid(kauth_cred_t cred, uid_t euid);
 extern kauth_cred_t	kauth_cred_setgid(kauth_cred_t cred, gid_t gid);
@@ -307,7 +307,17 @@ struct kauth_acl {
 /* this ACL must not be overwritten as part of an inheritance operation */
 #define KAUTH_ACL_NO_INHERIT	(1<<17)
 
-#define KAUTH_ACL_SIZE(c)	(sizeof(struct kauth_acl) + (c) * sizeof(struct kauth_ace))
+/* acl_entrycount that tells us the ACL is not valid */
+#define KAUTH_FILESEC_NOACL ((u_int32_t)(-1))
+
+/*
+ * If the acl_entrycount field is KAUTH_FILESEC_NOACL, then the size is the
+ * same as a kauth_acl structure; the intent is to put an actual entrycount of
+ * KAUTH_FILESEC_NOACL on disk to distinguish a kauth_filesec_t with an empty
+ * entry (Windows treats this as "deny all") from one that merely indicates a
+ * file group and/or owner guid values.
+ */
+#define KAUTH_ACL_SIZE(c)	(sizeof(struct kauth_acl) + ((u_int32_t)(c) != KAUTH_FILESEC_NOACL ? ((c) * sizeof(struct kauth_ace)) : 0))
 #define KAUTH_ACL_COPYSIZE(p)	KAUTH_ACL_SIZE((p)->acl_entrycount)
 
 
@@ -336,8 +346,6 @@ struct kauth_filesec {
 	guid_t		fsec_group;
 
 	struct kauth_acl fsec_acl;
-	/* acl_entrycount that tells us the ACL is not valid */
-#define KAUTH_FILESEC_NOACL ((u_int32_t)(-1))
 };
 
 /* backwards compatibility */
@@ -358,13 +366,19 @@ typedef struct kauth_filesec *kauth_filesec_t;
 #define KAUTH_FILESEC_SIZE(c)		(sizeof(struct kauth_filesec) + (c) * sizeof(struct kauth_ace))
 #define KAUTH_FILESEC_COPYSIZE(p)	KAUTH_FILESEC_SIZE(((p)->fsec_entrycount == KAUTH_FILESEC_NOACL) ? 0 : (p)->fsec_entrycount)
 #define KAUTH_FILESEC_COUNT(s)		((s  - sizeof(struct kauth_filesec)) / sizeof(struct kauth_ace))
+#define KAUTH_FILESEC_VALID(s)		((s) >= sizeof(struct kauth_filesec) && (((s) - sizeof(struct kauth_filesec)) % sizeof(struct kauth_ace)) == 0)
 
 #define KAUTH_FILESEC_XATTR	"com.apple.system.Security"
 
+/* Allowable first arguments to kauth_filesec_acl_setendian() */
+#define	KAUTH_ENDIAN_HOST	0x00000001	/* set host endianness */
+#define	KAUTH_ENDIAN_DISK	0x00000002	/* set disk endianness */
+
 __BEGIN_DECLS
 kauth_filesec_t	kauth_filesec_alloc(int size);
 void		kauth_filesec_free(kauth_filesec_t fsp);
 int		kauth_copyinfilesec(user_addr_t xsecurity, kauth_filesec_t *xsecdestpp);
+ void		kauth_filesec_acl_setendian(int, kauth_filesec_t, kauth_acl_t);
 __END_DECLS	
 
 #endif /* KERNEL || <sys/acl.h> */
diff --git a/bsd/sys/kdebug.h b/bsd/sys/kdebug.h
index b6cf0c3be..d7d85656a 100644
--- a/bsd/sys/kdebug.h
+++ b/bsd/sys/kdebug.h
@@ -73,29 +73,31 @@ __BEGIN_DECLS
 
 
 /* The Kernel Debug Classes  */
-#define DBG_MACH		1
-#define DBG_NETWORK		2	
-#define DBG_FSYSTEM		3
-#define DBG_BSD			4
-#define DBG_IOKIT		5
-#define DBG_DRIVERS		6
-#define DBG_TRACE               7
+#define DBG_MACH			1
+#define DBG_NETWORK			2	
+#define DBG_FSYSTEM			3
+#define DBG_BSD				4
+#define DBG_IOKIT			5
+#define DBG_DRIVERS			6
+#define DBG_TRACE           7
 #define DBG_DLIL	        8
 #define DBG_SECURITY		9
-#define DBG_MISC		20
-#define DBG_DYLD                31
-#define DBG_QT                  32
-#define DBG_APPS                33
-#define DBG_MIG			255
+#define DBG_MISC			20
+#define DBG_DYLD            31
+#define DBG_QT              32
+#define DBG_APPS            33
+#define DBG_MIG				255
 
 /* **** The Kernel Debug Sub Classes for Mach (DBG_MACH) **** */
+#define	DBG_MACH_EXCP_KTRAP_x86	0x02	/* Kernel Traps on x86 */
 #define	DBG_MACH_EXCP_DFLT	0x03	/* Data Translation Fault */
 #define	DBG_MACH_EXCP_IFLT	0x04	/* Inst Translation Fault */
 #define	DBG_MACH_EXCP_INTR	0x05	/* Interrupts */
 #define	DBG_MACH_EXCP_ALNG	0x06	/* Alignment Exception */
-#define	DBG_MACH_EXCP_TRAP	0x07	/* Traps */
+#define	DBG_MACH_EXCP_UTRAP_x86	0x07	/* User Traps on x86 */
 #define	DBG_MACH_EXCP_FP	0x08	/* FP Unavail */
 #define	DBG_MACH_EXCP_DECI	0x09	/* Decrementer Interrupt */
+#define	DBG_MACH_CHUD		0x0A	/* CHUD */
 #define	DBG_MACH_EXCP_SC	0x0C	/* System Calls */
 #define	DBG_MACH_EXCP_TRACE	0x0D	/* Trace exception */
 #define	DBG_MACH_EXCP_EMUL	0x0E	/* Instruction emulated */
@@ -145,33 +147,60 @@ __BEGIN_DECLS
 #define	DBG_NETIPSEC	128	/* IPsec Protocol  */
 
 /* **** The Kernel Debug Sub Classes for IOKIT (DBG_IOKIT) **** */
-#define DBG_IOSCSI	1	/* SCSI */
-#define DBG_IODISK	2	/* Disk layers */
-#define	DBG_IONETWORK	3	/* Network layers */
-#define	DBG_IOKEYBOARD	4	/* Keyboard */
-#define	DBG_IOPOINTING	5	/* Pointing Devices */
-#define	DBG_IOAUDIO	6	/* Audio */
-#define	DBG_IOFLOPPY	7	/* Floppy */
-#define	DBG_IOSERIAL	8	/* Serial */
-#define	DBG_IOTTY	9	/* TTY layers */
-#define DBG_IOWORKLOOP	10	/* Work from work loop */
-#define DBG_IOINTES	11	/* Interrupt event source */
-#define DBG_IOCLKES	12	/* Clock event source */
-#define DBG_IOCMDQ	13	/* Command queue latencies */
-#define DBG_IOMCURS	14	/* Memory Cursor */
-#define DBG_IOMDESC	15	/* Memory Descriptors */
-#define DBG_IOPOWER	16	/* Power Managerment */
+#define DBG_IOWORKLOOP		1	/* Work from work loop */
+#define DBG_IOINTES			2	/* Interrupt event source */
+#define DBG_IOCLKES			3	/* Clock event source */
+#define DBG_IOCMDQ			4	/* Command queue latencies */
+#define DBG_IOMCURS			5	/* Memory Cursor */
+#define DBG_IOMDESC			6	/* Memory Descriptors */
+#define DBG_IOPOWER			7	/* Power Managerment */
+
+/* **** 8-32 reserved for internal IOKit usage **** */
+
+#define DBG_IOSTORAGE		32	/* Storage layers */
+#define	DBG_IONETWORK		33	/* Network layers */
+#define	DBG_IOKEYBOARD		34	/* Keyboard */
+#define	DBG_IOHID			35	/* HID Devices */
+#define	DBG_IOAUDIO			36	/* Audio */
+#define	DBG_IOSERIAL		37	/* Serial */
+#define	DBG_IOTTY			38	/* TTY layers */
+#define DBG_IOSAM			39	/* SCSI Architecture Model layers */
+#define DBG_IOPARALLELATA   40	/* Parallel ATA */
+#define DBG_IOPARALLELSCSI	41	/* Parallel SCSI */
+#define DBG_IOSATA			42	/* Serial-ATA */
+#define DBG_IOSAS			43	/* SAS */
+#define DBG_IOFIBRECHANNEL	44	/* FiberChannel */
+#define DBG_IOUSB			45	/* USB */
+#define DBG_IOBLUETOOTH		46	/* Bluetooth */
+#define DBG_IOFIREWIRE		47	/* FireWire */
+#define DBG_IOINFINIBAND	48	/* Infiniband */
+#define DBG_IOCPUPM		49	/* CPU Power Management */
+
+/* Backwards compatibility */
+#define	DBG_IOPOINTING		DBG_IOHID			/* OBSOLETE: Use DBG_IOHID instead */
+#define DBG_IODISK			DBG_IOSTORAGE		/* OBSOLETE: Use DBG_IOSTORAGE instead */
 
 /* **** The Kernel Debug Sub Classes for Device Drivers (DBG_DRIVERS) **** */
-#define DBG_DRVSCSI	1	/* SCSI */
-#define DBG_DRVDISK	2	/* Disk layers */
-#define	DBG_DRVNETWORK	3	/* Network layers */
-#define	DBG_DRVKEYBOARD	4	/* Keyboard */
-#define	DBG_DRVPOINTING	5	/* Pointing Devices */
-#define	DBG_DRVAUDIO	6	/* Audio */
-#define	DBG_DRVFLOPPY	7	/* Floppy */
-#define	DBG_DRVSERIAL	8	/* Serial */
-#define DBG_DRVSPLT     9
+#define DBG_DRVSTORAGE		1	/* Storage layers */
+#define	DBG_DRVNETWORK		2	/* Network layers */
+#define	DBG_DRVKEYBOARD		3	/* Keyboard */
+#define	DBG_DRVHID			4	/* HID Devices */
+#define	DBG_DRVAUDIO		5	/* Audio */
+#define	DBG_DRVSERIAL		7	/* Serial */
+#define DBG_DRVSAM			8	/* SCSI Architecture Model layers */
+#define DBG_DRVPARALLELATA  9	/* Parallel ATA */
+#define DBG_DRVPARALLELSCSI	10	/* Parallel SCSI */
+#define DBG_DRVSATA			11	/* Serial ATA */
+#define DBG_DRVSAS			12	/* SAS */
+#define DBG_DRVFIBRECHANNEL	13	/* FiberChannel */
+#define DBG_DRVUSB			14	/* USB */
+#define DBG_DRVBLUETOOTH	15	/* Bluetooth */
+#define DBG_DRVFIREWIRE		16	/* FireWire */
+#define DBG_DRVINFINIBAND	17	/* Infiniband */
+
+/* Backwards compatibility */
+#define	DBG_DRVPOINTING		DBG_DRVHID		/* OBSOLETE: Use DBG_DRVHID instead */
+#define DBG_DRVDISK			DBG_DRVSTORAGE	/* OBSOLETE: Use DBG_DRVSTORAGE instead */
 
 /* **** The Kernel Debug Sub Classes for the DLIL Layer (DBG_DLIL) **** */
 #define DBG_DLIL_STATIC 1       /* Static DLIL code */
@@ -195,6 +224,10 @@ __BEGIN_DECLS
 #define DBG_TRACE_DATA      0
 #define DBG_TRACE_STRING    1
 
+/* The Kernel Debug Sub Classes for DBG_MISC */
+#define DBG_EVENT	0x10
+#define	DBG_BUFFER	0x20
+
 /* The Kernel Debug Sub Classes for DBG_DYLD */
 #define DBG_DYLD_STRING   5
 
@@ -205,6 +238,12 @@ __BEGIN_DECLS
 #define DKIO_META	0x08
 #define DKIO_PAGING	0x10
 
+/* The Kernel Debug Modifiers for the DBG_IOCPUPM sub-class */
+#define DCPM_PSTATE		0x0001
+#define DCPM_IDLE_CSTATE	0x0002
+#define DCPM_IDLE_HALT		0x0003
+#define DCPM_IDLE_LOOP		0x0004
+
 /**********************************************************************/
 
 #define KDBG_CODE(Class, SubClass, code) (((Class & 0xff) << 24) | ((SubClass & 0xff) << 16) | ((code & 0x3fff)  << 2))
@@ -224,6 +263,7 @@ __BEGIN_DECLS
 #define DYLDDBG_CODE(SubClass,code) KDBG_CODE(DBG_DYLD, SubClass, code)
 #define QTDBG_CODE(SubClass,code) KDBG_CODE(DBG_QT, SubClass, code)
 #define APPSDBG_CODE(SubClass,code) KDBG_CODE(DBG_APPS, SubClass, code)
+#define CPUPM_CODE(code) IOKDBG_CODE(DBG_IOCPUPM, code)
 
 /*   Usage:
 * kernel_debug((KDBG_CODE(DBG_NETWORK, DNET_PROTOCOL, 51) | DBG_FUNC_START), 
@@ -276,6 +316,7 @@ extern void kernel_debug1(unsigned int debugid, unsigned int arg1, unsigned int
 /*
  * LP64todo - for some reason these are problematic
  */
+struct proc;
 extern void kdbg_trace_data(struct proc *proc, long *arg_pid);
 
 extern void kdbg_trace_string(struct proc *proc, long *arg1, long *arg2, long *arg3, long *arg4);
diff --git a/bsd/sys/kernel_types.h b/bsd/sys/kernel_types.h
index 89eb0ed0b..c4d9a60be 100644
--- a/bsd/sys/kernel_types.h
+++ b/bsd/sys/kernel_types.h
@@ -33,7 +33,7 @@ typedef struct mount * mount_t;
 
 #ifdef TBDDDD
 typedef struct fsid { int32_t val[2]; } fsid_t;	/* file system id type */
-#endif TBDDDD
+#endif /* TBDDDD */
 
 struct vnode;
 typedef struct vnode * vnode_t;
diff --git a/bsd/sys/kpi_socket.h b/bsd/sys/kpi_socket.h
index f65b17472..95f026699 100644
--- a/bsd/sys/kpi_socket.h
+++ b/bsd/sys/kpi_socket.h
@@ -39,6 +39,7 @@
 
 #include <sys/types.h>
 #include <sys/kernel_types.h>
+#include <sys/socket.h>
 
 struct timeval;
 
diff --git a/bsd/sys/ktrace.h b/bsd/sys/ktrace.h
index b905d5107..9516df927 100644
--- a/bsd/sys/ktrace.h
+++ b/bsd/sys/ktrace.h
@@ -69,8 +69,8 @@
 
 #ifdef MACH_KERNEL_PRIVATE
 
-void ktrsyscall(void *, int, int, u_int64_t *, int);
-void ktrsysret(void *, int, int, int, int);
+void ktrsyscall(void *, int, int, void *);
+void ktrsysret(void *, int, int, int);
 
 #else
 #ifdef __APPLE_API_UNSTABLE
@@ -203,7 +203,7 @@ void	ktrnamei(struct vnode *,char *);
 void	ktrcsw(struct vnode *, int, int);
 void	ktrpsig(struct vnode *, int, sig_t, sigset_t *, int);
 void	ktrgenio(struct vnode *, int, enum uio_rw, struct uio *, int);
-void	ktrsyscall(struct proc *, int, int, u_int64_t args[]);
+void	ktrsyscall(struct proc *, int, int, syscall_arg_t args[]);
 void	ktrsysret(struct proc *, int, int, register_t);
 #endif /* __APPLE_API_PRIVATE */
 #else
diff --git a/bsd/sys/lock.h b/bsd/sys/lock.h
index 3563b766e..813bd4395 100644
--- a/bsd/sys/lock.h
+++ b/bsd/sys/lock.h
@@ -76,13 +76,6 @@
 
 #include <kern/locks.h>
 
-
-#if defined(thread_sleep_simple_lock)
-#undef thread_sleep_simple_lock
-#endif
-#define thread_sleep_simple_lock(l, e, i) thread_sleep_funnel((e), (i))
-
-
 #endif /* KERNEL */
 
 #ifdef BSD_KERNEL_PRIVATE
@@ -111,10 +104,6 @@ struct lock__bsd__ {
  * WARNING - keep in sync with lock__bsd__
  */
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_lock__bsd__ {
 	user_addr_t	lk_interlock[10];   /* lock on remaining fields */
 	u_int		lk_flags;			/* see below */
@@ -128,10 +117,6 @@ struct user_lock__bsd__ {
 	user_addr_t	lk_lockthread;		/* thread which acquired excl lock */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 /*
  * Lock request types:
  *   LK_SHARED - get one of many possible shared locks. If a process
diff --git a/bsd/sys/lockf.h b/bsd/sys/lockf.h
index 284299c9c..4ba9a3864 100644
--- a/bsd/sys/lockf.h
+++ b/bsd/sys/lockf.h
@@ -81,9 +81,7 @@ MALLOC_DECLARE(M_LOCKF);
  */
 TAILQ_HEAD(locklist, lockf);
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 struct lockf {
 	short	lf_flags;	    /* Semantics: F_POSIX, F_FLOCK, F_WAIT */
@@ -98,9 +96,7 @@ struct lockf {
 	TAILQ_ENTRY(lockf) lf_block;/* A request waiting for a lock */
 };
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 /* Maximum length of sleep chains to traverse to try and detect deadlock. */
 #define MAXDEPTH 50
diff --git a/bsd/sys/mbuf.h b/bsd/sys/mbuf.h
index 293a86515..80ef1c941 100644
--- a/bsd/sys/mbuf.h
+++ b/bsd/sys/mbuf.h
@@ -499,6 +499,7 @@ struct mbuf *m_pulldown(struct mbuf*, int, int, int*);
 struct mbuf *m_aux_add(struct mbuf *, int, int);
 struct mbuf *m_aux_find(struct mbuf *, int, int);
 void m_aux_delete(struct mbuf *, struct mbuf *);
+struct mbuf *m_aux_copy(struct mbuf *, struct mbuf *);
 
 struct mbuf *m_mclget(struct mbuf *, int);
 caddr_t m_mclalloc(int);
diff --git a/bsd/sys/mount.h b/bsd/sys/mount.h
index 114b5a405..6a98e3fdd 100644
--- a/bsd/sys/mount.h
+++ b/bsd/sys/mount.h
@@ -119,9 +119,7 @@ struct statfs {
 
 #define	MFSTYPENAMELEN	16	/* length of fs type name including null */
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 struct vfsstatfs {
 	uint32_t	f_bsize;	/* fundamental file system block size */
@@ -142,9 +140,7 @@ struct vfsstatfs {
 	void		*f_reserved[2];		/* For future use == 0 */
 };
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 #define VFSATTR_INIT(s)			((s)->f_supported = (s)->f_active = 0LL)
 #define VFSATTR_SET_SUPPORTED(s, a)	((s)->f_supported |= VFSATTR_ ## a)
@@ -183,9 +179,7 @@ struct vfsstatfs {
 /*
  * New VFS_STAT argument structure.
  */
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 struct vfs_attr {
 	uint64_t	f_supported;
@@ -223,9 +217,7 @@ struct vfs_attr {
 	uint16_t	f_carbon_fsid;	/* same as Carbon's FSVolumeInfo.filesystemID */
 };
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 /*
  * User specifiable flags.
@@ -350,32 +342,24 @@ struct vfsidctl {
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with vfsconf
  */
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_vfsconf {
        user_addr_t 	vfc_vfsops;				/* filesystem operations vector */
        char			vfc_name[MFSNAMELEN];	/* filesystem type name */
        int			vfc_typenum;            /* historic filesystem type number */
        int			vfc_refcount;           /* number mounted of this type */
        int			vfc_flags;				/* permanent flags */
-       user_addr_t 	vfc_mountroot;          /* if != NULL, routine to mount root */
+       user_addr_t 	vfc_mountroot __attribute((aligned(8)));	/* if != NULL, routine to mount root */
        user_addr_t 	vfc_next;				/* next in list */
 };
 
 struct user_vfsidctl {
        int				vc_vers;        /* should be VFSIDCTL_VERS1 (below) */
        fsid_t			vc_fsid;		/* fsid to operate on. */
-       user_addr_t 		vc_ptr;			/* pointer to data structure. */
+       user_addr_t 		vc_ptr __attribute((aligned(8)));	/* pointer to data structure. */
        user_size_t 		vc_len;			/* sizeof said structure. */
        u_int32_t		vc_spare[12];   /* spare (must be zero). */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif /* KERNEL */
 
 /*
diff --git a/bsd/sys/mount_internal.h b/bsd/sys/mount_internal.h
index 56b7e9929..baf82a077 100644
--- a/bsd/sys/mount_internal.h
+++ b/bsd/sys/mount_internal.h
@@ -232,14 +232,10 @@ struct vfsmount_args {
  * LP64 version of statfs structure.
  * NOTE - must be kept in sync with struct statfs in mount.h
  */
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_statfs {
 	short		f_otype;		/* TEMPORARY SHADOW COPY OF f_type */
 	short		f_oflags;		/* TEMPORARY SHADOW COPY OF f_flags */
-	user_long_t	f_bsize;		/* fundamental file system block size */
+	user_long_t	f_bsize __attribute((aligned(8)));		/* fundamental file system block size */
 	user_long_t	f_iosize;		/* optimal transfer block size */
 	user_long_t	f_blocks;		/* total data blocks in file system */
 	user_long_t	f_bfree;		/* free blocks in fs */
@@ -260,14 +256,10 @@ struct user_statfs {
 	user_long_t	f_reserved4[0];	/* For future use */
 #else
 	char		f_reserved3;	/* For alignment */
-	user_long_t	f_reserved4[4];	/* For future use */
+	user_long_t	f_reserved4[4] __attribute((aligned(8)));	/* For future use */
 #endif
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 __BEGIN_DECLS
 
 extern TAILQ_HEAD(mntlist, mount) mountlist;
@@ -289,7 +281,7 @@ errno_t	vfs_init_io_attributes(vnode_t, mount_t);
 int	vfs_mountroot(void);
 void	vfs_unmountall(void);
 int	safedounmount(struct mount *, int, struct proc *);
-int	dounmount(struct mount *, int, struct proc *);
+int	dounmount(struct mount *, int, int *, struct proc *);
 
 /* xnuy internal api */
 void  mount_dropcrossref(mount_t, vnode_t, int);
diff --git a/bsd/sys/namei.h b/bsd/sys/namei.h
index ac84da4fb..945efb2d5 100644
--- a/bsd/sys/namei.h
+++ b/bsd/sys/namei.h
@@ -215,7 +215,6 @@ int		reverse_lookup(vnode_t start_vp, vnode_t *lookup_vpp,
  * Stats on usefulness of namei caches.
  */
 struct	nchstats {
-        long	ncs_negtotal;
 	long	ncs_goodhits;		/* hits that we can really use */
 	long	ncs_neghits;		/* negative hits that we can use */
 	long	ncs_badhits;		/* hits we must drop */
diff --git a/bsd/sys/proc.h b/bsd/sys/proc.h
index 6411239bc..21d812cf1 100644
--- a/bsd/sys/proc.h
+++ b/bsd/sys/proc.h
@@ -183,7 +183,8 @@ struct extern_proc {
 #define	P_OWEUPC	0x00008000	/* Owe process an addupc() call at next ast. */
 
 #define	P_AFFINITY	0x00010000	/* xxx */
-#define	P_CLASSIC	0x00020000	/* xxx */
+#define	P_TRANSLATED	0x00020000	/* xxx */
+#define	P_CLASSIC	P_TRANSLATED	/* xxx */
 /*
 #define	P_FSTRACE	0x10000	/ * tracing via file system (elsewhere?) * /
 #define	P_SSTEP		0x20000	/ * process needs single-step fixup ??? * /
diff --git a/bsd/sys/proc_info.h b/bsd/sys/proc_info.h
new file mode 100644
index 000000000..7f8fa65ad
--- /dev/null
+++ b/bsd/sys/proc_info.h
@@ -0,0 +1,574 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _SYS_PROC_INFO_H
+#define _SYS_PROC_INFO_H
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mount.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <sys/kern_control.h>
+#include <net/if.h>
+#include <net/route.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+
+__BEGIN_DECLS
+
+
+#define PROC_ALL_PIDS 		1
+#define PROC_PGRP_ONLY		2
+#define PROC_TTY_ONLY		3
+#define PROC_UID_ONLY		4
+#define PROC_RUID_ONLY		5
+
+struct proc_bsdinfo {
+	uint32_t		pbi_flags;		/* 64bit; emulated etc */
+	uint32_t		pbi_status;
+	uint32_t		pbi_xstatus;
+	uint32_t		pbi_pid;
+	uint32_t		pbi_ppid;
+	uid_t			pbi_uid;
+	gid_t			pbi_gid;
+	uid_t			pbi_ruid;
+	gid_t			pbi_rgid;
+	uid_t			pbi_svuid;
+	gid_t			pbi_svgid;
+	char			pbi_comm[MAXCOMLEN + 1];
+	char			pbi_name[2*MAXCOMLEN + 1];	/* empty if no name is registered */
+	uint32_t		pbi_nfiles;
+	uint32_t		pbi_pgid;
+	uint32_t		pbi_pjobc;
+	uint32_t		e_tdev;			/* controlling tty dev */
+	uint32_t		e_tpgid;		/* tty process group id */
+	struct timeval 		pbi_start;
+	int32_t			pbi_nice;
+};
+
+
+
+/* pbi_flags values */
+#define PROC_FLAG_SYSTEM	1
+#define PROC_FLAG_TRACED	2
+#define PROC_FLAG_INEXIT	4
+#define PROC_FLAG_PPWAIT	8
+#define PROC_FLAG_LP64		0x10
+#define PROC_FLAG_SLEADER	0x20
+#define PROC_FLAG_CTTY		0x40
+#define PROC_FLAG_CONTROLT	0x80
+
+
+struct proc_taskinfo {
+	uint64_t		pti_virtual_size;   /* virtual memory size (bytes) */
+	uint64_t		pti_resident_size;  /* resident memory size (bytes) */
+	uint64_t		pti_total_user;		/* total time */
+	uint64_t		pti_total_system;
+	uint64_t		pti_threads_user;	/* existing threads only */
+	uint64_t		pti_threads_system;
+	int32_t			pti_policy;		/* default policy for new threads */
+	int32_t			pti_faults;		/* number of page faults */
+	int32_t			pti_pageins;	/* number of actual pageins */
+	int32_t			pti_cow_faults;	/* number of copy-on-write faults */
+	int32_t			pti_messages_sent;	/* number of messages sent */
+	int32_t			pti_messages_received; /* number of messages received */
+	int32_t			pti_syscalls_mach;  /* number of mach system calls */
+	int32_t			pti_syscalls_unix;  /* number of unix system calls */
+	int32_t			pti_csw;            /* number of context switches */
+	int32_t			pti_threadnum;		/* number of threads in the task */
+	int32_t			pti_numrunning;		/* number of running threads */
+	int32_t			pti_priority;		/* task priority*/
+};
+
+struct proc_taskallinfo {
+	struct  proc_bsdinfo pbsd;
+	struct  proc_taskinfo ptinfo;
+};
+
+
+struct proc_threadinfo {
+	uint64_t		pth_user_time;      /* user run time */
+	uint64_t		pth_system_time;    /* system run time */
+	int32_t			pth_cpu_usage;      /* scaled cpu usage percentage */
+	int32_t			pth_policy;		/* scheduling policy in effect */
+	int32_t			pth_run_state;      /* run state (see below) */
+	int32_t			pth_flags;          /* various flags (see below) */
+	int32_t			pth_sleep_time;     /* number of seconds that thread */
+	int32_t			pth_curpri;		/* cur priority*/
+	int32_t			pth_priority;		/*  priority*/
+	int32_t			pth_maxpriority;		/* max priority*/
+};
+
+struct proc_regioninfo {
+	uint32_t		pri_protection;
+	uint32_t		pri_max_protection;
+	uint32_t		pri_inheritance;
+	uint32_t		pri_flags;		/* shared, external pager, is submap */
+	uint64_t		pri_offset;
+	uint32_t		pri_behavior;
+	uint32_t		pri_user_wired_count;
+	uint32_t		pri_user_tag;
+	uint32_t		pri_pages_resident;
+	uint32_t		pri_pages_shared_now_private;
+    	uint32_t		pri_pages_swapped_out;
+    	uint32_t		pri_pages_dirtied;
+    	uint32_t		pri_ref_count;
+    	uint32_t		pri_shadow_depth;
+    	uint32_t		pri_share_mode;
+	uint32_t		pri_private_pages_resident;
+	uint32_t		pri_shared_pages_resident;
+	uint32_t		pri_obj_id;
+	uint64_t		pri_address;
+	uint64_t		pri_size;
+	uint32_t		pri_depth;
+};
+
+#define PROC_REGION_SUBMAP	1
+#define PROC_REGION_SHARED	2
+
+#define SM_COW             1
+#define SM_PRIVATE         2
+#define SM_EMPTY           3
+#define SM_SHARED          4
+#define SM_TRUESHARED      5
+#define SM_PRIVATE_ALIASED 6
+#define SM_SHARED_ALIASED  7
+
+
+/*
+ *	Thread run states (state field).
+ */
+
+#define TH_STATE_RUNNING	1	/* thread is running normally */
+#define TH_STATE_STOPPED	2	/* thread is stopped */
+#define TH_STATE_WAITING	3	/* thread is waiting normally */
+#define TH_STATE_UNINTERRUPTIBLE 4	/* thread is in an uninterruptible
+					   wait */
+#define TH_STATE_HALTED		5	/* thread is halted at a
+					   clean point */
+
+/*
+ *	Thread flags (flags field).
+ */
+#define TH_FLAGS_SWAPPED	0x1	/* thread is swapped out */
+#define TH_FLAGS_IDLE		0x2	/* thread is an idle thread */
+
+
+
+struct proc_fileinfo {
+	uint32_t		fi_openflags;
+	uint32_t		fi_status;	
+	off_t			fi_offset;
+	int32_t			fi_type;
+};
+
+
+struct vnode_info {
+	struct stat 	vi_stat;
+	int		vi_type;
+	fsid_t		vi_fsid;
+};
+
+struct vnode_info_path {
+	struct vnode_info	vip_vi;
+	char vip_path[MAXPATHLEN];  /* tail end of it  */
+};
+
+struct vnode_fdinfo {
+	struct proc_fileinfo pfi;
+	struct vnode_info	pvi;
+};
+
+struct vnode_fdinfowithpath {
+	struct proc_fileinfo pfi;
+	struct vnode_info_path pvip;
+
+};
+
+
+struct proc_regionwithpathinfo {
+	struct proc_regioninfo prp_prinfo;
+	struct vnode_info_path prp_vip;
+};
+
+struct proc_vnodepathinfo {
+	struct vnode_info_path pvi_cdir;
+	struct vnode_info_path pvi_rdir;
+};
+
+
+/*
+ *  Socket 
+ */
+
+
+/*
+ * IPv4 and IPv6 Sockets
+ */
+
+#define INI_IPV4        0x1
+#define INI_IPV6        0x2
+
+struct in4in6_addr {
+        u_int32_t       i46a_pad32[3];
+        struct  in_addr i46a_addr4;
+};
+
+struct in_sockinfo {
+	int						insi_fport;		/* foreign port */
+	int						insi_lport;		/* local port */
+	uint64_t				insi_gencnt;		/* generation count of this instance */
+	uint32_t				insi_flags;		/* generic IP/datagram flags */
+	uint32_t				insi_flow;
+
+	uint8_t					insi_vflag;		/* ini_IPV4 or ini_IPV6 */
+	uint8_t					insi_ip_ttl;	/* time to live proto */
+	/* protocol dependent part */
+	union {
+		struct in4in6_addr		ina_46;
+		struct in6_addr			ina_6;
+	} 						insi_faddr;		/* foreign host table entry */
+	union {
+		struct in4in6_addr		ina_46;
+		struct in6_addr			ina_6;
+	} 						insi_laddr;		/* local host table entry */
+	struct {
+		u_char 					in4_tos;	/* type of service  */
+	} 						insi_v4;
+	struct {
+		uint8_t					in6_hlim;
+		int						in6_cksum;
+		u_short					in6_ifindex;
+		short					in6_hops;
+	} 						insi_v6;
+};
+
+/*
+ * TCP Sockets
+ */
+
+#define TSI_T_REXMT			0		/* retransmit */
+#define TSI_T_PERSIST		1		/* retransmit persistence */
+#define TSI_T_KEEP			2		/* keep alive */
+#define TSI_T_2MSL			3		/* 2*msl quiet time timer */
+#define TSI_T_NTIMERS		4
+
+#define TSI_S_CLOSED		0       /* closed */
+#define TSI_S_LISTEN		1       /* listening for connection */
+#define TSI_S_SYN_SENT		2       /* active, have sent syn */
+#define TSI_S_SYN_RECEIVED	3       /* have send and received syn */
+#define TSI_S_ESTABLISHED	4       /* established */
+#define TSI_S__CLOSE_WAIT	5       /* rcvd fin, waiting for close */
+#define TSI_S_FIN_WAIT_1	6       /* have closed, sent fin */
+#define TSI_S_CLOSING		7       /* closed xchd FIN; await FIN ACK */
+#define TSI_S_LAST_ACK		8       /* had fin and close; await FIN ACK */
+#define TSI_S_FIN_WAIT_2	9       /* have closed, fin is acked */
+#define TSI_S_TIME_WAIT		10      /* in 2*msl quiet wait after close */
+#define TSI_S_RESERVED		11      /* pseudo state: reserved */
+
+struct tcp_sockinfo {
+	struct in_sockinfo 		tcpsi_ini;
+	int 					tcpsi_state;
+	int 					tcpsi_timer[TSI_T_NTIMERS];
+	int						tcpsi_mss;
+	uint32_t				tcpsi_flags;
+	uint64_t				tcpsi_tp;	/* opaque handle of TCP protocol control block */
+};
+
+/*
+ * Unix Domain Sockets
+ */
+
+
+struct un_sockinfo {
+	uint64_t					unsi_conn_so;	/* opaque handle of connected socket */
+	uint64_t					unsi_conn_pcb;	/* opaque handle of connected protocol control block */
+	union {
+		struct  sockaddr_un 	ua_sun;
+		char					ua_dummy[SOCK_MAXADDRLEN];
+	} 							unsi_addr;		/* bound address */
+	union {
+		struct  sockaddr_un 	ua_sun;
+		char					ua_dummy[SOCK_MAXADDRLEN];
+	} 							unsi_caddr;		/* address of socket connected to */
+};
+
+/*
+ * PF_NDRV Sockets
+ */
+
+struct ndrv_info {
+	uint32_t				ndrvsi_if_family;
+	uint32_t				ndrvsi_if_unit;
+	char					ndrvsi_if_name[IF_NAMESIZE];
+};
+
+/*
+ * Kernel Event Sockets
+ */
+
+struct kern_event_info {
+     uint32_t         		kesi_vendor_code_filter;
+     uint32_t         		kesi_class_filter;
+     uint32_t         		kesi_subclass_filter;
+};	
+
+/*
+ * Kernel Control Sockets
+ */
+
+struct kern_ctl_info {
+	uint32_t				kcsi_id;
+	uint32_t				kcsi_reg_unit;
+	uint32_t				kcsi_flags;                  /* support flags */
+	uint32_t				kcsi_recvbufsize;            /* request more than the default buffer size */
+	uint32_t				kcsi_sendbufsize;            /* request more than the default buffer size */
+	uint32_t				kcsi_unit;
+	char					kcsi_name[MAX_KCTL_NAME];	/* unique nke identifier, provided by DTS */
+};
+
+/* soi_state */
+
+#define SOI_S_NOFDREF				0x0001	/* no file table ref any more */
+#define SOI_S_ISCONNECTED			0x0002	/* socket connected to a peer */
+#define SOI_S_ISCONNECTING			0x0004	/* in process of connecting to peer */
+#define SOI_S_ISDISCONNECTING		0x0008	/* in process of disconnecting */
+#define SOI_S_CANTSENDMORE			0x0010	/* can't send more data to peer */
+#define SOI_S_CANTRCVMORE			0x0020	/* can't receive more data from peer */
+#define SOI_S_RCVATMARK				0x0040	/* at mark on input */
+#define SOI_S_PRIV					0x0080	/* privileged for broadcast, raw... */
+#define SOI_S_NBIO					0x0100	/* non-blocking ops */
+#define SOI_S_ASYNC					0x0200	/* async i/o notify */
+#define SOI_S_INCOMP				0x0800	/* Unaccepted, incomplete connection */
+#define SOI_S_COMP					0x1000	/* unaccepted, complete connection */
+#define SOI_S_ISDISCONNECTED		0x2000	/* socket disconnected from peer */
+#define SOI_S_DRAINING				0x4000	/* close waiting for blocked system calls to drain */
+
+struct sockbuf_info {
+	uint32_t			sbi_cc;
+	uint32_t			sbi_hiwat;			/* SO_RCVBUF, SO_SNDBUF */
+	uint32_t			sbi_mbcnt;
+	uint32_t			sbi_mbmax;
+	uint32_t			sbi_lowat;
+	short				sbi_flags;
+	short				sbi_timeo;
+};
+
+enum {
+	SOCKINFO_GENERIC 	= 0,
+	SOCKINFO_IN			= 1,
+	SOCKINFO_TCP		= 2,
+	SOCKINFO_UN			= 3,
+	SOCKINFO_NDRV		= 4,
+	SOCKINFO_KERN_EVENT	= 5,
+	SOCKINFO_KERN_CTL	= 6
+};
+
+struct socket_info {
+	struct stat 			soi_stat;
+	uint64_t				soi_so;		/* opaque handle of socket */
+	uint64_t				soi_pcb;	/* opaque handle of protocol control block */
+	int						soi_type;
+	int						soi_protocol;
+	int						soi_family;
+	short					soi_options;
+	short					soi_linger;
+	short					soi_state;
+	short					soi_qlen;
+	short					soi_incqlen;
+	short					soi_qlimit;
+	short					soi_timeo;
+	u_short					soi_error;
+	uint32_t				soi_oobmark;
+	struct sockbuf_info		soi_rcv;
+	struct sockbuf_info		soi_snd;
+	int						soi_kind;
+	union {
+		struct in_sockinfo		pri_in;			/* SOCKINFO_IN */
+		struct tcp_sockinfo		pri_tcp;		/* SOCKINFO_TCP */
+		struct un_sockinfo		pri_un;			/* SOCKINFO_UN */
+		struct ndrv_info		pri_ndrv;		/* SOCKINFO_NDRV */
+		struct kern_event_info	pri_kern_event;	/* SOCKINFO_KERN_EVENT */
+		struct kern_ctl_info	pri_kern_ctl;	/* SOCKINFO_KERN_CTL */
+	} 						soi_proto;
+};
+
+struct socket_fdinfo {
+	struct proc_fileinfo pfi;
+	struct socket_info  psi;
+};
+
+
+
+struct psem_info {
+	struct stat 	psem_stat;
+	char			psem_name[MAXPATHLEN];
+};
+
+struct psem_fdinfo {
+	struct proc_fileinfo pfi;
+	struct psem_info	pseminfo;
+};
+
+
+
+struct pshm_info  {
+	struct stat		pshm_stat;
+	uint64_t		pshm_mappaddr;
+	char			pshm_name[MAXPATHLEN];
+};
+
+struct pshm_fdinfo {
+	struct proc_fileinfo pfi;
+	struct pshm_info pshminfo;
+};
+
+
+struct pipe_info {
+	struct stat pipe_stat;
+	uint64_t	pipe_handle;
+	uint64_t	pipe_peerhandle;
+	int			pipe_status;
+};
+
+struct pipe_fdinfo {
+	struct proc_fileinfo pfi;
+	struct pipe_info	pipeinfo;
+};
+
+
+struct kqueue_info {
+	struct stat 	kq_stat;
+	uint32_t	kq_state;
+};
+#define PROC_KQUEUE_SELECT	1
+#define PROC_KQUEUE_SLEEP	2
+
+struct kqueue_fdinfo {
+	struct proc_fileinfo pfi;
+	struct kqueue_info	kqueueinfo;
+};
+
+struct appletalk_info {
+	struct stat 	atalk_stat;
+};
+
+struct appletalk_fdinfo {
+	struct proc_fileinfo pfi;
+	struct appletalk_info appletalkinfo;
+};
+
+
+
+/* defns of process file desc type */
+#define PROX_FDTYPE_ATALK 	0
+#define PROX_FDTYPE_VNODE 	1
+#define PROX_FDTYPE_SOCKET 	2
+#define PROX_FDTYPE_PSHM 	3
+#define PROX_FDTYPE_PSEM 	4
+#define PROX_FDTYPE_KQUEUE 	5
+#define PROX_FDTYPE_PIPE 	6
+#define PROX_FDTYPE_FSEVENTS 	7
+
+struct proc_fdinfo {
+	int32_t	proc_fd;
+	uint32_t proc_fdtype;	
+};
+
+/* Falvors for proc_pidinfo() */
+#define PROC_PIDLISTFDS 1
+#define PROC_PIDLISTFD_SIZE  (sizeof(struct proc_fdinfo))
+
+#define PROC_PIDTASKALLINFO 2
+#define PROC_PIDTASKALLINFO_SIZE  (sizeof(struct proc_taskallinfo))
+
+#define PROC_PIDTBSDINFO 3
+#define PROC_PIDTBSDINFO_SIZE  (sizeof(struct proc_bsdinfo))
+
+#define PROC_PIDTASKINFO 4
+#define PROC_PIDTASKINFO_SIZE  (sizeof(struct proc_taskinfo))
+
+#define PROC_PIDTHREADINFO 5
+#define PROC_PIDTHREADINFO_SIZE  (sizeof(struct proc_threadinfo))
+
+#define PROC_PIDLISTTHREADS 6
+#define PROC_PIDLISTTHREADS_SIZE  (2* sizeof(uint32_t))
+
+
+#define PROC_PIDREGIONINFO 7
+#define PROC_PIDREGIONINFO_SIZE  (sizeof(struct proc_regioninfo))
+
+#define PROC_PIDREGIONPATHINFO 8
+#define PROC_PIDREGIONPATHINFO_SIZE  (sizeof(struct proc_regionwithpathinfo))
+
+#define PROC_PIDVNODEPATHINFO 9
+#define PROC_PIDVNODEPATHINFO_SIZE  (sizeof(struct proc_vnodepathinfo))
+
+/* Flavors for proc_pidfdinfo */
+
+#define PROC_PIDFDVNODEINFO 1
+#define PROC_PIDFDVNODEINFO_SIZE  (sizeof(struct vnode_fdinfo))
+
+#define PROC_PIDFDVNODEPATHINFO 2
+#define PROC_PIDFDVNODEPATHINFO_SIZE  (sizeof(struct vnode_fdinfowithpath))
+
+#define PROC_PIDFDSOCKETINFO 3
+#define PROC_PIDFDSOCKETINFO_SIZE  (sizeof(struct socket_fdinfo))
+
+#define PROC_PIDFDPSEMINFO 4
+#define PROC_PIDFDPSEMINFO_SIZE  (sizeof(struct psem_fdinfo))
+
+#define PROC_PIDFDPSHMINFO 5
+#define PROC_PIDFDPSHMINFO_SIZE  (sizeof(struct pshm_fdinfo))
+
+#define PROC_PIDFDPIPEINFO 6
+#define PROC_PIDFDPIPEINFO_SIZE  (sizeof(struct pipe_fdinfo))
+
+#define PROC_PIDFDKQUEUEINFO 7
+#define PROC_PIDFDKQUEUEINFO_SIZE  (sizeof(struct kqueue_fdinfo))
+
+#define PROC_PIDFDATALKINFO 8
+#define PROC_PIDFDATALKINFO_SIZE  (sizeof(struct appletalk_fdinfo))
+
+
+
+#ifdef XNU_KERNEL_PRIVATE
+extern int fill_socketinfo(socket_t so, struct socket_info *si);
+extern int fill_pshminfo(struct pshmnode * pshm, struct pshm_info * pinfo);
+extern int fill_pseminfo(struct psemnode * psem, struct psem_info * pinfo);
+extern int fill_pipeinfo(struct pipe * cpipe, struct pipe_info * pinfo);
+extern int fill_kqueueinfo(struct kqueue * kq, struct kqueue_info * kinfo);
+#endif /* XNU_KERNEL_PRIVATE */
+
+
+__END_DECLS
+
+#endif /*_SYS_PROC_INFO_H */
diff --git a/bsd/sys/proc_internal.h b/bsd/sys/proc_internal.h
index 0583fab8c..d3806e04c 100644
--- a/bsd/sys/proc_internal.h
+++ b/bsd/sys/proc_internal.h
@@ -188,6 +188,7 @@ struct	proc {
 	u_char	p_usrpri;	/* User-priority based on p_cpu and p_nice. */
 	char	p_nice;		/* Process "nice" value. */
 	char	p_comm[MAXCOMLEN+1];
+	char	p_name[(2*MAXCOMLEN)+1];
 
 	struct 	pgrp *p_pgrp;	/* Pointer to process group. */
 
@@ -250,6 +251,7 @@ struct	proc {
 #define P_LREFDRAIN		0x40
 #define P_LREFDRAINWAIT		0x80
 #define P_LREFDEAD		0x100
+#define P_LTHSIGSTACK	0x200
 
 /* advisory flags in the proc */
 #define P_LADVLOCK		0x01
@@ -264,10 +266,6 @@ struct	proc {
 #ifdef KERNEL
 #include <sys/time.h>	/* user_timeval, user_itimerval */
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_extern_proc {
 	union {
 		struct {
@@ -284,7 +282,7 @@ struct user_extern_proc {
 	pid_t	p_oppid;		/* Save parent pid during ptrace. XXX */
 	int		p_dupfd;		/* Sideways return value from fdopen. XXX */
 	/* Mach related  */
-	user_addr_t user_stack;	/* where user stack was allocated */
+	user_addr_t user_stack __attribute((aligned(8)));	/* where user stack was allocated */
 	user_addr_t exit_thread;  /* XXX Which thread is exiting? */
 	int		p_debugger;		/* allow to debug */
 	boolean_t	sigwait;	/* indication to suspend */
@@ -292,7 +290,7 @@ struct user_extern_proc {
 	u_int	p_estcpu;	 /* Time averaged value of p_cpticks. */
 	int		p_cpticks;	 /* Ticks of cpu time. */
 	fixpt_t	p_pctcpu;	 /* %cpu for this process during p_swtime */
-	user_addr_t	p_wchan;	 /* Sleep address. */
+	user_addr_t	p_wchan __attribute((aligned(8)));	 /* Sleep address. */
 	user_addr_t	p_wmesg;	 /* Reason for sleep. */
 	u_int	p_swtime;	 /* Time swapped in or out. */
 	u_int	p_slptime;	 /* Time since last blocked. */
@@ -302,9 +300,9 @@ struct user_extern_proc {
 	u_quad_t p_sticks;		/* Statclock hits in system mode. */
 	u_quad_t p_iticks;		/* Statclock hits processing intr. */
 	int		p_traceflag;		/* Kernel trace points. */
-	user_addr_t	p_tracep;	/* Trace to vnode. */
+	user_addr_t	p_tracep __attribute((aligned(8)));	/* Trace to vnode. */
 	int		p_siglist;		/* DEPRECATED */
-	user_addr_t	p_textvp;	/* Vnode of executable. */
+	user_addr_t	p_textvp __attribute((aligned(8)));	/* Vnode of executable. */
 	int		p_holdcnt;		/* If non-zero, don't swap. */
 	sigset_t p_sigmask;	/* DEPRECATED. */
 	sigset_t p_sigignore;	/* Signals being ignored. */
@@ -313,16 +311,12 @@ struct user_extern_proc {
 	u_char	p_usrpri;	/* User-priority based on p_cpu and p_nice. */
 	char	p_nice;		/* Process "nice" value. */
 	char	p_comm[MAXCOMLEN+1];
-	user_addr_t	p_pgrp;	/* Pointer to process group. */
+	user_addr_t	p_pgrp __attribute((aligned(8)));	/* Pointer to process group. */
 	user_addr_t	p_addr;	/* Kernel virtual addr of u-area (PROC ONLY). */
 	u_short	p_xstat;	/* Exit status for wait; also stop signal. */
 	u_short	p_acflag;	/* Accounting flags. */
-	user_addr_t	p_ru;	/* Exit information. XXX */
+	user_addr_t	p_ru __attribute((aligned(8)));	/* Exit information. XXX */
 };
-
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
 #endif	/* KERNEL */
 
 /*
@@ -330,6 +324,7 @@ struct user_extern_proc {
  * as it is used to represent "no process group".
  */
 extern int nprocs, maxproc;		/* Current and max number of procs. */
+extern int maxprocperuid;		/* Current number of procs per uid */
 __private_extern__ int hard_maxproc;	/* hard limit */
 
 #define	PID_MAX		30000
diff --git a/bsd/sys/protosw.h b/bsd/sys/protosw.h
index 50aa9af07..17c791118 100644
--- a/bsd/sys/protosw.h
+++ b/bsd/sys/protosw.h
@@ -112,9 +112,7 @@ struct socket_filter;
 #include <kern/locks.h>
 #endif /* KERNEL */
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 struct protosw {
 	short	pr_type;		/* socket type used for */
@@ -162,9 +160,7 @@ struct protosw {
 #endif
 };
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 /*
  * Values for pr_flags.
diff --git a/bsd/sys/quota.h b/bsd/sys/quota.h
index 768f51ca1..570dde533 100644
--- a/bsd/sys/quota.h
+++ b/bsd/sys/quota.h
@@ -178,10 +178,6 @@ struct dqblk {
  * WARNING - keep in sync with struct dqblk
  */
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_dqblk {
 	u_int64_t dqb_bhardlimit;	/* absolute limit on disk bytes alloc */
 	u_int64_t dqb_bsoftlimit;	/* preferred limit on disk bytes */
@@ -189,15 +185,11 @@ struct user_dqblk {
 	u_int32_t dqb_ihardlimit;	/* maximum # allocated inodes + 1 */
 	u_int32_t dqb_isoftlimit;	/* preferred inode limit */
 	u_int32_t dqb_curinodes;	/* current # allocated inodes */
-	user_time_t	  dqb_btime;		/* time limit for excessive disk use */
+	user_time_t	  dqb_btime __attribute((aligned(8)));		/* time limit for excessive disk use */
 	user_time_t	  dqb_itime;		/* time limit for excessive files */
 	u_int32_t dqb_id;		/* identifier (0 for empty entries) */
 	u_int32_t dqb_spare[4];		/* pad struct to power of 2 */
 };
-
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
 #endif  /* KERNEL_PRIVATE */
 
 #define INITQMAGICS { \
@@ -297,8 +289,8 @@ struct quotafile {
 struct dquot {
 	LIST_ENTRY(dquot) dq_hash;	/* hash list */
 	TAILQ_ENTRY(dquot) dq_freelist;	/* free list */
+	uint32_t  dq_cnt;		/* count of active references */
 	u_int16_t dq_flags;		/* flags, see below */
-	u_int16_t dq_cnt;		/* count of active references */
         u_int16_t dq_lflags;		/* protected by the quota list lock */
 	u_int16_t dq_type;		/* quota type of this dquot */
 	u_int32_t dq_id;		/* identifier this applies to */
diff --git a/bsd/sys/reboot.h b/bsd/sys/reboot.h
index 0b8f3eb20..d99f572ec 100644
--- a/bsd/sys/reboot.h
+++ b/bsd/sys/reboot.h
@@ -86,6 +86,7 @@
 #define RB_DFLTROOT	0x20	/* use compiled-in rootdev */
 #define RB_ALTBOOT	0x40	/* use /boot.old vs /boot */
 #define RB_UNIPROC	0x80	/* don't start slaves */
+#define RB_UPSDELAY 0x200   /* Delays restart by 5 minutes */
 #define RB_PANIC	0	/* reboot due to panic */
 #define RB_BOOT		1	/* reboot due to boot() */
 
diff --git a/bsd/sys/resource.h b/bsd/sys/resource.h
index 65cece77f..622ad92c5 100644
--- a/bsd/sys/resource.h
+++ b/bsd/sys/resource.h
@@ -179,13 +179,9 @@ struct	rusage {
  * WARNING - keep in sync with struct timeval
  */
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_rusage_timeval {
 	user_time_t		tv_sec;		/* seconds */
-	__darwin_suseconds_t	tv_usec;	/* and microseconds */
+	__darwin_suseconds_t	tv_usec __attribute((aligned(8)));	/* and microseconds */
 };	
 struct	user_rusage {
 	struct user_rusage_timeval ru_utime;	/* user time used */
@@ -206,10 +202,6 @@ struct	user_rusage {
 	user_long_t	ru_nivcsw;		/* involuntary " */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif // KERNEL
 
 
diff --git a/bsd/sys/shm_internal.h b/bsd/sys/shm_internal.h
index 799cb9604..98dd9f8b9 100644
--- a/bsd/sys/shm_internal.h
+++ b/bsd/sys/shm_internal.h
@@ -95,13 +95,15 @@ struct user_shmid_ds {
  * might be of interest to user programs.  Also part of the ipcs interface.
  * Note: use of user_ssize_t intentional: permits 32 bit ipcs to provide
  * information about 64 bit programs shared segments.
+ * Note 2: user_ssize_t is not that useful if it's not 64-bit,
+ * so let's use int64_t explicitely...
  */
 struct shminfo {
-	user_ssize_t	shmmax;		/* max shm segment size (bytes) */
-	user_ssize_t	shmmin;		/* min shm segment size (bytes) */
-	user_ssize_t	shmmni;		/* max number of shm identifiers */
-	user_ssize_t	shmseg;		/* max shm segments per process */
-	user_ssize_t	shmall;		/* max amount of shm (pages) */
+	int64_t	shmmax;		/* max shm segment size (bytes) */
+	int64_t	shmmin;		/* min shm segment size (bytes) */
+	int64_t	shmmni;		/* max number of shm identifiers */
+	int64_t	shmseg;		/* max shm segments per process */
+	int64_t	shmall;		/* max amount of shm (pages) */
 };
 
 #ifdef KERNEL
diff --git a/bsd/sys/signal.h b/bsd/sys/signal.h
index ea489715e..9ccc3d464 100644
--- a/bsd/sys/signal.h
+++ b/bsd/sys/signal.h
@@ -210,10 +210,6 @@ struct sigevent {
 // LP64todo - should this move?
 #ifdef BSD_KERNEL_PRIVATE
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 union user_sigval {
 	struct {
 		int		pad;	/* assumes Motorolla byte order */
@@ -230,10 +226,6 @@ struct user_sigevent {
 	user_addr_t 	sigev_notify_attributes;	/* Notify attributes */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif	/* BSD_KERNEL_PRIVATE */
 
 typedef struct __siginfo {
@@ -251,10 +243,6 @@ typedef struct __siginfo {
 
 #ifdef BSD_KERNEL_PRIVATE
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 typedef struct __user_siginfo {
 	int		si_signo;	/* signal number */
 	int		si_errno;	/* errno association */
@@ -268,10 +256,6 @@ typedef struct __user_siginfo {
 	user_ulong_t	pad[7];		/* Reserved for Future Use */
 } user_siginfo_t;
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif	/* BSD_KERNEL_PRIVATE */
 
 /* 
@@ -377,10 +361,6 @@ struct	sigaction {
 #ifdef	BSD_KERNEL_PRIVATE
 #include <machine/types.h>
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 union __user_sigaction_u {
 	user_addr_t	__sa_handler;
 	user_addr_t	__sa_sigaction;
@@ -399,10 +379,6 @@ struct	__user_sigaction {
 	int	sa_flags;		/* see signal options below */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #undef SIG_DFL
 #undef SIG_IGN
 #undef SIG_ERR
@@ -454,20 +430,12 @@ typedef	void (*sig_t)(int);	/* type of signal function */
  */
 #ifdef	BSD_KERNEL_PRIVATE
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct  user_sigaltstack {
 	user_addr_t	ss_sp;		/* signal stack base */
 	user_size_t	ss_size;	/* signal stack length */
-	int		ss_flags;	/* SA_DISABLE and/or SA_ONSTACK */
+	int		ss_flags __attribute((aligned(8)));	/* SA_DISABLE and/or SA_ONSTACK */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif	/* BSD_KERNEL_PRIVATE */
 
 #ifndef _STACK_T
diff --git a/bsd/sys/socket.h b/bsd/sys/socket.h
index 5e1b2545a..d28b5fd73 100644
--- a/bsd/sys/socket.h
+++ b/bsd/sys/socket.h
@@ -491,24 +491,17 @@ struct msghdr {
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with struct msghdr
  */
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
 
 struct user_msghdr {
 	user_addr_t	msg_name;		/* optional address */
 	socklen_t	msg_namelen;		/* size of address */
-	user_addr_t	msg_iov;		/* scatter/gather array */
+	user_addr_t	msg_iov __attribute((aligned(8)));		/* scatter/gather array */
 	int		msg_iovlen;		/* # elements in msg_iov */
-	user_addr_t	msg_control;		/* ancillary data, see below */
+	user_addr_t	msg_control __attribute((aligned(8)));		/* ancillary data, see below */
 	socklen_t	msg_controllen;		/* ancillary data buffer len */
 	int		msg_flags;		/* flags on received message */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif // KERNEL
 
 #define	MSG_OOB		0x1		/* process out-of-band data */
diff --git a/bsd/sys/socketvar.h b/bsd/sys/socketvar.h
index e1b610dce..3a5c34b8a 100644
--- a/bsd/sys/socketvar.h
+++ b/bsd/sys/socketvar.h
@@ -210,6 +210,14 @@ struct socket {
 	int	so_usecount;		/* refcounting of socket use */;
 	int	so_retaincnt;
 	u_int32_t	so_filteruse; /* usecount for the socket filters */
+/* for debug pruposes */
+#define SO_LCKDBG_MAX		4	/* number of debug locking Link Registers recorded */
+	u_int32_t lock_lr[SO_LCKDBG_MAX];	/* socket locking calling history */ 
+	int	next_lock_lr;
+	u_int32_t unlock_lr[SO_LCKDBG_MAX];	/* socket unlocking caller history */
+	int	next_unlock_lr;
+	
+
 	void	*reserved3;		/* Temporarily in use/debug: last socket lock LR */
 	void	*reserved4;		/* Temporarily in use/debug: last socket unlock LR */
 	thread_t	so_send_filt_thread;
@@ -375,7 +383,6 @@ int	sbappendcontrol(struct sockbuf *sb, struct mbuf *m0,
 	    struct mbuf *control, int *error_out);
 int	sbappendrecord(struct sockbuf *sb, struct mbuf *m0);
 void	sbcheck(struct sockbuf *sb);
-int	sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n);
 struct mbuf *
 	sbcreatecontrol(caddr_t p, int size, int type, int level);
 void	sbdrop(struct sockbuf *sb, int len);
diff --git a/bsd/sys/stat.h b/bsd/sys/stat.h
index ac8146e0b..d983d0cec 100644
--- a/bsd/sys/stat.h
+++ b/bsd/sys/stat.h
@@ -128,6 +128,7 @@ typedef __darwin_off_t		off_t;
 typedef	__darwin_time_t		time_t;
 #endif
 
+
 /* [XSI] The timespec structure may be defined as described in <time.h> */
 #ifndef _TIMESPEC
 #define _TIMESPEC
@@ -135,6 +136,7 @@ struct timespec {
 	time_t	tv_sec;		/* seconds */
 	long	tv_nsec;	/* and nanoseconds */
 };
+
 // LP64todo - should this move?
 #ifdef KERNEL
 /* LP64 version of struct timespec.  time_t is a long and must grow when 
@@ -143,7 +145,7 @@ struct timespec {
  */
 struct user_timespec {
 	user_time_t	tv_sec;		/* seconds */
-	__int64_t	tv_nsec;	/* and nanoseconds */
+	int32_t	tv_nsec __attribute((aligned(8)));	/* and nanoseconds */
 };
 #endif // KERNEL
 #endif	/* _TIMESPEC */
@@ -218,9 +220,6 @@ struct stat {
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with struct stat
  */
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
 
 struct user_stat {
 	dev_t	 	st_dev;		/* [XSI] ID of device containing file */
@@ -251,10 +250,6 @@ struct user_stat {
 	__int64_t	st_qspare[2];	/* RESERVED: DO NOT USE! */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 extern void munge_stat(struct stat *sbp, struct user_stat *usbp);
 
 #endif // KERNEL
diff --git a/bsd/sys/sys_domain.h b/bsd/sys/sys_domain.h
index c4642c6ec..6b8b01b7f 100644
--- a/bsd/sys/sys_domain.h
+++ b/bsd/sys/sys_domain.h
@@ -33,6 +33,13 @@
 
 #include <sys/appleapiopts.h>
 #include <sys/cdefs.h>
+#include <sys/types.h>
+
+#ifdef KERNEL_PRIVATE
+#include <sys/queue.h>
+#include <libkern/locks.h>
+#include <sys/kern_control.h>
+#endif /* KERNEL_PRIVATE */
 
 /* Kernel Events Protocol */ 
 #define SYSPROTO_EVENT 		1	/* kernel events protocol */
@@ -44,16 +51,56 @@
 /* System family socket address */
 struct sockaddr_sys
 {
-    u_char	ss_len;		/* sizeof(struct sockaddr_sys) */
-    u_char	ss_family;	/* AF_SYSTEM */
-    u_int16_t 	ss_sysaddr; 	/* protocol address in AF_SYSTEM */
-    u_int32_t 	ss_reserved[7]; /* reserved to the protocol use */
+	u_char	ss_len;				/* sizeof(struct sockaddr_sys) */
+	u_char	ss_family;			/* AF_SYSTEM */
+	u_int16_t 	ss_sysaddr; 	/* protocol address in AF_SYSTEM */
+	u_int32_t 	ss_reserved[7]; /* reserved to the protocol use */
 };
 
 
-#ifdef KERNEL
 #ifdef KERNEL_PRIVATE
 
+/*
+ * internal structure maintained for each register controller
+ */
+struct ctl_cb;
+struct socket;
+
+struct kctl
+{
+	TAILQ_ENTRY(kctl)		next;					/* controller chain */
+
+	/* controller information provided when registering */
+	char					name[MAX_KCTL_NAME];	/* unique nke identifier, provided by DTS */
+	u_int32_t				id;
+	u_int32_t				reg_unit;
+
+	/* misc communication information */
+	u_int32_t				flags;					/* support flags */
+	u_int32_t				recvbufsize;			/* request more than the default buffer size */
+	u_int32_t				sendbufsize;			/* request more than the default buffer size */
+
+	/* Dispatch functions */
+	ctl_connect_func		connect;				/* Make contact */
+	ctl_disconnect_func		disconnect;				/* Break contact */
+	ctl_send_func			send;					/* Send data to nke */
+	ctl_setopt_func			setopt;					/* set kctl configuration */
+	ctl_getopt_func			getopt;					/* get kctl configuration */
+
+	TAILQ_HEAD(, ctl_cb)	kcb_head;
+	u_int32_t				lastunit;
+};
+
+struct ctl_cb {
+	TAILQ_ENTRY(ctl_cb)		next;					/* controller chain */
+	lck_mtx_t				*mtx;
+	struct socket			*so;					/* controlling socket */
+	struct kctl				*kctl;					/* back pointer to controller */
+	u_int32_t				unit;
+	void					*userdata;
+};
+
+
 extern struct domain systemdomain;
 
 /* built in system domain protocols init function */
@@ -63,7 +110,6 @@ int kern_control_init(void);
 __END_DECLS
 
 #endif /* KERNEL_PRIVATE */
-#endif /* KERNEL */
 
 #endif /* _SYSTEM_DOMAIN_H_ */
 
diff --git a/bsd/sys/syscall.h b/bsd/sys/syscall.h
index c23c4c1ad..f58cebb40 100644
--- a/bsd/sys/syscall.h
+++ b/bsd/sys/syscall.h
@@ -144,11 +144,7 @@
 #define	SYS_getpriority    100
 			/* 101  old send */
 			/* 102  old recv */
-#ifdef __ppc__
 			/* 103  old sigreturn */
-#else
-#define	SYS_sigreturn      103
-#endif
 #define	SYS_bind           104
 #define	SYS_setsockopt     105
 #define	SYS_listen         106
@@ -161,12 +157,7 @@
 			/* 113  old recvmsg */
 			/* 114  old sendmsg */
 			/* 115  old vtrace */
-#ifdef __ppc__
-#define	SYS_ppc_gettimeofday 116
-#define	SYS_gettimeofday 116
-#else
 #define	SYS_gettimeofday   116
-#endif
 #define	SYS_getrusage      117
 #define	SYS_getsockopt     118
 			/* 119  old resuba */
@@ -242,12 +233,8 @@
 #define	SYS_setgid         181
 #define	SYS_setegid        182
 #define	SYS_seteuid        183
-#ifdef __ppc__
 #define	SYS_sigreturn      184
-#else
-			/* 184  */
-#endif
-			/* 185  */
+#define	SYS_chud           185
 			/* 186  */
 			/* 187  */
 #define	SYS_stat           188
@@ -272,16 +259,6 @@
 #define	SYS_mlock          203
 #define	SYS_munlock        204
 #define	SYS_undelete       205
-#ifdef __ppc__
-#define	SYS_ATsocket       206
-#define	SYS_ATgetmsg       207
-#define	SYS_ATputmsg       208
-#define	SYS_ATPsndreq      209
-#define	SYS_ATPsndrsp      210
-#define	SYS_ATPgetreq      211
-#define	SYS_ATPgetrsp      212
-			/* 213  Reserved for AppleTalk */
-#else
 #define	SYS_ATsocket       206
 #define	SYS_ATgetmsg       207
 #define	SYS_ATputmsg       208
@@ -290,7 +267,6 @@
 #define	SYS_ATPgetreq      211
 #define	SYS_ATPgetrsp      212
 			/* 213  Reserved for AppleTalk */
-#endif /* __ppc__ */
 #define	SYS_kqueue_from_portset_np 214
 #define	SYS_kqueue_portset_np 215
 #define	SYS_mkcomplex      216
@@ -422,7 +398,7 @@
 #define	SYS___pthread_canceled 333
 #define	SYS___semwait_signal 334
 #define	SYS_utrace         335
-			/* 336  */
+#define	SYS_proc_info      336
 			/* 337  */
 			/* 338  */
 			/* 339  */
@@ -451,7 +427,7 @@
 #define	SYS_kqueue         362
 #define	SYS_kevent         363
 #define	SYS_lchown         364
-			/* 365  */
+#define	SYS_stack_snapshot 365
 			/* 366  */
 			/* 367  */
 			/* 368  */
diff --git a/bsd/sys/sysctl.h b/bsd/sys/sysctl.h
index e7e11accc..32d9a4fe8 100644
--- a/bsd/sys/sysctl.h
+++ b/bsd/sys/sysctl.h
@@ -356,8 +356,10 @@ SYSCTL_DECL(_user);
 #define	KERN_PANICINFO		41	/* node: panic UI information */
 #define	KERN_SYSV		42	/* node: System V IPC information */
 #define KERN_AFFINITY		43	/* xxx */
-#define KERN_CLASSIC	   	44	/* xxx */
-#define KERN_CLASSICHANDLER	45	/* xxx */
+#define KERN_TRANSLATE	   	44	/* xxx */
+#define KERN_CLASSIC	   	KERN_TRANSLATE	/* XXX backwards compat */
+#define KERN_EXEC		45	/* xxx */
+#define KERN_CLASSICHANDLER	KERN_EXEC /* XXX backwards compatibility */
 #define	KERN_AIOMAX		46	/* int: max aio requests */
 #define	KERN_AIOPROCMAX		47	/* int: max aio requests per process */
 #define	KERN_AIOTHREADS		48	/* int: max aio worker threads */
@@ -373,8 +375,12 @@ SYSCTL_DECL(_user);
 #define	KERN_LOW_PRI_WINDOW	56	/* int: set/reset throttle window - milliseconds */
 #define	KERN_LOW_PRI_DELAY	57	/* int: set/reset throttle delay - milliseconds */
 #define	KERN_POSIX		58	/* node: posix tunables */
-#define	KERN_USRSTACK64	59	/* LP64 user stack query */
-#define	KERN_MAXID		60	/* number of valid kern ids */
+#define	KERN_USRSTACK64		59	/* LP64 user stack query */
+#define KERN_NX_PROTECTION	60	/* int: whether no-execute protection is enabled */
+#define	KERN_TFP 		61	/* Task for pid settings */
+#define	KERN_PROCNAME 		62	/* setup process program  name(2*MAXCOMLEN) */
+#define	KERN_THALTSTACK 	63	/* setup process to have per thread sigaltstack */
+#define	KERN_MAXID		64	/* number of valid kern ids */
 
 #if defined(__LP64__)
 #define	KERN_USRSTACK KERN_USRSTACK64
@@ -382,6 +388,16 @@ SYSCTL_DECL(_user);
 #define	KERN_USRSTACK KERN_USRSTACK32
 #endif
 
+/* KERN_TFP types */
+#define KERN_TFP_POLICY 		1
+#define KERN_TFP_READ_GROUP 	2
+#define KERN_TFP_RW_GROUP 		3
+
+/* KERN_TFP_POLICY values . All policies allow task port for self */
+#define KERN_TFP_POLICY_DENY 		0 	/* Deny Mode: None allowed except privileged */
+#define KERN_TFP_POLICY_PERMISSIVE 	1	/* Permissive Mode: related ones allowed or privileged */
+#define KERN_TFP_POLICY_RESTRICTED 	2	/* Restricted Mode: privileged or setgid and realted */
+
 /* KERN_KDEBUG types */
 #define KERN_KDEFLAGS		1
 #define KERN_KDDFLAGS		2
@@ -475,7 +491,7 @@ SYSCTL_DECL(_user);
 	{ "sysv", CTLTYPE_NODE }, \
 	{ "dummy", CTLTYPE_INT }, \
 	{ "dummy", CTLTYPE_INT }, \
-	{ "dummy", CTLTYPE_INT }, \
+	{ "exec", CTLTYPE_NODE }, \
 	{ "aiomax", CTLTYPE_INT }, \
 	{ "aioprocmax", CTLTYPE_INT }, \
 	{ "aiothreads", CTLTYPE_INT }, \
@@ -488,7 +504,12 @@ SYSCTL_DECL(_user);
 	{ "proc_low_pri_io", CTLTYPE_INT }, \
 	{ "low_pri_window", CTLTYPE_INT }, \
 	{ "low_pri_delay", CTLTYPE_INT }, \
-	{ "posix", CTLTYPE_NODE } \
+	{ "posix", CTLTYPE_NODE }, \
+	{ "usrstack64", CTLTYPE_QUAD }, \
+	{ "nx", CTLTYPE_INT }, \
+	{ "tfp", CTLTYPE_NODE }, \
+	{ "procname", CTLTYPE_STRING }, \
+	{ "threadsigaltstack", CTLTYPE_INT } \
 }
 
 /*
@@ -570,10 +591,6 @@ struct kinfo_proc {
  * WARNING - keep in sync with _pcred
  */
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_pcred {
 	char	pc_lock[72];		/* opaque content */
 	user_addr_t	pc_ucred;	/* Current credentials. */
@@ -581,7 +598,7 @@ struct user_pcred {
 	uid_t	p_svuid;		/* Saved effective user id. */
 	gid_t	p_rgid;			/* Real group id. */
 	gid_t	p_svgid;		/* Saved effective group id. */
-	int	p_refcnt;		/* Number of references. */
+	int	p_refcnt __attribute((aligned(8)));		/* Number of references. */
 };
 
 /* LP64 version of kinfo_proc.  all pointers 
@@ -601,7 +618,7 @@ struct user_kinfo_proc {
 		short	e_jobc;			/* job control counter */
 		dev_t	e_tdev;			/* controlling tty dev */
 		pid_t	e_tpgid;		/* tty process group id */
-		user_addr_t	e_tsess;	/* tty session pointer */
+		user_addr_t	e_tsess __attribute((aligned(8)));	/* tty session pointer */
 		char	e_wmesg[WMESGLEN+1];	/* wchan message */
 		segsz_t e_xsize;		/* text size */
 		short	e_xrssize;		/* text rss */
@@ -613,10 +630,6 @@ struct user_kinfo_proc {
 	} kp_eproc;
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif	/* BSD_KERNEL_PRIVATE */
 
 #endif /* __APPLE_API_UNSTABLE */
@@ -677,18 +690,11 @@ extern struct loadavg averunnable;
 // LP64todo - should this move?
 #ifdef BSD_KERNEL_PRIVATE
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
 struct user_loadavg {
 	fixpt_t	ldavg[3];
-	user_long_t	fscale;
+        user_long_t	fscale __attribute((aligned(8)));
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif	/* BSD_KERNEL_PRIVATE */
 #endif /* __APPLE_API_PRIVATE */
 
@@ -805,9 +811,11 @@ struct user_loadavg {
  *   hw.l3cachesize            -
  *
  *
- * These are the selectors for optional processor features.  Selectors that return errors are not support on the system.
- * Supported features will return 1 if they are recommended or 0 if they are supported but are not expected to help performance.
- * Future versions of these selectors may return larger values as necessary so it is best to test for non zero.
+ * These are the selectors for optional processor features for specific processors.  Selectors that return errors are not support 
+ * on the system.  Supported features will return 1 if they are recommended or 0 if they are supported but are not expected to help .
+ * performance.  Future versions of these selectors may return larger values as necessary so it is best to test for non zero.
+ *
+ * For PowerPC:
  *
  *   hw.optional.floatingpoint - Floating Point Instructions
  *   hw.optional.altivec       - AltiVec Instructions
@@ -819,6 +827,14 @@ struct user_loadavg {
  *   hw.optional.datastreams   - Data Streams Instructions
  *   hw.optional.dcbtstreams   - Data Cache Block Touch Steams Instruction Form
  *
+ * For x86 Architecture:
+ * 
+ *   hw.optional.floatingpoint - Floating Point Instructions
+ *   hw.optional.mmx           - Original MMX vector instructions
+ *   hw.optional.sse           - Streaming SIMD Extensions
+ *   hw.optional.sse2          - Streaming SIMD Extensions 2
+ *   hw.optional.sse3          - Streaming SIMD Extensions 3
+ *   hw.optional.x86_64        - 64-bit support
  */
 
 
diff --git a/bsd/sys/syslimits.h b/bsd/sys/syslimits.h
index e63d3cf4f..8719ad3c6 100644
--- a/bsd/sys/syslimits.h
+++ b/bsd/sys/syslimits.h
@@ -71,7 +71,7 @@
  * compile time; you *cannot* set it higher than the hard limit!!
  */
 #define	ARG_MAX		   (256 * 1024)	/* max bytes for an exec function */
-#define	CHILD_MAX		   100	/* max simultaneous processes */
+#define	CHILD_MAX		   266	/* max simultaneous processes */
 #define	GID_MAX		   2147483647U	/* max value for a gid_t (2^31-2) */
 #define	LINK_MAX		32767	/* max file link count */
 #define	MAX_CANON		  255	/* max bytes in term canon input line */
diff --git a/bsd/sys/sysproto.h b/bsd/sys/sysproto.h
index cc26b437d..ec5eaa8a6 100644
--- a/bsd/sys/sysproto.h
+++ b/bsd/sys/sysproto.h
@@ -46,13 +46,8 @@
 
 #ifdef KERNEL
 #ifdef __APPLE_API_PRIVATE
-#ifdef __ppc__
 #define	PAD_(t)	(sizeof(uint64_t) <= sizeof(t) \
  		? 0 : sizeof(uint64_t) - sizeof(t))
-#else
-#define	PAD_(t)	(sizeof(register_t) <= sizeof(t) \
- 		? 0 : sizeof(register_t) - sizeof(t))
-#endif
 #if BYTE_ORDER == LITTLE_ENDIAN
 #define	PADL_(t)	0
 #define	PADR_(t)	PAD_(t)
@@ -64,7 +59,6 @@
 __BEGIN_DECLS
 #ifndef __MUNGE_ONCE
 #define __MUNGE_ONCE
-#ifdef __ppc__
 void munge_w(const void *, void *);  
 void munge_ww(const void *, void *);  
 void munge_www(const void *, void *);  
@@ -73,31 +67,25 @@ void munge_wwwww(const void *, void *);
 void munge_wwwwww(const void *, void *);  
 void munge_wwwwwww(const void *, void *);  
 void munge_wwwwwwww(const void *, void *);  
-void munge_d(const void *, void *);  
-void munge_dd(const void *, void *);  
-void munge_ddd(const void *, void *);  
-void munge_dddd(const void *, void *);  
-void munge_ddddd(const void *, void *);  
-void munge_dddddd(const void *, void *);  
-void munge_ddddddd(const void *, void *);  
-void munge_dddddddd(const void *, void *);  
 void munge_wl(const void *, void *);  
 void munge_wlw(const void *, void *);  
 void munge_wwwl(const void *, void *);  
+void munge_wwwlww(const void *, void *); 
 void munge_wwwwl(const void *, void *);  
 void munge_wwwwwl(const void *, void *);  
 void munge_wsw(const void *, void *);  
 void munge_wws(const void *, void *);  
 void munge_wwwsw(const void *, void *);  
+#ifdef __ppc__
+void munge_d(const void *, void *);  
+void munge_dd(const void *, void *);  
+void munge_ddd(const void *, void *);  
+void munge_dddd(const void *, void *);  
+void munge_ddddd(const void *, void *);  
+void munge_dddddd(const void *, void *);  
+void munge_ddddddd(const void *, void *);  
+void munge_dddddddd(const void *, void *);  
 #else 
-#define munge_w  NULL 
-#define munge_ww  NULL 
-#define munge_www  NULL 
-#define munge_wwww  NULL 
-#define munge_wwwww  NULL 
-#define munge_wwwwww  NULL 
-#define munge_wwwwwww  NULL 
-#define munge_wwwwwwww  NULL 
 #define munge_d  NULL 
 #define munge_dd  NULL 
 #define munge_ddd  NULL 
@@ -106,14 +94,6 @@ void munge_wwwsw(const void *, void *);
 #define munge_dddddd  NULL 
 #define munge_ddddddd  NULL 
 #define munge_dddddddd  NULL 
-#define munge_wl  NULL 
-#define munge_wlw  NULL 
-#define munge_wwwl  NULL 
-#define munge_wwwwl  NULL 
-#define munge_wwwwwl  NULL 
-#define munge_wsw  NULL 
-#define munge_wws  NULL 
-#define munge_wwwsw  NULL 
 #endif // __ppc__
 #endif /* !__MUNGE_ONCE */
 
@@ -453,12 +433,6 @@ struct getpriority_args {
 	char which_l_[PADL_(int)]; int which; char which_r_[PADR_(int)];
 	char who_l_[PADL_(int)]; int who; char who_r_[PADR_(int)];
 };
-#ifdef __ppc__
-#else
-struct sigreturn_args {
-	char sigcntxp_l_[PADL_(struct sigcontext *)]; struct sigcontext * sigcntxp; char sigcntxp_r_[PADR_(struct sigcontext *)];
-};
-#endif
 struct bind_args {
 	char s_l_[PADL_(int)]; int s; char s_r_[PADR_(int)];
 	char name_l_[PADL_(user_addr_t)]; user_addr_t name; char name_r_[PADR_(user_addr_t)];
@@ -478,17 +452,10 @@ struct listen_args {
 struct sigsuspend_args {
 	char mask_l_[PADL_(sigset_t)]; sigset_t mask; char mask_r_[PADR_(sigset_t)];
 };
-#ifdef __ppc__
-struct ppc_gettimeofday_args {
-	char tp_l_[PADL_(user_addr_t)]; user_addr_t tp; char tp_r_[PADR_(user_addr_t)];
-	char tzp_l_[PADL_(user_addr_t)]; user_addr_t tzp; char tzp_r_[PADR_(user_addr_t)];
-};
-#else
 struct gettimeofday_args {
 	char tp_l_[PADL_(user_addr_t)]; user_addr_t tp; char tp_r_[PADR_(user_addr_t)];
 	char tzp_l_[PADL_(user_addr_t)]; user_addr_t tzp; char tzp_r_[PADR_(user_addr_t)];
 };
-#endif
 struct getrusage_args {
 	char who_l_[PADL_(int)]; int who; char who_r_[PADR_(int)];
 	char rusage_l_[PADL_(user_addr_t)]; user_addr_t rusage; char rusage_r_[PADR_(user_addr_t)];
@@ -660,13 +627,18 @@ struct setegid_args {
 struct seteuid_args {
 	char euid_l_[PADL_(uid_t)]; uid_t euid; char euid_r_[PADR_(uid_t)];
 };
-#ifdef __ppc__
 struct sigreturn_args {
 	char uctx_l_[PADL_(user_addr_t)]; user_addr_t uctx; char uctx_r_[PADR_(user_addr_t)];
 	char infostyle_l_[PADL_(int)]; int infostyle; char infostyle_r_[PADR_(int)];
 };
-#else
-#endif
+struct chud_args {
+	char code_l_[PADL_(int)]; int code; char code_r_[PADR_(int)];
+	char arg1_l_[PADL_(int)]; int arg1; char arg1_r_[PADR_(int)];
+	char arg2_l_[PADL_(int)]; int arg2; char arg2_r_[PADR_(int)];
+	char arg3_l_[PADL_(int)]; int arg3; char arg3_r_[PADR_(int)];
+	char arg4_l_[PADL_(int)]; int arg4; char arg4_r_[PADR_(int)];
+	char arg5_l_[PADL_(int)]; int arg5; char arg5_r_[PADR_(int)];
+};
 struct stat_args {
 	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
 	char ub_l_[PADL_(user_addr_t)]; user_addr_t ub; char ub_r_[PADR_(user_addr_t)];
@@ -749,7 +721,6 @@ struct munlock_args {
 struct undelete_args {
 	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
 };
-#ifdef __ppc__
 struct ATsocket_args {
 	char proto_l_[PADL_(int)]; int proto; char proto_r_[PADR_(int)];
 };
@@ -786,8 +757,6 @@ struct ATPgetrsp_args {
 	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
 	char bdsp_l_[PADL_(unsigned char *)]; unsigned char * bdsp; char bdsp_r_[PADR_(unsigned char *)];
 };
-#else
-#endif /* __ppc__ */
 struct kqueue_from_portset_np_args {
 	char portset_l_[PADL_(int)]; int portset; char portset_r_[PADR_(int)];
 };
@@ -1266,6 +1235,14 @@ struct utrace_args {
 	char addr_l_[PADL_(user_addr_t)]; user_addr_t addr; char addr_r_[PADR_(user_addr_t)];
 	char len_l_[PADL_(user_size_t)]; user_size_t len; char len_r_[PADR_(user_size_t)];
 };
+struct proc_info_args {
+	char callnum_l_[PADL_(int32_t)]; int32_t callnum; char callnum_r_[PADR_(int32_t)];
+	char pid_l_[PADL_(int32_t)]; int32_t pid; char pid_r_[PADR_(int32_t)];
+	char flavor_l_[PADL_(uint32_t)]; uint32_t flavor; char flavor_r_[PADR_(uint32_t)];
+	char arg_l_[PADL_(uint64_t)]; uint64_t arg; char arg_r_[PADR_(uint64_t)];
+	char buffer_l_[PADL_(user_addr_t)]; user_addr_t buffer; char buffer_r_[PADR_(user_addr_t)];
+	char buffersize_l_[PADL_(int32_t)]; int32_t buffersize; char buffersize_r_[PADR_(int32_t)];
+};
 struct audit_args {
 	char record_l_[PADL_(user_addr_t)]; user_addr_t record; char record_r_[PADR_(user_addr_t)];
 	char length_l_[PADL_(int)]; int length; char length_r_[PADR_(int)];
@@ -1314,6 +1291,12 @@ struct lchown_args {
 	char owner_l_[PADL_(uid_t)]; uid_t owner; char owner_r_[PADR_(uid_t)];
 	char group_l_[PADL_(gid_t)]; gid_t group; char group_r_[PADR_(gid_t)];
 };
+struct stack_snapshot_args {
+	char pid_l_[PADL_(pid_t)]; pid_t pid; char pid_r_[PADR_(pid_t)];
+	char tracebuf_l_[PADL_(user_addr_t)]; user_addr_t tracebuf; char tracebuf_r_[PADR_(user_addr_t)];
+	char tracebuf_size_l_[PADL_(uint32_t)]; uint32_t tracebuf_size; char tracebuf_size_r_[PADR_(uint32_t)];
+	char options_l_[PADL_(uint32_t)]; uint32_t options; char options_r_[PADR_(uint32_t)];
+};
 int nosys(struct proc *, struct nosys_args *, int *);
 void exit(struct proc *, struct exit_args *, int *);
 int fork(struct proc *, struct fork_args *, int *);
@@ -1398,19 +1381,11 @@ int setpriority(struct proc *, struct setpriority_args *, int *);
 int socket(struct proc *, struct socket_args *, int *);
 int connect(struct proc *, struct connect_args *, int *);
 int getpriority(struct proc *, struct getpriority_args *, int *);
-#ifdef __ppc__
-#else
-int sigreturn(struct proc *, struct sigreturn_args *, int *);
-#endif
 int bind(struct proc *, struct bind_args *, int *);
 int setsockopt(struct proc *, struct setsockopt_args *, int *);
 int listen(struct proc *, struct listen_args *, int *);
 int sigsuspend(struct proc *, struct sigsuspend_args *, int *);
-#ifdef __ppc__
-int ppc_gettimeofday(struct proc *, struct ppc_gettimeofday_args *, int *);
-#else
 int gettimeofday(struct proc *, struct gettimeofday_args *, int *);
-#endif
 int getrusage(struct proc *, struct getrusage_args *, int *);
 int getsockopt(struct proc *, struct getsockopt_args *, int *);
 int readv(struct proc *, struct readv_args *, user_ssize_t *);
@@ -1453,10 +1428,8 @@ int kdebug_trace(struct proc *, struct kdebug_trace_args *, int *);
 int setgid(struct proc *, struct setgid_args *, int *);
 int setegid(struct proc *, struct setegid_args *, int *);
 int seteuid(struct proc *, struct seteuid_args *, int *);
-#ifdef __ppc__
 int sigreturn(struct proc *, struct sigreturn_args *, int *);
-#else
-#endif
+int chud(struct proc *, struct chud_args *, int *);
 int stat(struct proc *, struct stat_args *, int *);
 int fstat(struct proc *, struct fstat_args *, int *);
 int lstat(struct proc *, struct lstat_args *, int *);
@@ -1477,7 +1450,6 @@ int __sysctl(struct proc *, struct __sysctl_args *, int *);
 int mlock(struct proc *, struct mlock_args *, int *);
 int munlock(struct proc *, struct munlock_args *, int *);
 int undelete(struct proc *, struct undelete_args *, int *);
-#ifdef __ppc__
 int ATsocket(struct proc *, struct ATsocket_args *, int *);
 int ATgetmsg(struct proc *, struct ATgetmsg_args *, int *);
 int ATputmsg(struct proc *, struct ATputmsg_args *, int *);
@@ -1485,8 +1457,6 @@ int ATPsndreq(struct proc *, struct ATPsndreq_args *, int *);
 int ATPsndrsp(struct proc *, struct ATPsndrsp_args *, int *);
 int ATPgetreq(struct proc *, struct ATPgetreq_args *, int *);
 int ATPgetrsp(struct proc *, struct ATPgetrsp_args *, int *);
-#else
-#endif /* __ppc__ */
 int kqueue_from_portset_np(struct proc *, struct kqueue_from_portset_np_args *, int *);
 int kqueue_portset_np(struct proc *, struct kqueue_portset_np_args *, int *);
 int getattrlist(struct proc *, struct getattrlist_args *, int *);
@@ -1588,6 +1558,7 @@ int __pthread_markcancel(struct proc *, struct __pthread_markcancel_args *, int
 int __pthread_canceled(struct proc *, struct __pthread_canceled_args *, int *);
 int __semwait_signal(struct proc *, struct __semwait_signal_args *, int *);
 int utrace(struct proc *, struct utrace_args *, int *);
+int proc_info(struct proc *, struct proc_info_args *, int *);
 int audit(struct proc *, struct audit_args *, int *);
 int auditon(struct proc *, struct auditon_args *, int *);
 int getauid(struct proc *, struct getauid_args *, int *);
@@ -1600,6 +1571,7 @@ int auditctl(struct proc *, struct auditctl_args *, int *);
 int kqueue(struct proc *, struct kqueue_args *, int *);
 int kevent(struct proc *, struct kevent_args *, int *);
 int lchown(struct proc *, struct lchown_args *, int *);
+int stack_snapshot(struct proc *, struct stack_snapshot_args *, int *);
 
 __END_DECLS
 #undef PAD_
diff --git a/bsd/sys/termios.h b/bsd/sys/termios.h
index 3945a91e9..74d338e50 100644
--- a/bsd/sys/termios.h
+++ b/bsd/sys/termios.h
@@ -280,24 +280,16 @@ typedef unsigned long long	user_speed_t;
  * WARNING - keep in sync with struct termios
  */
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_termios {
 	user_tcflag_t	c_iflag;	/* input flags */
 	user_tcflag_t	c_oflag;	/* output flags */
 	user_tcflag_t	c_cflag;	/* control flags */
 	user_tcflag_t	c_lflag;	/* local flags */
 	cc_t		c_cc[NCCS];	/* control chars */
-	user_speed_t	c_ispeed;	/* input speed */
+	user_speed_t	c_ispeed __attribute((aligned(8)));	/* input speed */
 	user_speed_t	c_ospeed;	/* output speed */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif	/* KERNEL */
 
 /*
diff --git a/bsd/sys/time.h b/bsd/sys/time.h
index 42539d33a..342daf1c6 100644
--- a/bsd/sys/time.h
+++ b/bsd/sys/time.h
@@ -171,17 +171,33 @@ struct timespec {
 // LP64todo - should this move?
 #include <machine/types.h>	/* user_time_t */
 
-/* LP64 version of struct timeval.  time_t is a long and must grow when 
+/* LP64 version of struct timespec.  time_t is a long and must grow when 
  * we're dealing with a 64-bit process.
- * WARNING - keep in sync with struct timeval
+ * WARNING - keep in sync with struct timespec
  */
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
+struct user_timespec {
+	user_time_t	tv_sec;		/* seconds */
+	int32_t	tv_nsec __attribute((aligned(8)));	/* and nanoseconds */
+};
+
+#endif
 #endif
 
+
+#ifdef KERNEL
+#ifndef _USERTIMEVAL
+#define _USERTIMEVAL
+
+#include <machine/types.h>	/* user_time_t */
+/*
+ * LP64 version of struct timeval.  time_t is a long and must grow when 
+ * we're dealing with a 64-bit process.
+ * WARNING - keep in sync with struct timeval
+ */
+
 struct user_timeval {
 	user_time_t	tv_sec;		/* seconds */
-	suseconds_t	tv_usec;	/* and microseconds */
+	suseconds_t	tv_usec __attribute((aligned(8)));	/* and microseconds */
 };	
 
 struct	user_itimerval {
@@ -189,21 +205,9 @@ struct	user_itimerval {
 	struct	user_timeval it_value;		/* current value */
 };
 
-/* LP64 version of struct timespec.  time_t is a long and must grow when 
- * we're dealing with a 64-bit process.
- * WARNING - keep in sync with struct timespec
- */
-struct user_timespec {
-	user_time_t	tv_sec;		/* seconds */
-	int32_t	tv_nsec;	/* and nanoseconds */
-};
-
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
 #endif
-
 #endif // KERNEL
-#endif
+
 
 #define	TIMEVAL_TO_TIMESPEC(tv, ts) {					\
 	(ts)->tv_sec = (tv)->tv_sec;					\
diff --git a/bsd/sys/ucontext.h b/bsd/sys/ucontext.h
index 18f0c4a02..7e151fe78 100644
--- a/bsd/sys/ucontext.h
+++ b/bsd/sys/ucontext.h
@@ -62,10 +62,6 @@ typedef struct ucontext64	ucontext64_t;
 #ifdef KERNEL
 #include <machine/types.h>	/* user_addr_t, user_size_t */
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 /* kernel representation of struct ucontext64 for 64 bit processes */
 struct user_ucontext64 {
 	int				uc_onstack;
@@ -76,10 +72,6 @@ struct user_ucontext64 {
 	user_addr_t			uc_mcontext64;	/* machine context */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 typedef struct user_ucontext64 user_ucontext64_t;
 #endif	/* KERNEL */
 
diff --git a/bsd/sys/ucred.h b/bsd/sys/ucred.h
index 0603ebab8..6860d9214 100644
--- a/bsd/sys/ucred.h
+++ b/bsd/sys/ucred.h
@@ -109,6 +109,8 @@ struct xucred {
 #define NOCRED ((kauth_cred_t )0)	/* no credential available */
 #define FSCRED ((kauth_cred_t )-1)	/* filesystem credential */
 
+#define IS_VALID_CRED(_cr)	((_cr) != NOCRED && (_cr) != FSCRED)
+
 #ifdef KERNEL
 #ifdef __APPLE_API_OBSOLETE
 __BEGIN_DECLS
diff --git a/bsd/sys/user.h b/bsd/sys/user.h
index 4a2278ce0..636dac0e5 100644
--- a/bsd/sys/user.h
+++ b/bsd/sys/user.h
@@ -88,15 +88,15 @@
  *	Per-thread U area.
  */
 struct uthread {
-	int	*uu_ar0;		/* address of users saved R0 */
-
 	/* syscall parameters, results and catches */
 	u_int64_t uu_arg[8]; /* arguments to current system call */
 	int	*uu_ap;			/* pointer to arglist */
-    int uu_rval[2];
+        int uu_rval[2];
 
 	/* thread exception handling */
+	int	uu_exception;
 	int	uu_code;			/* ``code'' to trap */
+	int	uu_subcode;
 	char uu_cursig;			/* p_cursig for exc. */
 	/* support for select - across system calls */
 	struct _select {
@@ -157,7 +157,7 @@ struct uthread {
 	struct ucred	*uu_ucred;		/* per thread credential */
         int		uu_defer_reclaims;
         vnode_t		uu_vreclaims;
-
+		struct user_sigaltstack uu_sigstk;
 #ifdef JOE_DEBUG
         int		uu_iocount;
         int		uu_vpindex;
@@ -174,6 +174,7 @@ typedef struct uthread * uthread_t;
 #define UT_CANCEL	0x00000008             /* thread marked for cancel */
 #define UT_CANCELED	0x00000010            /* thread cancelled */
 #define UT_CANCELDISABLE 0x00000020            /* thread cancel disabled */
+#define UT_ALTSTACK 0x00000040  /* this thread has alt stack for signals */
 
 #define	UT_VFORK	0x02000000	/* thread has vfork children */
 #define	UT_SETUID	0x04000000	/* thread is settugid() */
diff --git a/bsd/sys/vm.h b/bsd/sys/vm.h
index 04fe68e83..18e5c1779 100644
--- a/bsd/sys/vm.h
+++ b/bsd/sys/vm.h
@@ -107,27 +107,19 @@ struct vmspace {
  * WARNING - keep in sync with vmspace
  */
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_vmspace {
 	int     	vm_refcnt;      /* number of references */
-	user_addr_t	vm_shm;			/* SYS5 shared memory private data XXX */
+	user_addr_t	vm_shm __attribute((aligned(8)));			/* SYS5 shared memory private data XXX */
 	segsz_t 	vm_rssize;		/* current resident set size in pages */
 	segsz_t 	vm_swrss;		/* resident set size before last swap */
 	segsz_t 	vm_tsize;		/* text size (pages) XXX */
 	segsz_t		vm_dsize;		/* data size (pages) XXX */
 	segsz_t 	vm_ssize;		/* stack size (pages) */
-	user_addr_t	vm_taddr;       /* user virtual address of text XXX */
+	user_addr_t	vm_taddr __attribute((aligned(8)));       /* user virtual address of text XXX */
 	user_addr_t	vm_daddr;       /* user virtual address of data XXX */
 	user_addr_t vm_maxsaddr; 	/* user VA at max stack growth */
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif /* KERNEL */
 
 #include <kern/thread.h>
diff --git a/bsd/sys/vnioctl.h b/bsd/sys/vnioctl.h
index 43f0c86f9..36f8d28c3 100644
--- a/bsd/sys/vnioctl.h
+++ b/bsd/sys/vnioctl.h
@@ -94,20 +94,12 @@ struct vn_ioctl {
 
 #ifdef KERNEL_PRIVATE
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=natural
-#endif
-
 struct user_vn_ioctl {
 	u_int64_t	vn_file;	/* pathname of file to mount */
 	int			vn_size;	/* (returned) size of disk */
 	vncontrol_t	vn_control;
 };
 
-#if __DARWIN_ALIGN_NATURAL
-#pragma options align=reset
-#endif
-
 #endif /* KERNEL_PRIVATE */
 
 /*
diff --git a/bsd/sys/vnode_if.h b/bsd/sys/vnode_if.h
index 4112e27d2..3061e27ae 100644
--- a/bsd/sys/vnode_if.h
+++ b/bsd/sys/vnode_if.h
@@ -124,7 +124,6 @@ extern struct vnodeop_desc vnop_truncate_desc;
 extern struct vnodeop_desc vnop_allocate_desc;
 extern struct vnodeop_desc vnop_pagein_desc;
 extern struct vnodeop_desc vnop_pageout_desc;
-extern struct vnodeop_desc vnop_devblocksize_desc;
 extern struct vnodeop_desc vnop_searchfs_desc;
 extern struct vnodeop_desc vnop_copyfile_desc;
 extern struct vnodeop_desc vnop_blktooff_desc;
@@ -710,18 +709,6 @@ struct vnop_pageout_args {
 extern errno_t VNOP_PAGEOUT(vnode_t, upl_t, vm_offset_t, off_t, size_t, int, vfs_context_t);
 
 
-#ifdef BSD_KERNEL_PRIVATE
-/*
- *#% devblocksize vp      = = =
- *#
- */
-struct vnop_devblocksize_args {
-	struct vnodeop_desc *a_desc;
-	vnode_t a_vp;
-	register_t *a_retval;
-};
-#endif /* BSD_KERNEL_PRIVATE */
-
 /*
  *#
  *#% searchfs     vp      L L L
diff --git a/bsd/sys/vnode_internal.h b/bsd/sys/vnode_internal.h
index f2dd79ba2..fb561c741 100644
--- a/bsd/sys/vnode_internal.h
+++ b/bsd/sys/vnode_internal.h
@@ -303,7 +303,7 @@ extern void * mntvnode_slock;
 #define VDESC(OP) (& __CONCAT(OP,_desc))
 #define VOFFSET(OP) (VDESC(OP)->vdesc_offset)
 
-
+struct ostat;
 
 int	build_path(vnode_t first_vp, char *buff, int buflen, int *outlen);
 int 	bdevvp(dev_t dev, struct vnode **vpp);
@@ -337,6 +337,7 @@ int	vn_setxattr(vnode_t, const char *, uio_t, int, vfs_context_t);
 int	vn_removexattr(vnode_t, const char *, int, vfs_context_t);
 int	vn_listxattr(vnode_t, uio_t, size_t *, int, vfs_context_t);
 
+void	name_cache_lock_shared(void);
 void	name_cache_lock(void);
 void	name_cache_unlock(void);
 
diff --git a/bsd/ufs/ffs/ffs_alloc.c b/bsd/ufs/ffs/ffs_alloc.c
index 815d53b3f..0c127dcfd 100644
--- a/bsd/ufs/ffs/ffs_alloc.c
+++ b/bsd/ufs/ffs/ffs_alloc.c
@@ -84,7 +84,7 @@
 
 #if REV_ENDIAN_FS
 #include <ufs/ufs/ufs_byte_order.h>
-#include <architecture/byte_order.h>
+#include <libkern/OSByteOrder.h>
 #endif /* REV_ENDIAN_FS */
 
 extern u_long nextgennumber;
@@ -140,7 +140,7 @@ ffs_alloc(ip, lbn, bpref, size, cred, bnp)
 		    ip->i_dev, fs->fs_bsize, size, fs->fs_fsmnt);
 		panic("ffs_alloc: bad size");
 	}
-	if (cred == NOCRED)
+	if (!IS_VALID_CRED(cred))
 		panic("ffs_alloc: missing credential\n");
 #endif /* DIAGNOSTIC */
 	if (size == fs->fs_bsize && fs->fs_cstotal.cs_nbfree == 0)
@@ -210,7 +210,7 @@ ffs_realloccg(ip, lbprev, bpref, osize, nsize, cred, bpp)
 		    ip->i_dev, fs->fs_bsize, osize, nsize, fs->fs_fsmnt);
 		panic("ffs_realloccg: bad size");
 	}
-	if (cred == NOCRED)
+	if (!IS_VALID_CRED(cred))
 		panic("ffs_realloccg: missing credential\n");
 #endif /* DIAGNOSTIC */
 	if (suser(cred, NULL) != 0 && freespace(fs, fs->fs_minfree) <= 0)
@@ -599,7 +599,7 @@ ffs_blkpref(ip, lbn, indx, bap)
 	if (indx && bap) {
 	if (rev_endian) {
 		if (bap != &ip->i_db[0])
-			prev = NXSwapLong(bap[indx - 1]);
+			prev = OSSwapInt32(bap[indx - 1]);
 		else
 			prev = bap[indx - 1];
 	} else prev = bap[indx - 1];
@@ -657,7 +657,7 @@ ffs_blkpref(ip, lbn, indx, bap)
 			return (nextblk);
 		}
 		if (bap != &ip->i_db[0])
-			prev = NXSwapLong(bap[indx - fs->fs_maxcontig]);
+			prev = OSSwapInt32(bap[indx - fs->fs_maxcontig]);
 		else
 			prev = bap[indx - fs->fs_maxcontig];
 		if (prev + blkstofrags(fs, fs->fs_maxcontig) != nextblk)
diff --git a/bsd/ufs/ffs/ffs_balloc.c b/bsd/ufs/ffs/ffs_balloc.c
index 961f8f633..cf998b4e5 100644
--- a/bsd/ufs/ffs/ffs_balloc.c
+++ b/bsd/ufs/ffs/ffs_balloc.c
@@ -87,7 +87,7 @@
 
 #if REV_ENDIAN_FS
 #include <ufs/ufs/ufs_byte_order.h>
-#include <architecture/byte_order.h>
+#include <libkern/OSByteOrder.h>
 #endif /* REV_ENDIAN_FS */
 
 /*
@@ -312,7 +312,7 @@ ffs_balloc(
 		bap = (ufs_daddr_t *)buf_dataptr(bp);
 #if	REV_ENDIAN_FS
 	if (rev_endian)
-		nb = NXSwapLong(bap[indirs[i].in_off]);
+		nb = OSSwapInt32(bap[indirs[i].in_off]);
 	else {
 #endif	/* REV_ENDIAN_FS */
 		nb = bap[indirs[i].in_off];
@@ -350,7 +350,7 @@ ffs_balloc(
 		}
 #if	REV_ENDIAN_FS
 	if (rev_endian)
-		bap[indirs[i - 1].in_off] = NXSwapLong(nb);
+		bap[indirs[i - 1].in_off] = OSSwapInt32(nb);
 	else {
 #endif	/* REV_ENDIAN_FS */
 		bap[indirs[i - 1].in_off] = nb;
@@ -381,7 +381,7 @@ ffs_balloc(
 		*allocblk++ = nb;
 #if	REV_ENDIAN_FS
 	if (rev_endian)
-		bap[indirs[i].in_off] = NXSwapLong(nb);
+		bap[indirs[i].in_off] = OSSwapInt32(nb);
 	else {
 #endif	/* REV_ENDIAN_FS */
 		bap[indirs[i].in_off] = nb;
@@ -585,7 +585,7 @@ ffs_blkalloc(
 		bap = (ufs_daddr_t *)buf_dataptr(bp);
 #if	REV_ENDIAN_FS
 	if (rev_endian)
-		nb = NXSwapLong(bap[indirs[i].in_off]);
+		nb = OSSwapInt32(bap[indirs[i].in_off]);
 	else {
 #endif	/* REV_ENDIAN_FS */
 		nb = bap[indirs[i].in_off];
@@ -623,7 +623,7 @@ ffs_blkalloc(
 		}
 #if	REV_ENDIAN_FS
 	if (rev_endian)
-		bap[indirs[i - 1].in_off] = NXSwapLong(nb);
+		bap[indirs[i - 1].in_off] = OSSwapInt32(nb);
 	else {
 #endif	/* REV_ENDIAN_FS */
 		bap[indirs[i - 1].in_off] = nb;
@@ -654,7 +654,7 @@ ffs_blkalloc(
 		*allocblk++ = nb;
 #if	REV_ENDIAN_FS
 	if (rev_endian)
-		bap[indirs[i].in_off] = NXSwapLong(nb);
+		bap[indirs[i].in_off] = OSSwapInt32(nb);
 	else {
 #endif	/* REV_ENDIAN_FS */
 		bap[indirs[i].in_off] = nb;
diff --git a/bsd/ufs/ffs/ffs_inode.c b/bsd/ufs/ffs/ffs_inode.c
index bdedb24f0..b04128754 100644
--- a/bsd/ufs/ffs/ffs_inode.c
+++ b/bsd/ufs/ffs/ffs_inode.c
@@ -90,7 +90,7 @@
 
 #if REV_ENDIAN_FS
 #include <ufs/ufs/ufs_byte_order.h>
-#include <architecture/byte_order.h>
+#include <libkern/OSByteOrder.h>
 #endif /* REV_ENDIAN_FS */
 
 static int ffs_indirtrunc(struct inode *, ufs_daddr_t, ufs_daddr_t,
@@ -549,7 +549,7 @@ ffs_indirtrunc(ip, lbn, dbn, lastbn, level, countp)
 	    i--, nlbn += factor) {
 #if	REV_ENDIAN_FS
 		if (rev_endian)
-			nb = NXSwapLong(bap[i]);
+			nb = OSSwapInt32(bap[i]);
 		else {
 #endif	/* REV_ENDIAN_FS */
 			nb = bap[i];
@@ -575,7 +575,7 @@ ffs_indirtrunc(ip, lbn, dbn, lastbn, level, countp)
 		last = lastbn % factor;
 #if	REV_ENDIAN_FS
 		if (rev_endian)
-			nb = NXSwapLong(bap[i]);
+			nb = OSSwapInt32(bap[i]);
 		else {
 #endif	/* REV_ENDIAN_FS */
 			nb = bap[i];
diff --git a/bsd/ufs/ffs/ffs_subr.c b/bsd/ufs/ffs/ffs_subr.c
index 3f93eef05..d226a5680 100644
--- a/bsd/ufs/ffs/ffs_subr.c
+++ b/bsd/ufs/ffs/ffs_subr.c
@@ -82,7 +82,6 @@
 #include <ufs/ffs/ffs_extern.h>
 #if REV_ENDIAN_FS
 #include <ufs/ufs/ufs_byte_order.h>
-#include <architecture/byte_order.h>
 #endif /* REV_ENDIAN_FS */
 
 /*
diff --git a/bsd/ufs/ffs/ffs_vfsops.c b/bsd/ufs/ffs/ffs_vfsops.c
index a9e787a49..18b090997 100644
--- a/bsd/ufs/ffs/ffs_vfsops.c
+++ b/bsd/ufs/ffs/ffs_vfsops.c
@@ -93,7 +93,7 @@
 #include <ufs/ffs/ffs_extern.h>
 #if REV_ENDIAN_FS
 #include <ufs/ufs/ufs_byte_order.h>
-#include <architecture/byte_order.h>
+#include <libkern/OSByteOrder.h>
 #endif /* REV_ENDIAN_FS */
 
 int ffs_sbupdate(struct ufsmount *, int);
@@ -542,7 +542,6 @@ ffs_mountfs(devvp, mp, context)
         /*
          * Buffer cache does not handle multiple pages in a buf when
 
-
 	/*
 	 * Buffer cache does not handle multiple pages in a buf when
 	 * invalidating incore buffer in pageout. There are no locks 
@@ -984,7 +983,7 @@ ffs_vfs_getattr(mp, fsap, context)
 			length = ulp->ul_namelen;
 #if REV_ENDIAN_FS
 			if (mp->mnt_flag & MNT_REVEND)
-				length = NXSwapShort(length);
+				length = OSSwapInt16(length);
 #endif
 			if (length > 0 && length <= UFS_MAX_LABEL_NAME) {
 				bcopy(ulp->ul_name, fsap->f_vol_name, length);
@@ -1112,14 +1111,15 @@ ffs_vfs_setattr(mp, fsap, context)
 
 		/* Copy new name over existing name */
 		ulp->ul_namelen = strlen(fsap->f_vol_name);
-#if REV_ENDIAN_FS
-		if (mp->mnt_flag & MNT_REVEND)
-			ulp->ul_namelen = NXSwapShort(ulp->ul_namelen);
-#endif
 		bcopy(fsap->f_vol_name, ulp->ul_name, ulp->ul_namelen);
 		ulp->ul_name[UFS_MAX_LABEL_NAME - 1] = '\0';
 		ulp->ul_name[ulp->ul_namelen] = '\0';
 
+#if REV_ENDIAN_FS
+		if (mp->mnt_flag & MNT_REVEND)
+			ulp->ul_namelen = OSSwapInt16(ulp->ul_namelen);
+#endif
+
 		/* Update the checksum */
 		ulp->ul_checksum = 0;
 		ulp->ul_checksum = ul_cksum(ulp, sizeof(*ulp));
@@ -1461,21 +1461,22 @@ ffs_fhtovp(mp, fhlen, fhp, vpp, context)
 	struct vnode *nvp;
 	struct fs *fs;
 	int error;
+	ino_t	  ino;
 
 	if (fhlen < (int)sizeof(struct ufid))
 		return (EINVAL);
 	ufhp = (struct ufid *)fhp;
 	fs = VFSTOUFS(mp)->um_fs;
-	if (ufhp->ufid_ino < ROOTINO ||
-	    ufhp->ufid_ino >= fs->fs_ncg * fs->fs_ipg)
+	ino = ntohl(ufhp->ufid_ino);
+	if (ino < ROOTINO || ino >= fs->fs_ncg * fs->fs_ipg)
 		return (ESTALE);
-	error = ffs_vget_internal(mp, ufhp->ufid_ino, &nvp, NULL, NULL, 0, 1);
+	error = ffs_vget_internal(mp, ino, &nvp, NULL, NULL, 0, 1);
 	if (error) {
 		*vpp = NULLVP;
 		return (error);
 	}
 	ip = VTOI(nvp);
-	if (ip->i_mode == 0 || ip->i_gen != ufhp->ufid_gen) {
+	if (ip->i_mode == 0 || ip->i_gen != ntohl(ufhp->ufid_gen)) {
 		vnode_put(nvp);
 		*vpp = NULLVP;
 		return (ESTALE);
@@ -1502,8 +1503,8 @@ ffs_vptofh(vp, fhlenp, fhp, context)
 		return (EOVERFLOW);
 	ip = VTOI(vp);
 	ufhp = (struct ufid *)fhp;
-	ufhp->ufid_ino = ip->i_number;
-	ufhp->ufid_gen = ip->i_gen;
+	ufhp->ufid_ino = htonl(ip->i_number);
+	ufhp->ufid_gen = htonl(ip->i_gen);
 	*fhlenp = sizeof(struct ufid);
 	return (0);
 }
@@ -1629,7 +1630,7 @@ ffs_sbupdate(mp, waitfor)
 	*  before writing
 	*/
 	if (rev_endian) {
-		dfs->fs_maxfilesize = NXSwapLongLong(mp->um_savedmaxfilesize);		/* XXX */
+		dfs->fs_maxfilesize = OSSwapInt64(mp->um_savedmaxfilesize);		/* XXX */
 	} else {
 #endif /* REV_ENDIAN_FS */
 		dfs->fs_maxfilesize = mp->um_savedmaxfilesize;	/* XXX */
diff --git a/bsd/ufs/ffs/ffs_vnops.c b/bsd/ufs/ffs/ffs_vnops.c
index bca1f6e38..718d1feb0 100644
--- a/bsd/ufs/ffs/ffs_vnops.c
+++ b/bsd/ufs/ffs/ffs_vnops.c
@@ -92,7 +92,6 @@
 #include <ufs/ffs/ffs_extern.h>
 #if REV_ENDIAN_FS
 #include <ufs/ufs/ufs_byte_order.h>
-#include <architecture/byte_order.h>
 #endif /* REV_ENDIAN_FS */
 
 #define VOPFUNC int (*)(void *)
@@ -173,7 +172,6 @@ struct vnodeopv_entry_desc ffs_specop_entries[] = {
 	{ &vnop_pathconf_desc, (VOPFUNC)spec_pathconf },		/* pathconf */
 	{ &vnop_advlock_desc, (VOPFUNC)err_advlock },		/* advlock */
 	{ &vnop_bwrite_desc, (VOPFUNC)vn_bwrite },
-	{ &vnop_devblocksize_desc, (VOPFUNC)spec_devblocksize },	/* devblocksize */
 	{ &vnop_pagein_desc, (VOPFUNC)ffs_pagein },		/* Pagein */
 	{ &vnop_pageout_desc, (VOPFUNC)ffs_pageout },		/* Pageout */
 	{ &vnop_copyfile_desc, (VOPFUNC)err_copyfile },		/* Copy File */
diff --git a/bsd/ufs/ufs/ufs_attrlist.c b/bsd/ufs/ufs/ufs_attrlist.c
index 9f3ac59ab..99a0940ee 100644
--- a/bsd/ufs/ufs/ufs_attrlist.c
+++ b/bsd/ufs/ufs/ufs_attrlist.c
@@ -40,7 +40,6 @@
 #include <sys/kernel.h>
 #include <sys/kauth.h>
 
-#include <architecture/byte_order.h>
 #include <ufs/ufs/dinode.h>
 #include <ufs/ffs/fs.h>
 #include <sys/mount_internal.h>
diff --git a/bsd/ufs/ufs/ufs_bmap.c b/bsd/ufs/ufs/ufs_bmap.c
index 269866e44..22e8a7a62 100644
--- a/bsd/ufs/ufs/ufs_bmap.c
+++ b/bsd/ufs/ufs/ufs_bmap.c
@@ -88,7 +88,7 @@
 #include <ufs/ufs/ufs_extern.h>
 #if REV_ENDIAN_FS
 #include <ufs/ufs/ufs_byte_order.h>
-#include <architecture/byte_order.h>
+#include <libkern/OSByteOrder.h>
 #endif /* REV_ENDIAN_FS */
 
 
@@ -226,7 +226,7 @@ ufs_bmaparray(vp, bn, bnp, ap, nump, runp)
 		daddr = dataptr[xap->in_off];
 #if REV_ENDIAN_FS
 		if (rev_endian)
-			daddr = NXSwapLong(daddr);
+			daddr = OSSwapInt32(daddr);
 #endif /* REV_ENDIAN_FS */
 		if (num == 1 && daddr && runp) {
 #if REV_ENDIAN_FS
@@ -234,8 +234,8 @@ ufs_bmaparray(vp, bn, bnp, ap, nump, runp)
 			for (bn = xap->in_off + 1;
 			    bn < MNINDIR(ump) && *runp < maxrun &&
 			    is_sequential(ump,
-			    NXSwapLong(dataptr[bn - 1]),
-			    NXSwapLong(dataptr[bn]));
+			    OSSwapInt32(dataptr[bn - 1]),
+			    OSSwapInt32(dataptr[bn]));
 			    ++bn, ++*runp);
 		 } else {
 #endif /* REV_ENDIAN_FS */
diff --git a/bsd/ufs/ufs/ufs_byte_order.c b/bsd/ufs/ufs/ufs_byte_order.c
index bc7a05788..6562d1376 100644
--- a/bsd/ufs/ufs/ufs_byte_order.c
+++ b/bsd/ufs/ufs/ufs_byte_order.c
@@ -40,11 +40,11 @@
 #include <sys/param.h>
 #include <sys/quota.h>
 #include <ufs/ufs/ufs_byte_order.h>
-#include <architecture/byte_order.h>
+#include <libkern/OSByteOrder.h>
 
-#define	byte_swap_longlong(thing) ((thing) = NXSwapLongLong(thing))
-#define	byte_swap_int(thing) ((thing) = NXSwapLong(thing))
-#define	byte_swap_short(thing) ((thing) = NXSwapShort(thing))
+#define	byte_swap_longlong(thing) ((thing) = OSSwapInt64(thing))
+#define	byte_swap_int(thing) ((thing) = OSSwapInt32(thing))
+#define	byte_swap_short(thing) ((thing) = OSSwapInt16(thing))
 
 void
 byte_swap_longlongs(unsigned long long *array, int count)
@@ -252,32 +252,32 @@ byte_swap_inode_in(struct dinode *di, struct inode *ip)
 {
 	int		i;
 
-	ip->i_mode = NXSwapShort(di->di_mode);
-	ip->i_nlink = NXSwapShort(di->di_nlink);
-	ip->i_oldids[0] = NXSwapShort(di->di_u.oldids[0]);
-	ip->i_oldids[1] = NXSwapShort(di->di_u.oldids[1]);
-	ip->i_size = NXSwapLongLong(di->di_size);
-	ip->i_atime = NXSwapLong(di->di_atime);
-	ip->i_atimensec = NXSwapLong(di->di_atimensec);
-	ip->i_mtime = NXSwapLong(di->di_mtime);
-	ip->i_mtimensec = NXSwapLong(di->di_mtimensec);
-	ip->i_ctime = NXSwapLong(di->di_ctime);
-	ip->i_ctimensec = NXSwapLong(di->di_ctimensec);
+	ip->i_mode = OSSwapInt16(di->di_mode);
+	ip->i_nlink = OSSwapInt16(di->di_nlink);
+	ip->i_oldids[0] = OSSwapInt16(di->di_u.oldids[0]);
+	ip->i_oldids[1] = OSSwapInt16(di->di_u.oldids[1]);
+	ip->i_size = OSSwapInt64(di->di_size);
+	ip->i_atime = OSSwapInt32(di->di_atime);
+	ip->i_atimensec = OSSwapInt32(di->di_atimensec);
+	ip->i_mtime = OSSwapInt32(di->di_mtime);
+	ip->i_mtimensec = OSSwapInt32(di->di_mtimensec);
+	ip->i_ctime = OSSwapInt32(di->di_ctime);
+	ip->i_ctimensec = OSSwapInt32(di->di_ctimensec);
 	if (((ip->i_mode & IFMT) == IFLNK ) && (ip->i_size <= RESYMLNKLEN)) {
 		bcopy(&di->di_shortlink,  &ip->i_shortlink, RESYMLNKLEN);
 	} else {
 		for (i=0; i < NDADDR; i++)	/* direct blocks */
-			ip->i_db[i] = NXSwapLong(di->di_db[i]);
+			ip->i_db[i] = OSSwapInt32(di->di_db[i]);
 		for (i=0; i < NIADDR; i++)	/* indirect blocks */
-			ip->i_ib[i] = NXSwapLong(di->di_ib[i]);
+			ip->i_ib[i] = OSSwapInt32(di->di_ib[i]);
 	} 
-	ip->i_flags = NXSwapLong(di->di_flags);
-	ip->i_blocks = NXSwapLong(di->di_blocks);
-	ip->i_gen = NXSwapLong(di->di_gen);
-	ip->i_uid = NXSwapLong(di->di_uid);
-	ip->i_gid = NXSwapLong(di->di_gid);
-	ip->i_spare[0] = NXSwapLong(di->di_spare[0]);
-	ip->i_spare[1] = NXSwapLong(di->di_spare[1]);
+	ip->i_flags = OSSwapInt32(di->di_flags);
+	ip->i_blocks = OSSwapInt32(di->di_blocks);
+	ip->i_gen = OSSwapInt32(di->di_gen);
+	ip->i_uid = OSSwapInt32(di->di_uid);
+	ip->i_gid = OSSwapInt32(di->di_gid);
+	ip->i_spare[0] = OSSwapInt32(di->di_spare[0]);
+	ip->i_spare[1] = OSSwapInt32(di->di_spare[1]);
 }
 
 void
@@ -289,32 +289,32 @@ byte_swap_inode_out(struct inode *ip, struct dinode *di)
 	mode = (ip->i_mode & IFMT);
 	inosize = ip->i_size;
  
-	di->di_mode = NXSwapShort(ip->i_mode);
-	di->di_nlink = NXSwapShort(ip->i_nlink);
-	di->di_u.oldids[0] = NXSwapShort(ip->i_oldids[0]);
-	di->di_u.oldids[1] = NXSwapShort(ip->i_oldids[1]);
-	di->di_size = NXSwapLongLong(ip->i_size);
-	di->di_atime = NXSwapLong(ip->i_atime);
-	di->di_atimensec = NXSwapLong(ip->i_atimensec);
-	di->di_mtime = NXSwapLong(ip->i_mtime);
-	di->di_mtimensec = NXSwapLong(ip->i_mtimensec);
-	di->di_ctime = NXSwapLong(ip->i_ctime);
-	di->di_ctimensec = NXSwapLong(ip->i_ctimensec);
+	di->di_mode = OSSwapInt16(ip->i_mode);
+	di->di_nlink = OSSwapInt16(ip->i_nlink);
+	di->di_u.oldids[0] = OSSwapInt16(ip->i_oldids[0]);
+	di->di_u.oldids[1] = OSSwapInt16(ip->i_oldids[1]);
+	di->di_size = OSSwapInt64(ip->i_size);
+	di->di_atime = OSSwapInt32(ip->i_atime);
+	di->di_atimensec = OSSwapInt32(ip->i_atimensec);
+	di->di_mtime = OSSwapInt32(ip->i_mtime);
+	di->di_mtimensec = OSSwapInt32(ip->i_mtimensec);
+	di->di_ctime = OSSwapInt32(ip->i_ctime);
+	di->di_ctimensec = OSSwapInt32(ip->i_ctimensec);
 	if ((mode == IFLNK) && (inosize <= RESYMLNKLEN)) {
 		bcopy( &ip->i_shortlink, &di->di_shortlink, RESYMLNKLEN);
 	} else {
 		for (i=0; i < NDADDR; i++)	/* direct blocks */
-			di->di_db[i] = NXSwapLong(ip->i_db[i]);
+			di->di_db[i] = OSSwapInt32(ip->i_db[i]);
 		for (i=0; i < NIADDR; i++)	/* indirect blocks */
-			di->di_ib[i] = NXSwapLong(ip->i_ib[i]);
+			di->di_ib[i] = OSSwapInt32(ip->i_ib[i]);
 	}
-	di->di_flags = NXSwapLong(ip->i_flags);
-	di->di_blocks = NXSwapLong(ip->i_blocks);
-	di->di_gen = NXSwapLong(ip->i_gen);
-	di->di_uid = NXSwapLong(ip->i_uid);
-	di->di_gid = NXSwapLong(ip->i_gid);
-	di->di_spare[0] = NXSwapLong(ip->i_spare[0]);
-	di->di_spare[1] = NXSwapLong(ip->i_spare[1]);
+	di->di_flags = OSSwapInt32(ip->i_flags);
+	di->di_blocks = OSSwapInt32(ip->i_blocks);
+	di->di_gen = OSSwapInt32(ip->i_gen);
+	di->di_uid = OSSwapInt32(ip->i_uid);
+	di->di_gid = OSSwapInt32(ip->i_gid);
+	di->di_spare[0] = OSSwapInt32(ip->i_spare[0]);
+	di->di_spare[1] = OSSwapInt32(ip->i_spare[1]);
 }
 
 void
diff --git a/bsd/ufs/ufs/ufs_lookup.c b/bsd/ufs/ufs/ufs_lookup.c
index 6d48ca5af..52eb936ff 100644
--- a/bsd/ufs/ufs/ufs_lookup.c
+++ b/bsd/ufs/ufs/ufs_lookup.c
@@ -84,10 +84,9 @@
 #include <ufs/ffs/ffs_extern.h>
 #if REV_ENDIAN_FS
 #include <ufs/ufs/ufs_byte_order.h>
-#include <architecture/byte_order.h>
 #endif /* REV_ENDIAN_FS */
 
-extern struct	nchstats nchstats;
+struct	nchstats ufs_nchstats;
 #if DIAGNOSTIC
 int	dirchk = 1;
 #else
@@ -238,7 +237,7 @@ ufs_lookup(ap)
 		    (error = ffs_blkatoff(vdp, (off_t)dp->i_offset, NULL, &bp)))
 		    	goto out;
 		numdirpasses = 2;
-		nchstats.ncs_2passes++;
+		ufs_nchstats.ncs_2passes++;
 	}
 	prevoff = dp->i_offset;
 	endsearch = roundup(dp->i_size, DIRBLKSIZ);
@@ -441,7 +440,7 @@ notfound:
 
 found:
 	if (numdirpasses == 2)
-		nchstats.ncs_pass2++;
+		ufs_nchstats.ncs_pass2++;
 	/*
 	 * Check that directory length properly reflects presence
 	 * of this entry.
diff --git a/bsd/ufs/ufs/ufs_quota.c b/bsd/ufs/ufs/ufs_quota.c
index 69b93ee6f..5708eb085 100644
--- a/bsd/ufs/ufs/ufs_quota.c
+++ b/bsd/ufs/ufs/ufs_quota.c
@@ -160,8 +160,12 @@ chkdq(struct inode *ip, int64_t change, kauth_cred_t cred, int flags)
 		return (0);
 	}
 #warning "hack for no cred passed to chkdq()"
+	/*
+	 * This use of proc_ucred() is safe because kernproc credential never
+	 * changes.
+	 */
 	p = current_proc();
-	if (cred == NOCRED)
+	if (!IS_VALID_CRED(cred))
 		cred = proc_ucred(kernproc);
 	if ((flags & FORCE) == 0 && (suser(cred, NULL) || (proc_forcequota(p)))) {
 		for (i = 0; i < MAXQUOTAS; i++) {
@@ -292,8 +296,12 @@ chkiq(struct inode *ip, long change, kauth_cred_t cred, int flags)
 		return (0);
 	}
 #warning "hack for no cred passed to chkiq()"
+	/*
+	 * This use of proc_ucred() is safe because kernproc credential never
+	 * changes.
+	 */
 	p = current_proc();
-	if (cred == NOCRED)
+	if (!IS_VALID_CRED(cred))
 		cred = proc_ucred(kernproc);
 	if ((flags & FORCE) == 0 && (suser(cred, NULL) || (proc_forcequota(p)))) {
 		for (i = 0; i < MAXQUOTAS; i++) {
@@ -480,8 +488,7 @@ quotaon(context, mp, type, fnamep)
 	if ( (error = dqfileopen(&ump->um_qfiles[type], type)) ) {
                 (void) vnode_close(vp, FREAD|FWRITE, NULL);
 
-		kauth_cred_rele(qfp->qf_cred);
-		qfp->qf_cred = NOCRED;
+		kauth_cred_unref(&qfp->qf_cred);
                 qfp->qf_vp = NULLVP;
                 goto out;
         }
@@ -547,7 +554,6 @@ quotaoff(struct mount *mp, register int type)
 	struct ufsmount *ump = VFSTOUFS(mp);
 	struct quotafile *qfp;
 	int error = 0;
-	kauth_cred_t cred;
 	struct ufs_quotaoff_cargs args;
 	
 	qfp = &ump->um_qfiles[type];
@@ -582,10 +588,8 @@ quotaoff(struct mount *mp, register int type)
 	error = vnode_close(qvp, FREAD|FWRITE, NULL);
 
 	qfp->qf_vp = NULLVP;
-	cred = qfp->qf_cred;
-	if (cred != NOCRED) {
-		qfp->qf_cred = NOCRED;
-		kauth_cred_rele(cred);
+	if (IS_VALID_CRED(qfp->qf_cred)) {
+		kauth_cred_unref(&qfp->qf_cred);
 	}
 	for (type = 0; type < MAXQUOTAS; type++)
 		if (ump->um_qfiles[type].qf_vp != NULLVP)
diff --git a/bsd/ufs/ufs/ufs_vnops.c b/bsd/ufs/ufs/ufs_vnops.c
index e7725a69a..1584d60fe 100644
--- a/bsd/ufs/ufs/ufs_vnops.c
+++ b/bsd/ufs/ufs/ufs_vnops.c
@@ -100,7 +100,6 @@
 
 #if REV_ENDIAN_FS
 #include <ufs/ufs/ufs_byte_order.h>
-#include <architecture/byte_order.h>
 #endif /* REV_ENDIAN_FS */
 
 
diff --git a/bsd/uxkern/ux_exception.c b/bsd/uxkern/ux_exception.c
index c0d094c2f..7764e78c9 100644
--- a/bsd/uxkern/ux_exception.c
+++ b/bsd/uxkern/ux_exception.c
@@ -240,8 +240,12 @@ catch_exception_raise(
 	    /*
 	     *	Send signal.
 	     */
-	    if (ux_signal != 0)
-		threadsignal(th_act, ux_signal, ucode);
+	    if (ux_signal != 0) {
+			ut->uu_exception = exception;
+			//ut->uu_code = code[0]; // filled in by threadsignal
+			ut->uu_subcode = code[1];			
+			threadsignal(th_act, ux_signal, code[0]);
+		}
 
 	    thread_deallocate(th_act);
 	}
diff --git a/bsd/vfs/kpi_vfs.c b/bsd/vfs/kpi_vfs.c
index 313b2a324..ba4b4c877 100644
--- a/bsd/vfs/kpi_vfs.c
+++ b/bsd/vfs/kpi_vfs.c
@@ -102,6 +102,8 @@
 #include <kern/assert.h>
 #include <kern/kalloc.h>
 
+#include <libkern/OSByteOrder.h>
+
 #include <miscfs/specfs/specdev.h>
 
 #include <mach/mach_types.h>
@@ -1051,14 +1053,18 @@ vfs_context_create(vfs_context_t context)
 	newcontext = (struct vfs_context *)kalloc(sizeof(struct vfs_context));
 
 	if (newcontext) {
+		kauth_cred_t safecred;
 		if (context) {
 			newcontext->vc_proc = context->vc_proc;
-			newcontext->vc_ucred = context->vc_ucred;
+			safecred = context->vc_ucred;
 		} else {
 			newcontext->vc_proc = proc_self();
-			newcontext->vc_ucred = kauth_cred_get();
+			safecred = kauth_cred_get();
 		}
-	   return(newcontext);
+		if (IS_VALID_CRED(safecred))
+			kauth_cred_ref(safecred);
+		newcontext->vc_ucred = safecred;
+		return(newcontext);
 	}
 	return((vfs_context_t)0);	
 }
@@ -1066,8 +1072,11 @@ vfs_context_create(vfs_context_t context)
 int
 vfs_context_rele(vfs_context_t context)
 {
-	if (context)
+	if (context) {
+		if (IS_VALID_CRED(context->vc_ucred))
+			kauth_cred_unref(&context->vc_ucred);
 		kfree(context, sizeof(struct vfs_context));
+	}
 	return(0);
 }
 
@@ -1511,6 +1520,39 @@ current_rootdir(void)
 	return vp;
 }
 
+/*
+ * Get a filesec and optional acl contents from an extended attribute.
+ * Function will attempt to retrive ACL, UUID, and GUID information using a
+ * read of a named extended attribute (KAUTH_FILESEC_XATTR).
+ *
+ * Parameters:	vp			The vnode on which to operate.
+ *		fsecp			The filesec (and ACL, if any) being
+ *					retrieved.
+ *		ctx			The vnode context in which the
+ *					operation is to be attempted.
+ *
+ * Returns:	0			Success
+ *		!0			errno value
+ *
+ * Notes:	The kauth_filesec_t in '*fsecp', if retrieved, will be in
+ *		host byte order, as will be the ACL contents, if any.
+ *		Internally, we will cannonize these values from network (PPC)
+ *		byte order after we retrieve them so that the on-disk contents
+ *		of the extended attribute are identical for both PPC and Intel
+ *		(if we were not being required to provide this service via
+ *		fallback, this would be the job of the filesystem
+ *		'VNOP_GETATTR' call).
+ *
+ *		We use ntohl() because it has a transitive property on Intel
+ *		machines and no effect on PPC mancines.  This guarantees us
+ *
+ * XXX:		Deleting rather than ignoreing a corrupt security structure is
+ *		probably the only way to reset it without assistance from an
+ *		file system integrity checking tool.  Right now we ignore it.
+ *
+ * XXX:		We should enummerate the possible errno values here, and where
+ *		in the code they originated.
+ */
 static int
 vnode_get_filesec(vnode_t vp, kauth_filesec_t *fsecp, vfs_context_t ctx)
 {
@@ -1519,6 +1561,9 @@ vnode_get_filesec(vnode_t vp, kauth_filesec_t *fsecp, vfs_context_t ctx)
 	size_t	fsec_size;
 	size_t	xsize, rsize;
 	int	error;
+	int	i;
+	uint32_t	host_fsec_magic;
+	uint32_t	host_acl_entrycount;
 
 	fsec = NULL;
 	fsec_uio = NULL;
@@ -1532,6 +1577,18 @@ vnode_get_filesec(vnode_t vp, kauth_filesec_t *fsecp, vfs_context_t ctx)
 		/* either way, we are done */
 		goto out;
 	}
+
+	/*
+	 * To be valid, a kauth_filesec_t must be large enough to hold a zero
+	 * ACE entrly ACL, and if it's larger than that, it must have the right
+	 * number of bytes such that it contains an atomic number of ACEs,
+	 * rather than partial entries.  Otherwise, we ignore it.
+	 */
+	if (!KAUTH_FILESEC_VALID(xsize)) {
+		KAUTH_DEBUG("    ERROR - Bogus kauth_fiilesec_t: %ld bytes", xsize);	
+		error = 0;
+		goto out;
+	}
 				
 	/* how many entries would fit? */
 	fsec_size = KAUTH_FILESEC_COUNT(xsize);
@@ -1562,28 +1619,38 @@ vnode_get_filesec(vnode_t vp, kauth_filesec_t *fsecp, vfs_context_t ctx)
 	}
 
 	/*
-	 * Validate security structure.  If it's corrupt, we will
-	 * just ignore it.
+	 * Validate security structure; the validation must take place in host
+	 * byte order.  If it's corrupt, we will just ignore it.
 	 */
+
+	/* Validate the size before trying to convert it */
 	if (rsize < KAUTH_FILESEC_SIZE(0)) {
 		KAUTH_DEBUG("ACL - DATA TOO SMALL (%d)", rsize);
 		goto out;
 	}
-	if (fsec->fsec_magic != KAUTH_FILESEC_MAGIC) {
-		KAUTH_DEBUG("ACL - BAD MAGIC %x", fsec->fsec_magic);
-		goto out;
-	}
-	if ((fsec->fsec_acl.acl_entrycount != KAUTH_FILESEC_NOACL) &&
-	    (fsec->fsec_acl.acl_entrycount > KAUTH_ACL_MAX_ENTRIES)) {
-		KAUTH_DEBUG("ACL - BAD ENTRYCOUNT %x", fsec->fsec_entrycount);
+
+	/* Validate the magic number before trying to convert it */
+	host_fsec_magic = ntohl(KAUTH_FILESEC_MAGIC);
+	if (fsec->fsec_magic != host_fsec_magic) {
+		KAUTH_DEBUG("ACL - BAD MAGIC %x", host_fsec_magic);
 		goto out;
 	}
-	if ((fsec->fsec_acl.acl_entrycount != KAUTH_FILESEC_NOACL) &&
-	    (KAUTH_FILESEC_SIZE(fsec->fsec_acl.acl_entrycount) > rsize)) {
-		KAUTH_DEBUG("ACL - BUFFER OVERFLOW (%d entries too big for %d)", fsec->fsec_acl.acl_entrycount, rsize);
-		goto out;
+
+	/* Validate the entry count before trying to convert it. */
+	host_acl_entrycount = ntohl(fsec->fsec_acl.acl_entrycount);
+	if (host_acl_entrycount != KAUTH_FILESEC_NOACL) {
+		if (host_acl_entrycount > KAUTH_ACL_MAX_ENTRIES) {
+			KAUTH_DEBUG("ACL - BAD ENTRYCOUNT %x", host_acl_entrycount);
+			goto out;
+		}
+	    	if (KAUTH_FILESEC_SIZE(host_acl_entrycount) > rsize) {
+			KAUTH_DEBUG("ACL - BUFFER OVERFLOW (%d entries too big for %d)", host_acl_entrycount, rsize);
+			goto out;
+		}
 	}
 
+	kauth_filesec_acl_setendian(KAUTH_ENDIAN_HOST, fsec, NULL);
+
 	*fsecp = fsec;
 	fsec = NULL;
 	error = 0;
@@ -1597,11 +1664,44 @@ out:
 	return(error);
 }
 
+/*
+ * Set a filesec and optional acl contents into an extended attribute.
+ * function will attempt to store ACL, UUID, and GUID information using a
+ * write to a named extended attribute (KAUTH_FILESEC_XATTR).  The 'acl'
+ * may or may not point to the `fsec->fsec_acl`, depending on whether the
+ * original caller supplied an acl.
+ *
+ * Parameters:	vp			The vnode on which to operate.
+ *		fsec			The filesec being set.
+ *		acl			The acl to be associated with 'fsec'.
+ *		ctx			The vnode context in which the
+ *					operation is to be attempted.
+ *
+ * Returns:	0			Success
+ *		!0			errno value
+ *
+ * Notes:	Both the fsec and the acl are always valid.
+ *
+ *		The kauth_filesec_t in 'fsec', if any, is in host byte order,
+ *		as are the acl contents, if they are used.  Internally, we will
+ *		cannonize these values into network (PPC) byte order before we
+ *		attempt to write them so that the on-disk contents of the
+ *		extended attribute are identical for both PPC and Intel (if we
+ *		were not being required to provide this service via fallback,
+ *		this would be the job of the filesystem 'VNOP_SETATTR' call).
+ *		We reverse this process on the way out, so we leave with the
+ *		same byte order we started with.
+ *
+ * XXX:		We should enummerate the possible errno values here, and where
+ *		in the code they originated.
+ */
 static int
 vnode_set_filesec(vnode_t vp, kauth_filesec_t fsec, kauth_acl_t acl, vfs_context_t ctx)
 {
-	uio_t	fsec_uio;
-	int	error;
+	uio_t		fsec_uio;
+	int		error;
+	int		i;
+	uint32_t	saved_acl_copysize;
 
 	fsec_uio = NULL;
 	
@@ -1610,8 +1710,16 @@ vnode_set_filesec(vnode_t vp, kauth_filesec_t fsec, kauth_acl_t acl, vfs_context
 		error = ENOMEM;
 		goto out;
 	}
+	/*
+	 * Save the pre-converted ACL copysize, because it gets swapped too
+	 * if we are running with the wrong endianness.
+	 */
+	saved_acl_copysize = KAUTH_ACL_COPYSIZE(acl);
+
+	kauth_filesec_acl_setendian(KAUTH_ENDIAN_DISK, fsec, acl);
+
 	uio_addiov(fsec_uio, CAST_USER_ADDR_T(fsec), sizeof(struct kauth_filesec) - sizeof(struct kauth_acl));
-	uio_addiov(fsec_uio, CAST_USER_ADDR_T(acl), KAUTH_ACL_COPYSIZE(acl));
+	uio_addiov(fsec_uio, CAST_USER_ADDR_T(acl), saved_acl_copysize);
 	error = vn_setxattr(vp,
 	    KAUTH_FILESEC_XATTR,
 	    fsec_uio,
@@ -1619,6 +1727,8 @@ vnode_set_filesec(vnode_t vp, kauth_filesec_t fsec, kauth_acl_t acl, vfs_context
 	    ctx);
 	VFS_DEBUG(ctx, vp, "SETATTR - set ACL returning %d", error);
 
+	kauth_filesec_acl_setendian(KAUTH_ENDIAN_HOST, fsec, acl);
+
 out:
 	if (fsec_uio != NULL)
 		uio_free(fsec_uio);
@@ -1832,6 +1942,28 @@ out:
 	return(error);
 }
 
+/*
+ * Set the attributes on a vnode in a vnode context.
+ *
+ * Parameters:	vp			The vnode whose attributes to set.
+ *		vap			A pointer to the attributes to set.
+ *		ctx			The vnode context in which the
+ *					operation is to be attempted.
+ *
+ * Returns:	0			Success
+ *		!0			errno value
+ *
+ * Notes:	The kauth_filesec_t in 'vap', if any, is in host byte order.
+ *
+ *		The contents of the data area pointed to by 'vap' may be
+ *		modified if the vnode is on a filesystem which has been
+ *		mounted with ingore ownership flags, or by the underlyng
+ *		VFS itself, or by the fallback code, if the underlying VFS
+ *		does not support ACL, UUID, or GUUID attributes directly.
+ *
+ * XXX:		We should enummerate the possible errno values here, and where
+ *		in the code they originated.
+ */
 int
 vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 {
@@ -1841,8 +1973,10 @@ vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 	 * Make sure the filesystem is mounted R/W.
 	 * If not, return an error.
 	 */
-	if (vfs_isrdonly(vp->v_mount))
-		return(EROFS);
+	if (vfs_isrdonly(vp->v_mount)) {
+		error = EROFS;
+		goto out;
+	}
 	
 	/*
 	 * If ownership is being ignored on this volume, we silently discard
@@ -1864,7 +1998,8 @@ vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 	if (!vfs_extendedsecurity(vnode_mount(vp)) &&
 	    (VATTR_IS_ACTIVE(vap, va_acl) || VATTR_IS_ACTIVE(vap, va_uuuid) || VATTR_IS_ACTIVE(vap, va_guuid))) {
 		KAUTH_DEBUG("SETATTR - returning ENOTSUP to request to set extended security");
-		return(ENOTSUP);
+		error = ENOTSUP;
+		goto out;
 	}
 
 	error = VNOP_SETATTR(vp, vap, ctx);
@@ -1895,13 +2030,35 @@ vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 		        add_fsevent(FSE_CHOWN, ctx, FSE_ARG_VNODE, vp, FSE_ARG_DONE);
 	    }
 	}
+
+out:
 	return(error);
 }
 
 /*
- * Following an operation which sets attributes (setattr, create, etc.) we may
- * need to perform fallback operations to get attributes saved.
- */ 
+ * Fallback for setting the attributes on a vnode in a vnode context.  This
+ * Function will attempt to store ACL, UUID, and GUID information utilizing
+ * a read/modify/write operation against an EA used as a backing store for
+ * the object.
+ *
+ * Parameters:	vp			The vnode whose attributes to set.
+ *		vap			A pointer to the attributes to set.
+ *		ctx			The vnode context in which the
+ *					operation is to be attempted.
+ *
+ * Returns:	0			Success
+ *		!0			errno value
+ *
+ * Notes:	The kauth_filesec_t in 'vap', if any, is in host byte order,
+ *		as are the fsec and lfsec, if they are used.
+ *
+ *		The contents of the data area pointed to by 'vap' may be
+ *		modified to indicate that the attribute is supported for
+ *		any given requested attribute.
+ *
+ * XXX:		We should enummerate the possible errno values here, and where
+ *		in the code they originated.
+ */
 int
 vnode_setattr_fallback(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 {
@@ -1915,7 +2072,8 @@ vnode_setattr_fallback(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 	/*
 	 * Extended security fallback via extended attributes.
 	 *
-	 * Note that we do not free the filesec; the caller is expected to do this.
+	 * Note that we do not free the filesec; the caller is expected to
+	 * do this.
 	 */
 	if (VATTR_NOT_RETURNED(vap, va_acl) ||
 	    VATTR_NOT_RETURNED(vap, va_uuuid) ||
@@ -1923,7 +2081,8 @@ vnode_setattr_fallback(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 		VFS_DEBUG(ctx, vp, "SETATTR - doing filesec fallback");
 
 		/*
-		 * Fail for file types that we don't permit extended security to be set on.
+		 * Fail for file types that we don't permit extended security
+		 * to be set on.
 		 */
 		if ((vp->v_type != VDIR) && (vp->v_type != VLNK) && (vp->v_type != VREG)) {
 			VFS_DEBUG(ctx, vp, "SETATTR - Can't write ACL to file type %d", vnode_vtype(vp));
@@ -1932,8 +2091,9 @@ vnode_setattr_fallback(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 		}
 
 		/*
-		 * If we don't have all the extended security items, we need to fetch the existing
-		 * data to perform a read-modify-write operation.
+		 * If we don't have all the extended security items, we need
+		 * to fetch the existing data to perform a read-modify-write
+		 * operation.
 		 */
 		fsec = NULL;
 		if (!VATTR_IS_ACTIVE(vap, va_acl) ||
@@ -1988,7 +2148,8 @@ vnode_setattr_fallback(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 		}
 		
 		/*
-		 * If the filesec data is all invalid, we can just remove the EA completely.
+		 * If the filesec data is all invalid, we can just remove
+		 * the EA completely.
 		 */
 		if ((facl->acl_entrycount == KAUTH_FILESEC_NOACL) &&
 		    kauth_guid_equal(&fsec->fsec_owner, &kauth_null_guid) &&
diff --git a/bsd/vfs/vfs_bio.c b/bsd/vfs/vfs_bio.c
index 971c8d350..b8dfb6fc8 100644
--- a/bsd/vfs/vfs_bio.c
+++ b/bsd/vfs/vfs_bio.c
@@ -221,7 +221,7 @@ buf_timestamp(void)
 int lru_is_stale = LRU_IS_STALE;
 int age_is_stale = AGE_IS_STALE;
 int meta_is_stale = META_IS_STALE;
-
+static int boot_nbuf = 0;
 
 
 /* LIST_INSERT_HEAD() with assertions */
@@ -239,15 +239,19 @@ blistenterhead(struct bufhashhdr * head, buf_t bp)
 static __inline__ void 
 binshash(buf_t bp, struct bufhashhdr *dp)
 {
+#if DIAGNOSTIC
 	buf_t	nbp;
+#endif /* DIAGNOSTIC */
 
 	BHASHENTCHECK(bp);
 
+#if DIAGNOSTIC
 	nbp = dp->lh_first;
 	for(; nbp != NULL; nbp = nbp->b_hash.le_next) {
 		if(nbp == bp) 
 			panic("buf already in hashlist");
 	}
+#endif /* DIAGNOSTIC */
 
 	blistenterhead(dp, bp);
 }
@@ -1377,15 +1381,17 @@ bufinit()
 	int	metabuf;
 	long	whichq;
 
+	nbuf = 0;
 	/* Initialize the buffer queues ('freelists') and the hash table */
 	for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++)
 		TAILQ_INIT(dp);
-	bufhashtbl = hashinit(nbuf, M_CACHE, &bufhash);
+	bufhashtbl = hashinit(nbuf_hashelements, M_CACHE, &bufhash);
 
-	metabuf = nbuf/8; /* reserved for meta buf */
+	metabuf = max_nbuf_headers/8; /* reserved for meta buf */
 
 	/* Initialize the buffer headers */
-	for (i = 0; i < nbuf; i++) {
+	for (i = 0; i < max_nbuf_headers; i++) {
+		nbuf++;
 		bp = &buf[i];
 		bufhdrinit(bp);
 
@@ -1404,24 +1410,24 @@ bufinit()
 		binshash(bp, &invalhash);
 	}
 
+	boot_nbuf = nbuf;
+
 	for (; i < nbuf + niobuf; i++) {
 		bp = &buf[i];
 		bufhdrinit(bp);
 		binsheadfree(bp, &iobufqueue, -1);
 	}
 
-        /*
+    /*
 	 * allocate lock group attribute and group
 	 */
-        buf_mtx_grp_attr = lck_grp_attr_alloc_init();
-	//lck_grp_attr_setstat(buf_mtx_grp_attr);
+    buf_mtx_grp_attr = lck_grp_attr_alloc_init();
 	buf_mtx_grp = lck_grp_alloc_init("buffer cache", buf_mtx_grp_attr);
 		
 	/*
 	 * allocate the lock attribute
 	 */
 	buf_mtx_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(buf_mtx_attr);
 
 	/*
 	 * allocate and initialize mutex's for the buffer and iobuffer pools
@@ -1477,7 +1483,7 @@ bio_doread(vnode_t vp, daddr64_t blkno, int size, ucred_t cred, int async, int q
 
 		/* Start I/O for the buffer (keeping credentials). */
 		SET(bp->b_flags, B_READ | async);
-		if (cred != NOCRED && bp->b_rcred == NOCRED) {
+		if (IS_VALID_CRED(cred) && !IS_VALID_CRED(bp->b_rcred)) {
 			kauth_cred_ref(cred);
 			bp->b_rcred = cred;
 		}
@@ -2497,7 +2503,7 @@ allocbuf(buf_t bp, int size)
 						bp->b_datap = (uintptr_t)zalloc(z);
 					} else {
 					        bp->b_datap = (uintptr_t)NULL;
-					        kmem_alloc(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size);
+					        kmem_alloc_wired(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size);
 						CLR(bp->b_flags, B_ZALLOC);
 					}
 					bcopy((void *)elem, (caddr_t)bp->b_datap, bp->b_bufsize);
@@ -2510,7 +2516,7 @@ allocbuf(buf_t bp, int size)
 				if ((vm_size_t)bp->b_bufsize < desired_size) {
 					/* reallocate to a bigger size */
 				        bp->b_datap = (uintptr_t)NULL;
-					kmem_alloc(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size);
+					kmem_alloc_wired(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size);
 					bcopy((const void *)elem, (caddr_t)bp->b_datap, bp->b_bufsize);
 					kmem_free(kernel_map, elem, bp->b_bufsize); 
 				} else {
@@ -2525,7 +2531,7 @@ allocbuf(buf_t bp, int size)
 				bp->b_datap = (uintptr_t)zalloc(z);
 				SET(bp->b_flags, B_ZALLOC);
 			} else
-				kmem_alloc(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size);
+				kmem_alloc_wired(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size);
 		}
 	}
 	bp->b_bufsize = desired_size;
@@ -2573,6 +2579,16 @@ start:
 	if ((*queue > BQUEUES) || (*queue < 0)
 		|| (*queue == BQ_LAUNDRY) || (*queue == BQ_LOCKED))
 		*queue = BQ_EMPTY;
+	/* need to grow number of bufs, add another one rather than recycling */
+	if (nbuf < max_nbuf_headers) {
+		/*
+		 * Increment  count now as lock 
+		 * is dropped for allocation.
+		 * That avoids over commits
+		 */
+		nbuf++;
+		goto add_newbufs;
+	}
 
 	/*
 	 * (*queue == BQUEUES) means no preference
@@ -2599,6 +2615,13 @@ start:
 			*queue = BQ_EMPTY;
 			goto found;
 		}
+		/*
+		 * We have seen is this is hard to trigger.
+		 * This is an overcommit of nbufs but needed 
+		 * in some scenarios with diskiamges
+		 */
+
+add_newbufs:
 		lck_mtx_unlock(buf_mtxp);
 
 		/* Create a new temporary buffer header */
@@ -2616,6 +2639,9 @@ start:
 			buf_hdr_count++;
 			goto found;
 		}
+		/* subtract already accounted bufcount */
+		nbuf--;
+
 		bufstats.bufs_sleeps++;
 
 		/* wait for a free buffer of any kind */
@@ -2625,7 +2651,6 @@ start:
 		/* the hz value is 100; which leads to 10ms */
 		ts.tv_nsec = (slptimeo % 1000) * NSEC_PER_USEC * 1000 * 10;
 		msleep(&needbuffer, buf_mtxp, slpflag|(PRIBIO+1), (char *)"getnewbuf", &ts);
-
 		return (0);
 	}
 
@@ -2713,9 +2738,6 @@ found:
 static int
 bcleanbuf(buf_t bp)
 {
-	ucred_t	cred;
-
-
 	/* Remove from the queue */
 	bremfree_locked(bp);
 
@@ -2796,15 +2818,11 @@ bcleanbuf(buf_t bp)
 	bp->b_validoff = bp->b_validend = 0;
 
 	/* nuke any credentials we were holding */
-	cred = bp->b_rcred;
-	if (cred != NOCRED) {
-		bp->b_rcred = NOCRED; 
-		kauth_cred_rele(cred);
+	if (IS_VALID_CRED(bp->b_rcred)) {
+		kauth_cred_unref(&bp->b_rcred);
 	}
-	cred = bp->b_wcred;
-	if (cred != NOCRED) {
-		bp->b_wcred = NOCRED;
-		kauth_cred_rele(cred);
+	if (IS_VALID_CRED(bp->b_wcred)) {
+		kauth_cred_unref(&bp->b_wcred);
 	}
 	lck_mtx_lock(buf_mtxp);
 
@@ -3119,9 +3137,12 @@ count_busy_buffers(void)
 	buf_t	bp;
 	int	nbusy = 0;
 
-	for (bp = &buf[nbuf]; --bp >= buf; )
+	lck_mtx_lock(buf_mtxp);
+	for (bp = &buf[boot_nbuf]; --bp >= buf; )
 	        if (!ISSET(bp->b_flags, B_INVAL) && ISSET(bp->b_lflags, BL_BUSY))
 			nbusy++;
+	lck_mtx_unlock(buf_mtxp);
+
 	return (nbusy);
 }
 
diff --git a/bsd/vfs/vfs_cache.c b/bsd/vfs/vfs_cache.c
index 1c2c9230c..954f63715 100644
--- a/bsd/vfs/vfs_cache.c
+++ b/bsd/vfs/vfs_cache.c
@@ -102,15 +102,37 @@ u_long	nchash;				/* size of hash table - 1 */
 long	numcache;			/* number of cache entries allocated */
 int 	desiredNodes;
 int 	desiredNegNodes;
+int	ncs_negtotal;
 TAILQ_HEAD(, namecache) nchead;		/* chain of all name cache entries */
 TAILQ_HEAD(, namecache) neghead;	/* chain of only negative cache entries */
+
+
+#if COLLECT_STATS
+
 struct	nchstats nchstats;		/* cache effectiveness statistics */
 
+#define	NCHSTAT(v) {		\
+        nchstats.v++;		\
+}
+#define NAME_CACHE_LOCK()		name_cache_lock()
+#define NAME_CACHE_UNLOCK()		name_cache_unlock()
+#define	NAME_CACHE_LOCK_SHARED()	name_cache_lock()
+
+#else
+
+#define NCHSTAT(v)
+#define NAME_CACHE_LOCK()		name_cache_lock()
+#define NAME_CACHE_UNLOCK()		name_cache_unlock()
+#define	NAME_CACHE_LOCK_SHARED()	name_cache_lock_shared()
+
+#endif
+
+
 /* vars for name cache list lock */
 lck_grp_t * namecache_lck_grp;
 lck_grp_attr_t * namecache_lck_grp_attr;
 lck_attr_t * namecache_lck_attr;
-lck_mtx_t * namecache_mtx_lock;
+lck_rw_t * namecache_rw_lock;
 
 static vnode_t cache_lookup_locked(vnode_t dvp, struct componentname *cnp);
 static int  remove_name_locked(const char *);
@@ -163,7 +185,7 @@ build_path(vnode_t first_vp, char *buff, int buflen, int *outlen)
 		        vp = vp->v_mount->mnt_vnodecovered;
 		}
 	}
-	name_cache_lock();
+	NAME_CACHE_LOCK_SHARED();
 
 	while (vp && vp->v_parent != vp) {
 	        /*
@@ -224,7 +246,7 @@ build_path(vnode_t first_vp, char *buff, int buflen, int *outlen)
 		        vp = vp->v_mount->mnt_vnodecovered;
 		}
 	}
-	name_cache_unlock();
+	NAME_CACHE_UNLOCK();
 out:
 	/*
 	 * slide it down to the beginning of the buffer
@@ -248,7 +270,7 @@ vnode_getparent(vnode_t vp)
         vnode_t pvp = NULLVP;
 	int	pvid;
 
-        name_cache_lock();
+	NAME_CACHE_LOCK_SHARED();
 	/*
 	 * v_parent is stable behind the name_cache lock
 	 * however, the only thing we can really guarantee
@@ -259,13 +281,12 @@ vnode_getparent(vnode_t vp)
 	if ( (pvp = vp->v_parent) != NULLVP ) {
 	        pvid = pvp->v_id;
 
-		name_cache_unlock();
+		NAME_CACHE_UNLOCK();
 
 		if (vnode_getwithvid(pvp, pvid) != 0)
 		        pvp = NULL;
 	} else
-	        name_cache_unlock();
-
+	        NAME_CACHE_UNLOCK();
 	return (pvp);
 }
 
@@ -274,11 +295,11 @@ vnode_getname(vnode_t vp)
 {
         char *name = NULL;
 
-        name_cache_lock();
+	NAME_CACHE_LOCK();
 	
 	if (vp->v_name)
 	        name = add_name_locked(vp->v_name, strlen(vp->v_name), 0, 0);
-	name_cache_unlock();
+	NAME_CACHE_UNLOCK();
 
 	return (name);
 }
@@ -286,11 +307,11 @@ vnode_getname(vnode_t vp)
 void
 vnode_putname(char *name)
 {
-	name_cache_lock();
+        NAME_CACHE_LOCK();
 
 	remove_name_locked(name);
 
-	name_cache_unlock();
+	NAME_CACHE_UNLOCK();
 }
 
 
@@ -319,7 +340,7 @@ vnode_update_identity(vnode_t vp, vnode_t dvp, char *name, int name_len, int nam
 		        dvp = NULLVP;
 	} else
 	        dvp = NULLVP;
-	name_cache_lock();
+	NAME_CACHE_LOCK();
 
 	if ( (flags & VNODE_UPDATE_NAME) && (name != vp->v_name) ) {
 	        if (vp->v_name != NULL) {
@@ -346,7 +367,7 @@ vnode_update_identity(vnode_t vp, vnode_t dvp, char *name, int name_len, int nam
 	        while ( (ncp = LIST_FIRST(&vp->v_nclinks)) )
 		        cache_delete(ncp, 1);
 	}
-	name_cache_unlock();
+	NAME_CACHE_UNLOCK();
 	
 	if (dvp != NULLVP)
 	        vnode_rele(dvp);
@@ -399,10 +420,10 @@ vnode_update_identity(vnode_t vp, vnode_t dvp, char *name, int name_len, int nam
 				 * vnode_reclaim for each of the vnodes in the uu_vreclaims
 				 * list, we won't recurse back through here
 				 */
-			        name_cache_lock();
+			        NAME_CACHE_LOCK();
 				old_parentvp = vp->v_parent;
 				vp->v_parent = NULLVP;
-				name_cache_unlock();
+				NAME_CACHE_UNLOCK();
 			} else {
 			        /*
 				 * we're done... we ran into a vnode that isn't
@@ -459,19 +480,17 @@ void vnode_set_hard_link(vnode_t vp)
 
 void vnode_uncache_credentials(vnode_t vp)
 {
-        kauth_cred_t ucred = NULL;
-
-        if (vp->v_cred) {
-	        vnode_lock(vp);
+        kauth_cred_t ucred = NOCRED;
 
+	vnode_lock(vp);
+        if (IS_VALID_CRED(vp->v_cred)) {
 		ucred = vp->v_cred;
-		vp->v_cred = NULL;
-
-		vnode_unlock(vp);
-
-		if (ucred)
-		        kauth_cred_rele(ucred);
+		vp->v_cred = NOCRED;
 	}
+	vnode_unlock(vp);
+
+	if (ucred != NOCRED)
+		kauth_cred_unref(&ucred);
 }
 
 
@@ -483,8 +502,8 @@ void vnode_cache_credentials(vnode_t vp, vfs_context_t context)
 
 	ucred = vfs_context_ucred(context);
 
-	if (vp->v_cred != ucred || (vp->v_mount->mnt_kern_flag & MNTK_AUTH_OPAQUE)) {
-		vnode_lock(vp);
+	vnode_lock(vp);
+	if (IS_VALID_CRED(ucred) && (vp->v_cred != ucred || (vp->v_mount->mnt_kern_flag & MNTK_AUTH_OPAQUE))) {
 
 		microuptime(&tv);
 		vp->v_cred_timestamp = tv.tv_sec;
@@ -495,11 +514,11 @@ void vnode_cache_credentials(vnode_t vp, vfs_context_t context)
 			tcred = vp->v_cred;
 			vp->v_cred = ucred;
 		}
-		vnode_unlock(vp);
-	
-		if (tcred)
-			kauth_cred_rele(tcred);
 	}
+	vnode_unlock(vp);
+
+	if (IS_VALID_CRED(tcred))
+		kauth_cred_unref(&tcred);
 }
 
 /*	reverse_lookup - lookup by walking back up the parent chain while leveraging
@@ -521,7 +540,7 @@ reverse_lookup(vnode_t start_vp, vnode_t *lookup_vpp, struct filedesc *fdp, vfs_
 	ucred = vfs_context_ucred(context);
 	*lookup_vpp = start_vp;
 
-	name_cache_lock();
+	NAME_CACHE_LOCK_SHARED();
 
 	if ( dp->v_mount && (dp->v_mount->mnt_kern_flag & MNTK_AUTH_OPAQUE) ) {
 		auth_opaque = 1;
@@ -532,6 +551,7 @@ reverse_lookup(vnode_t start_vp, vnode_t *lookup_vpp, struct filedesc *fdp, vfs_
 
 		if (auth_opaque && ((tv.tv_sec - dp->v_cred_timestamp) > VCRED_EXPIRED))
 			break;
+		/* XXX should be safe without vnode_lock() */
 		if (dp->v_cred != ucred)
 			break;
 		/*
@@ -557,7 +577,7 @@ reverse_lookup(vnode_t start_vp, vnode_t *lookup_vpp, struct filedesc *fdp, vfs_
 
 	vid = dp->v_id;
 	
-	name_cache_unlock();
+	NAME_CACHE_UNLOCK();
 	
 	if (done == 0 && dp != start_vp) {
 		if (vnode_getwithvid(dp, vid) != 0) {
@@ -583,8 +603,7 @@ cache_lookup_path(struct nameidata *ndp, struct componentname *cnp, vnode_t dp,
 	ucred = vfs_context_ucred(context);
 	*trailing_slash = 0;
 
-	name_cache_lock();
-
+	NAME_CACHE_LOCK_SHARED();
 
 	if ( dp->v_mount && (dp->v_mount->mnt_kern_flag & MNTK_AUTH_OPAQUE) ) {
 		auth_opaque = 1;
@@ -648,6 +667,7 @@ cache_lookup_path(struct nameidata *ndp, struct componentname *cnp, vnode_t dp,
 		if (auth_opaque && ((tv.tv_sec - dp->v_cred_timestamp) > VCRED_EXPIRED))
 		        break;
 
+		/* XXX should be safe without vnode_lock() */
 		if (dp->v_cred != ucred)
 		        break;
 		/*
@@ -714,7 +734,7 @@ cache_lookup_path(struct nameidata *ndp, struct componentname *cnp, vnode_t dp,
 	        vvid = vp->v_id;
 	vid = dp->v_id;
 	
-	name_cache_unlock();
+	NAME_CACHE_UNLOCK();
 
 
 	if ((vp != NULLVP) && (vp->v_type != VLNK) &&
@@ -800,11 +820,14 @@ cache_lookup_locked(vnode_t dvp, struct componentname *cnp)
 			        break;
 		}
 	}
-	if (ncp == 0)
+	if (ncp == 0) {
 		/*
 		 * We failed to find an entry
 		 */
+		NCHSTAT(ncs_miss);
 		return (NULL);
+	}
+	NCHSTAT(ncs_goodhits);
 
 	vp = ncp->nc_vp;
 	if (vp && (vp->v_flag & VISHARDLINK)) {
@@ -873,12 +896,14 @@ cache_lookup(dvp, vpp, cnp)
 	register long namelen = cnp->cn_namelen;
 	char *nameptr = cnp->cn_nameptr;
 	unsigned int hashval = (cnp->cn_hash & NCHASHMASK);
+	boolean_t	have_exclusive = FALSE;
 	uint32_t vid;
 	vnode_t	 vp;
 
-	name_cache_lock();
+	NAME_CACHE_LOCK_SHARED();
 
 	ncpp = NCHHASH(dvp, cnp->cn_hash);
+relook:
 	LIST_FOREACH(ncp, ncpp, nc_hash) {
 	        if ((ncp->nc_dvp == dvp) && (ncp->nc_hashval == hashval)) {
 		        if (memcmp(ncp->nc_name, nameptr, namelen) == 0 && ncp->nc_name[namelen] == 0)
@@ -887,31 +912,39 @@ cache_lookup(dvp, vpp, cnp)
 	}
 	/* We failed to find an entry */
 	if (ncp == 0) {
-		nchstats.ncs_miss++;
-		name_cache_unlock();
+		NCHSTAT(ncs_miss);
+		NAME_CACHE_UNLOCK();
 		return (0);
 	}
 
 	/* We don't want to have an entry, so dump it */
 	if ((cnp->cn_flags & MAKEENTRY) == 0) {
-		nchstats.ncs_badhits++;
-		cache_delete(ncp, 1);
-		name_cache_unlock();
-		return (0);
+	        if (have_exclusive == TRUE) {
+		        NCHSTAT(ncs_badhits);
+			cache_delete(ncp, 1);
+			NAME_CACHE_UNLOCK();
+			return (0);
+		}
+		NAME_CACHE_UNLOCK();
+		NAME_CACHE_LOCK();
+		have_exclusive = TRUE;
+		goto relook;
 	} 
 	vp = ncp->nc_vp;
 
 	/* We found a "positive" match, return the vnode */
         if (vp) {
-		nchstats.ncs_goodhits++;
+		NCHSTAT(ncs_goodhits);
 
 		vid = vp->v_id;
-		name_cache_unlock();
+		NAME_CACHE_UNLOCK();
 
 		if (vnode_getwithvid(vp, vid)) {
-		        name_cache_lock();
-			nchstats.ncs_badvid++;
-			name_cache_unlock();
+#if COLLECT_STATS
+		        NAME_CACHE_LOCK();
+			NCHSTAT(ncs_badvid);
+			NAME_CACHE_UNLOCK();
+#endif
 			return (0);
 		}
 		*vpp = vp;
@@ -920,21 +953,27 @@ cache_lookup(dvp, vpp, cnp)
 
 	/* We found a negative match, and want to create it, so purge */
 	if (cnp->cn_nameiop == CREATE || cnp->cn_nameiop == RENAME) {
-		nchstats.ncs_badhits++;
-		cache_delete(ncp, 1);
-		name_cache_unlock();
-		return (0);
+	        if (have_exclusive == TRUE) {
+		        NCHSTAT(ncs_badhits);
+			cache_delete(ncp, 1);
+			NAME_CACHE_UNLOCK();
+			return (0);
+		}
+		NAME_CACHE_UNLOCK();
+		NAME_CACHE_LOCK();
+		have_exclusive = TRUE;
+		goto relook;
 	}
 
 	/*
 	 * We found a "negative" match, ENOENT notifies client of this match.
 	 * The nc_whiteout field records whether this is a whiteout.
 	 */
-	nchstats.ncs_neghits++;
+	NCHSTAT(ncs_neghits);
 
 	if (ncp->nc_whiteout)
 	        cnp->cn_flags |= ISWHITEOUT;
-	name_cache_unlock();
+	NAME_CACHE_UNLOCK();
 	return (ENOENT);
 }
 
@@ -953,7 +992,7 @@ cache_enter(dvp, vp, cnp)
         if (cnp->cn_hash == 0)
 	        cnp->cn_hash = hash_string(cnp->cn_nameptr, cnp->cn_namelen);
 
-	name_cache_lock();
+	NAME_CACHE_LOCK();
 
 	/* if the entry is for -ve caching vp is null */
 	if ((vp != NULLVP) && (LIST_FIRST(&vp->v_nclinks))) {
@@ -961,8 +1000,8 @@ cache_enter(dvp, vp, cnp)
 		 * someone beat us to the punch..
 		 * this vnode is already in the cache
 		 */
-	        name_cache_unlock();
-			return;
+	        NAME_CACHE_UNLOCK();
+		return;
 	}
 	/*
 	 * We allocate a new entry if we are less than the maximum
@@ -989,11 +1028,11 @@ cache_enter(dvp, vp, cnp)
 			* still in use... we need to
 			* delete it before re-using it
 			*/
-			nchstats.ncs_stolen++;
+			NCHSTAT(ncs_stolen);
 			cache_delete(ncp, 0);
 		}
 	}
-	nchstats.ncs_enters++;
+	NCHSTAT(ncs_enters);
 
 	/*
 	 * Fill in cache info, if vp is NULL this is a "negative" cache entry.
@@ -1041,9 +1080,9 @@ cache_enter(dvp, vp, cnp)
 	  
 		if (cnp->cn_flags & ISWHITEOUT)
 		        ncp->nc_whiteout = TRUE;
-		nchstats.ncs_negtotal++;
+		ncs_negtotal++;
 
-		if (nchstats.ncs_negtotal > desiredNegNodes) {
+		if (ncs_negtotal > desiredNegNodes) {
 		       /*
 			* if we've reached our desired limit
 			* of negative cache entries, delete
@@ -1061,7 +1100,7 @@ cache_enter(dvp, vp, cnp)
 	 */
 	LIST_INSERT_HEAD(&dvp->v_ncchildren, ncp, nc_child);
 
-	name_cache_unlock();
+	NAME_CACHE_UNLOCK();
 }
 
 
@@ -1117,31 +1156,34 @@ nchinit(void)
 	
 	/* Allocate mount list lock group attribute and group */
 	namecache_lck_grp_attr= lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(namecache_lck_grp_attr);
 
 	namecache_lck_grp = lck_grp_alloc_init("Name Cache",  namecache_lck_grp_attr);
 	
 	/* Allocate mount list lock attribute */
 	namecache_lck_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(namecache_lck_attr);
 
 	/* Allocate mount list lock */
-	namecache_mtx_lock = lck_mtx_alloc_init(namecache_lck_grp, namecache_lck_attr);
+	namecache_rw_lock = lck_rw_alloc_init(namecache_lck_grp, namecache_lck_attr);
 
 
 }
 
+void
+name_cache_lock_shared(void)
+{
+	lck_rw_lock_shared(namecache_rw_lock);
+}
+
 void
 name_cache_lock(void)
 {
-	lck_mtx_lock(namecache_mtx_lock);
+	lck_rw_lock_exclusive(namecache_rw_lock);
 }
 
 void
 name_cache_unlock(void)
 {
-	lck_mtx_unlock(namecache_mtx_lock);
-
+	lck_rw_done(namecache_rw_lock);
 }
 
 
@@ -1170,7 +1212,7 @@ resize_namecache(u_int newsize)
 	return ENOMEM;
     }
 
-    name_cache_lock();
+    NAME_CACHE_LOCK();
     // do the switch!
     old_table = nchashtbl;
     nchashtbl = new_table;
@@ -1198,7 +1240,7 @@ resize_namecache(u_int newsize)
     desiredNodes = dNodes;
     desiredNegNodes = dNegNodes;
     
-    name_cache_unlock();
+    NAME_CACHE_UNLOCK();
     FREE(old_table, M_CACHE);
 
     return 0;
@@ -1207,13 +1249,13 @@ resize_namecache(u_int newsize)
 static void
 cache_delete(struct namecache *ncp, int age_entry)
 {
-        nchstats.ncs_deletes++;
+        NCHSTAT(ncs_deletes);
 
         if (ncp->nc_vp) {
 	        LIST_REMOVE(ncp, nc_un.nc_link);
 	} else {
 	        TAILQ_REMOVE(&neghead, ncp, nc_un.nc_negentry);
-	        nchstats.ncs_negtotal--;
+	        ncs_negtotal--;
 	}
         LIST_REMOVE(ncp, nc_child);
 
@@ -1251,7 +1293,7 @@ cache_purge(vnode_t vp)
 	if ((LIST_FIRST(&vp->v_nclinks) == NULL) && (LIST_FIRST(&vp->v_ncchildren) == NULL))
 	        return;
 
-	name_cache_lock();
+	NAME_CACHE_LOCK();
 
 	while ( (ncp = LIST_FIRST(&vp->v_nclinks)) )
 	        cache_delete(ncp, 1);
@@ -1259,7 +1301,7 @@ cache_purge(vnode_t vp)
 	while ( (ncp = LIST_FIRST(&vp->v_ncchildren)) )
 	        cache_delete(ncp, 1);
 
-	name_cache_unlock();
+	NAME_CACHE_UNLOCK();
 }
 
 /*
@@ -1274,13 +1316,13 @@ cache_purge_negatives(vnode_t vp)
 {
 	struct namecache *ncp;
 
-	name_cache_lock();
+	NAME_CACHE_LOCK();
 
 	LIST_FOREACH(ncp, &vp->v_ncchildren, nc_child)
 		if (ncp->nc_vp == NULL)
 			cache_delete(ncp , 1);
 
-	name_cache_unlock();
+	NAME_CACHE_UNLOCK();
 }
 
 /*
@@ -1296,7 +1338,7 @@ cache_purgevfs(mp)
 	struct nchashhead *ncpp;
 	struct namecache *ncp;
 
-	name_cache_lock();
+	NAME_CACHE_LOCK();
 	/* Scan hash tables for applicable entries */
 	for (ncpp = &nchashtbl[nchash - 1]; ncpp >= nchashtbl; ncpp--) {
 restart:	  
@@ -1307,7 +1349,7 @@ restart:
 			}
 		}
 	}
-	name_cache_unlock();
+	NAME_CACHE_UNLOCK();
 }
 
 
@@ -1393,9 +1435,9 @@ vfs_addname(const char *name, size_t len, u_int hashval, u_int flags)
 {
         char * ptr;
 
-	name_cache_lock();
+	NAME_CACHE_LOCK();
 	ptr = add_name_locked(name, len, hashval, flags);
-	name_cache_unlock();
+	NAME_CACHE_UNLOCK();
 
 	return(ptr);
 }
@@ -1462,9 +1504,9 @@ vfs_removename(const char *nameref)
 {
 	int i;
 
-	name_cache_lock();
+	NAME_CACHE_LOCK();
 	i = remove_name_locked(nameref);
-	name_cache_unlock();
+	NAME_CACHE_UNLOCK();
 
 	return(i);
 	
@@ -1513,12 +1555,13 @@ dump_string_table(void)
     string_t          *entry;
     u_long            i;
     
-    name_cache_lock();
+    NAME_CACHE_LOCK_SHARED();
+
     for (i = 0; i <= string_table_mask; i++) {
 	head = &string_ref_table[i];
 	for (entry=head->lh_first; entry != NULL; entry=entry->hash_chain.le_next) {
 	    printf("%6d - %s\n", entry->refcount, entry->str);
 	}
     }
-    name_cache_unlock();
+    NAME_CACHE_UNLOCK();
 }
diff --git a/bsd/vfs/vfs_cluster.c b/bsd/vfs/vfs_cluster.c
index 9218925b5..984cb4254 100644
--- a/bsd/vfs/vfs_cluster.c
+++ b/bsd/vfs/vfs_cluster.c
@@ -162,18 +162,16 @@ struct timeval priority_IO_timestamp_for_root;
 
 void
 cluster_init(void) {
-        /*
+    /*
 	 * allocate lock group attribute and group
 	 */
-        cl_mtx_grp_attr = lck_grp_attr_alloc_init();
-	//lck_grp_attr_setstat(cl_mtx_grp_attr);
+    cl_mtx_grp_attr = lck_grp_attr_alloc_init();
 	cl_mtx_grp = lck_grp_alloc_init("cluster I/O", cl_mtx_grp_attr);
 		
 	/*
 	 * allocate the lock attribute
 	 */
 	cl_mtx_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(clf_mtx_attr);
 
 	/*
 	 * allocate and initialize mutex's used to protect updates and waits
diff --git a/bsd/vfs/vfs_fsevents.c b/bsd/vfs/vfs_fsevents.c
index 109a8f72e..922bce40f 100644
--- a/bsd/vfs/vfs_fsevents.c
+++ b/bsd/vfs/vfs_fsevents.c
@@ -47,7 +47,6 @@
 #include <miscfs/specfs/specdev.h>
 #include <miscfs/devfs/devfs.h>
 #include <sys/filio.h>
-#include <architecture/byte_order.h>
 #include <kern/locks.h>
 #include <libkern/OSAtomic.h>
 
@@ -94,6 +93,7 @@ typedef struct fs_event_watcher {
     int32_t      eventq_size;            // number of event pointers in queue
     int32_t      rd, wr;                 // indices to the event_queue
     int32_t      blockers;
+    int32_t      num_readers;
 } fs_event_watcher;
 
 // fs_event_watcher flags
@@ -255,7 +255,7 @@ add_fsevent(int type, vfs_context_t ctx, ...)
     kfs_event        *kfse;
     fs_event_watcher *watcher;
     va_list           ap;
-    int 	      error = 0;
+    int 	      error = 0, base;
     dev_t             dev = 0;
 
     va_start(ap, ctx);
@@ -273,8 +273,9 @@ add_fsevent(int type, vfs_context_t ctx, ...)
     //       the lock is dropped.
     lock_fs_event_buf();
     
+    base = free_event_idx;
     for(i=0; i < MAX_KFS_EVENTS; i++) {
-	if (fs_event_buf[(free_event_idx + i) % MAX_KFS_EVENTS].type == FSE_INVALID) {
+	if (fs_event_buf[(base + i) % MAX_KFS_EVENTS].type == FSE_INVALID) {
 	    break;
 	}
     }
@@ -296,12 +297,12 @@ add_fsevent(int type, vfs_context_t ctx, ...)
 	return ENOSPC;
     }
 
-    kfse = &fs_event_buf[(free_event_idx + i) % MAX_KFS_EVENTS];
+    kfse = &fs_event_buf[(base + i) % MAX_KFS_EVENTS];
 
-    free_event_idx++;
+    free_event_idx = ((base + i) % MAX_KFS_EVENTS) + 1;
     
     kfse->type     = type;
-    kfse->refcount = 0;
+    kfse->refcount = 1;
     kfse->pid      = p->p_pid;
 
     unlock_fs_event_buf();  // at this point it's safe to unlock
@@ -468,9 +469,8 @@ add_fsevent(int type, vfs_context_t ctx, ...)
     
   clean_up:
     // just in case no one was interested after all...
-    if (num_deliveries == 0) {
+    if (OSAddAtomic(-1, (SInt32 *)&kfse->refcount) == 1) {
 	do_free_event(kfse);
-	free_event_idx = (int)(kfse - &fs_event_buf[0]);
     }	
 
     lck_rw_done(&fsevent_big_lock);
@@ -485,8 +485,10 @@ do_free_event(kfs_event *kfse)
     
     lock_fs_event_buf();
     
-    // mark this fsevent as invalid
-    kfse->type = FSE_INVALID;
+    if (kfse->refcount > 0) {
+	panic("do_free_event: free'ing a kfsevent w/refcount == %d (kfse %p)\n",
+	    kfse->refcount, kfse);
+    }
 
     // make a copy of this so we can free things without
     // holding the fs_event_buf lock
@@ -496,6 +498,9 @@ do_free_event(kfs_event *kfse)
     // and just to be anal, set this so that there are no args
     kfse->args[0].type = FSE_ARG_DONE;
     
+    // mark this fsevent as invalid
+    kfse->type = FSE_INVALID;
+
     free_event_idx = (kfse - fs_event_buf);
 
     unlock_fs_event_buf();
@@ -548,6 +553,7 @@ add_watcher(int8_t *event_list, int32_t num_events, int32_t eventq_size, fs_even
     watcher->rd           = 0;
     watcher->wr           = 0;
     watcher->blockers     = 0;
+    watcher->num_readers  = 0;
 
     lock_watch_list();
 
@@ -656,9 +662,15 @@ fmod_watch(fs_event_watcher *watcher, struct uio *uio)
 	return EINVAL;
     }
 
+    if (OSAddAtomic(1, (SInt32 *)&watcher->num_readers) != 0) {
+	// don't allow multiple threads to read from the fd at the same time
+	OSAddAtomic(-1, (SInt32 *)&watcher->num_readers);
+	return EAGAIN;
+    }
 
     if (watcher->rd == watcher->wr) {
 	if (watcher->flags & WATCHER_CLOSING) {
+	    OSAddAtomic(-1, (SInt32 *)&watcher->num_readers);
 	    return 0;
 	}
 	OSAddAtomic(1, (SInt32 *)&watcher->blockers);
@@ -669,6 +681,7 @@ fmod_watch(fs_event_watcher *watcher, struct uio *uio)
 	OSAddAtomic(-1, (SInt32 *)&watcher->blockers);
 
 	if (error != 0 || (watcher->flags & WATCHER_CLOSING)) {
+	    OSAddAtomic(-1, (SInt32 *)&watcher->num_readers);
 	    return error;
 	}
     }
@@ -687,6 +700,7 @@ fmod_watch(fs_event_watcher *watcher, struct uio *uio)
 	} 
 
 	if (error) {
+	    OSAddAtomic(-1, (SInt32 *)&watcher->num_readers);
 	    return error;
 	}
 	
@@ -856,6 +870,7 @@ fmod_watch(fs_event_watcher *watcher, struct uio *uio)
     }
 
   get_out:
+    OSAddAtomic(-1, (SInt32 *)&watcher->num_readers);
     return error;
 }
 
@@ -908,6 +923,9 @@ fsevent_unmount(struct mount *mp)
 
 			if (vname)
 			        vnode_putname(vname);
+
+			strcpy(pathbuff, "UNKNOWN-FILE");
+			pathbuff_len = strlen(pathbuff) + 1;
 		}
 
 		// switch the type of the string
@@ -934,10 +952,13 @@ static int fsevents_installed = 0;
 static struct lock__bsd__ fsevents_lck;
 
 typedef struct fsevent_handle {
+    UInt32            flags;
+    SInt32            active;
     fs_event_watcher *watcher;
     struct selinfo    si;
 } fsevent_handle;
 
+#define FSEH_CLOSING   0x0001
 
 static int
 fseventsf_read(struct fileproc *fp, struct uio *uio,
@@ -969,15 +990,27 @@ fseventsf_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, struct proc *p)
     pid_t pid = 0;
     fsevent_dev_filter_args *devfilt_args=(fsevent_dev_filter_args *)data;
 
+    OSAddAtomic(1, &fseh->active);
+    if (fseh->flags & FSEH_CLOSING) {
+	OSAddAtomic(-1, &fseh->active);
+	return 0;
+    }
+
     switch (cmd) {
 	case FIONBIO:
 	case FIOASYNC:
-	    return 0;
+	    ret = 0;
+	    break;
 
 	case FSEVENTS_DEVICE_FILTER: {
 	    int new_num_devices;
 	    dev_t *devices_to_watch, *tmp=NULL;
 	    
+	    if (fseh->flags & FSEH_CLOSING) {
+		ret = 0;
+		break;
+	    }
+
 	    if (devfilt_args->num_devices > 256) {
 		ret = EINVAL;
 		break;
@@ -1032,6 +1065,7 @@ fseventsf_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, struct proc *p)
 	    break;
     }
 
+    OSAddAtomic(-1, &fseh->active);
     return (ret);
 }
 
@@ -1073,11 +1107,18 @@ static int
 fseventsf_close(struct fileglob *fg, struct proc *p)
 {
     fsevent_handle *fseh = (struct fsevent_handle *)fg->fg_data;
+    fs_event_watcher *watcher;
+    
+    OSBitOrAtomic(FSEH_CLOSING, &fseh->flags);
+    while (OSAddAtomic(0, &fseh->active) > 0) {
+	tsleep((caddr_t)fseh->watcher, PRIBIO, "fsevents-close", 1);
+    }
 
-    remove_watcher(fseh->watcher);
-
-    fg->fg_data = NULL;
+    watcher = fseh->watcher;
     fseh->watcher = NULL;
+    fg->fg_data = NULL;
+
+    remove_watcher(watcher);
     FREE(fseh, M_TEMP);
 
     return 0;
diff --git a/bsd/vfs/vfs_init.c b/bsd/vfs/vfs_init.c
index 78790e025..a8e96370a 100644
--- a/bsd/vfs/vfs_init.c
+++ b/bsd/vfs/vfs_init.c
@@ -288,14 +288,12 @@ vfsinit()
 	struct mount * mp;
 	
 	/* Allocate vnode list lock group attribute and group */
-	vnode_list_lck_grp_attr= lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(vnode_list_lck_grp_attr);
+	vnode_list_lck_grp_attr = lck_grp_attr_alloc_init();
 
 	vnode_list_lck_grp = lck_grp_alloc_init("vnode list",  vnode_list_lck_grp_attr);
 	
 	/* Allocate vnode list lock attribute */
 	vnode_list_lck_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(vnode_list_lck_attr);
 
 	/* Allocate vnode list lock */
 	vnode_list_mtx_lock = lck_mtx_alloc_init(vnode_list_lck_grp, vnode_list_lck_attr);
@@ -305,36 +303,29 @@ vfsinit()
 
 	/* allocate vnode lock group attribute and group */
 	vnode_lck_grp_attr= lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(vnode_lck_grp_attr);
 
 	vnode_lck_grp = lck_grp_alloc_init("vnode",  vnode_lck_grp_attr);
 
 	/* Allocate vnode lock attribute */
 	vnode_lck_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(vnode_lck_attr);
 
 	/* Allocate fs config lock group attribute and group */
 	fsconf_lck_grp_attr= lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(fsconf_lck_grp_attr);
 
 	fsconf_lck_grp = lck_grp_alloc_init("fs conf",  fsconf_lck_grp_attr);
 	
 	/* Allocate fs config lock attribute */
 	fsconf_lck_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(fsconf_lck_attr);
-
 
 	/* Allocate mount point related lock structures  */
 
 	/* Allocate mount list lock group attribute and group */
 	mnt_list_lck_grp_attr= lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(mnt_list_lck_grp_attr);
 
 	mnt_list_lck_grp = lck_grp_alloc_init("mount list",  mnt_list_lck_grp_attr);
 	
 	/* Allocate mount list lock attribute */
 	mnt_list_lck_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(mnt_list_lck_attr);
 
 	/* Allocate mount list lock */
 	mnt_list_mtx_lock = lck_mtx_alloc_init(mnt_list_lck_grp, mnt_list_lck_attr);
@@ -342,13 +333,11 @@ vfsinit()
 
 	/* allocate mount lock group attribute and group */
 	mnt_lck_grp_attr= lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(mnt_lck_grp_attr);
 
 	mnt_lck_grp = lck_grp_alloc_init("mount",  mnt_lck_grp_attr);
 
 	/* Allocate mount lock attribute */
 	mnt_lck_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(mnt_lck_attr);
 
 	/*
 	 * Initialize the "console user" for access purposes:
diff --git a/bsd/vfs/vfs_journal.c b/bsd/vfs/vfs_journal.c
index 495d0079e..6d82743a1 100644
--- a/bsd/vfs/vfs_journal.c
+++ b/bsd/vfs/vfs_journal.c
@@ -203,9 +203,6 @@ journal_init()
 	jnl_lock_attr    = lck_attr_alloc_init();
 	jnl_group_attr   = lck_grp_attr_alloc_init();
 	jnl_mutex_group  = lck_grp_alloc_init("jnl-mutex", jnl_group_attr);
-
-	/* Turn on lock debugging */
-	//lck_attr_setdebug(jnl_lock_attr);
 }
 
 static __inline__ void
diff --git a/bsd/vfs/vfs_lookup.c b/bsd/vfs/vfs_lookup.c
index f0366b9c3..b2d027f19 100644
--- a/bsd/vfs/vfs_lookup.c
+++ b/bsd/vfs/vfs_lookup.c
@@ -938,7 +938,7 @@ kdebug_lookup(dp, cnp)
 	struct vnode *dp;
 	struct componentname *cnp;
 {
-	register unsigned int i, n;
+	register unsigned int i, n, code;
 	register int dbg_namelen;
 	register int save_dbg_namelen;
 	register char *dbg_nameptr;
@@ -982,19 +982,27 @@ kdebug_lookup(dp, cnp)
 	    else
 	        dbg_parms[i++] = 0;
 	}
+	dbg_namelen = save_dbg_namelen - 12;
 
 	/*
-	  In the event that we collect multiple, consecutive pathname
-	  entries, we must mark the start of the path's string.
-	*/
-	KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW,36)) | DBG_FUNC_START,
-		(unsigned int)dp, dbg_parms[0], dbg_parms[1], dbg_parms[2], 0);
-
-	for (dbg_namelen = save_dbg_namelen-12, i=3;
-	     dbg_namelen > 0;
-	     dbg_namelen -=(4 * sizeof(long)), i+= 4)
-	  {
-	    KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW,36)) | DBG_FUNC_NONE,
-				  dbg_parms[i], dbg_parms[i+1], dbg_parms[i+2], dbg_parms[i+3], 0);
-	  }
+	 * In the event that we collect multiple, consecutive pathname
+	 * entries, we must mark the start of the path's string and the end
+	 */
+	code = (FSDBG_CODE(DBG_FSRW,36)) | DBG_FUNC_START;
+
+	if (dbg_namelen <= 0)
+	        code |= DBG_FUNC_END;
+
+	KERNEL_DEBUG_CONSTANT(code, (unsigned int)dp, dbg_parms[0], dbg_parms[1], dbg_parms[2], 0);
+
+	code &= ~DBG_FUNC_START;
+
+	for (i = 3; dbg_namelen > 0; i += 4) {
+
+	        dbg_namelen -= (4 * sizeof(long));
+	        if (dbg_namelen <= 0)
+		        code |= DBG_FUNC_END;
+
+	        KERNEL_DEBUG_CONSTANT(code, dbg_parms[i], dbg_parms[i+1], dbg_parms[i+2], dbg_parms[i+3], 0);
+	}
 }
diff --git a/bsd/vfs/vfs_quota.c b/bsd/vfs/vfs_quota.c
index 41e2fe355..4e95edaa0 100644
--- a/bsd/vfs/vfs_quota.c
+++ b/bsd/vfs/vfs_quota.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2002-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -75,6 +75,8 @@
 #include <sys/quota.h>
 #include <sys/uio_internal.h>
 
+#include <libkern/OSByteOrder.h>
+
 
 /* vars for quota file lock */
 lck_grp_t	* qf_lck_grp;
@@ -88,7 +90,7 @@ lck_attr_t	* quota_list_lck_attr;
 lck_mtx_t	* quota_list_mtx_lock;
 
 /* Routines to lock and unlock the quota global data */
-static void dq_list_lock(void);
+static int dq_list_lock(void);
 static void dq_list_unlock(void);
 
 static void dq_lock_internal(struct dquot *dq);
@@ -142,14 +144,12 @@ dqinit()
 	 * Allocate quota list lock group attribute and group
 	 */
 	quota_list_lck_grp_attr= lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(quota_list_lck_grp_attr);
 	quota_list_lck_grp = lck_grp_alloc_init("quota list",  quota_list_lck_grp_attr);
 	
 	/*
 	 * Allocate qouta list lock attribute
 	 */
 	quota_list_lck_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(quota_list_lck_attr);
 
 	/*
 	 * Allocate quota list lock
@@ -161,22 +161,32 @@ dqinit()
 	 * allocate quota file lock group attribute and group
 	 */
 	qf_lck_grp_attr= lck_grp_attr_alloc_init();
-	lck_grp_attr_setstat(qf_lck_grp_attr);
 	qf_lck_grp = lck_grp_alloc_init("quota file", qf_lck_grp_attr);
 
 	/*
 	 * Allocate quota file lock attribute
 	 */
 	qf_lck_attr = lck_attr_alloc_init();
-	//lck_attr_setdebug(qf_lck_attr);
 }
 
 
+static volatile int dq_list_lock_cnt = 0;
 
-void
+static int
 dq_list_lock(void)
 {
 	lck_mtx_lock(quota_list_mtx_lock);
+	return ++dq_list_lock_cnt;
+}
+
+static int
+dq_list_lock_changed(int oldval) {
+	return (dq_list_lock_cnt != oldval);
+}
+
+static int
+dq_list_lock_val(void) {
+	return dq_list_lock_cnt;
 }
 
 void
@@ -408,27 +418,27 @@ dqfileopen(qfp, type)
 		goto out;
 	}
 	/* Sanity check the quota file header. */
-	if ((header.dqh_magic != quotamagic[type]) ||
-	    (header.dqh_version > QF_VERSION) ||
-	    (!powerof2(header.dqh_maxentries)) ||
-	    (header.dqh_maxentries > (file_size / sizeof(struct dqblk)))) {
+	if ((OSSwapBigToHostInt32(header.dqh_magic) != quotamagic[type]) ||
+	    (OSSwapBigToHostInt32(header.dqh_version) > QF_VERSION) ||
+	    (!powerof2(OSSwapBigToHostInt32(header.dqh_maxentries))) ||
+	    (OSSwapBigToHostInt32(header.dqh_maxentries) > (file_size / sizeof(struct dqblk)))) {
 		error = EINVAL;
 		goto out;
 	}
 	/* Set up the time limits for this quota. */
-	if (header.dqh_btime > 0)
-		qfp->qf_btime = header.dqh_btime;
+	if (header.dqh_btime != 0)
+		qfp->qf_btime = OSSwapBigToHostInt32(header.dqh_btime);
 	else
 		qfp->qf_btime = MAX_DQ_TIME;
-	if (header.dqh_itime > 0)
-		qfp->qf_itime = header.dqh_itime;
+	if (header.dqh_itime != 0)
+		qfp->qf_itime = OSSwapBigToHostInt32(header.dqh_itime);
 	else
 		qfp->qf_itime = MAX_IQ_TIME;
 
 	/* Calculate the hash table constants. */
-	qfp->qf_maxentries = header.dqh_maxentries;
-	qfp->qf_entrycnt = header.dqh_entrycnt;
-	qfp->qf_shift = dqhashshift(header.dqh_maxentries);
+	qfp->qf_maxentries = OSSwapBigToHostInt32(header.dqh_maxentries);
+	qfp->qf_entrycnt = OSSwapBigToHostInt32(header.dqh_entrycnt);
+	qfp->qf_shift = dqhashshift(qfp->qf_maxentries);
 out:
 	return (error);
 }
@@ -452,7 +462,7 @@ dqfileclose(struct quotafile *qfp, __unused int type)
 	context.vc_ucred = qfp->qf_cred;
 	
 	if (VNOP_READ(qfp->qf_vp, auio, 0, &context) == 0) {
-		header.dqh_entrycnt = qfp->qf_entrycnt;
+		header.dqh_entrycnt = OSSwapHostToBigInt32(qfp->qf_entrycnt);
 		uio_reset(auio, 0, UIO_SYSSPACE, UIO_WRITE);
 		uio_addiov(auio, CAST_USER_ADDR_T(&header), sizeof (header));
 		(void) VNOP_WRITE(qfp->qf_vp, auio, 0, &context);
@@ -477,6 +487,7 @@ dqget(id, qfp, type, dqp)
 	struct dqhash *dqh;
 	struct vnode *dqvp;
 	int error = 0;
+	int listlockval = 0;
 
 	if ( id == 0 || qfp->qf_vp == NULLVP ) {
 		*dqp = NODQUOT;
@@ -500,6 +511,8 @@ dqget(id, qfp, type, dqp)
 	dqh = DQHASH(dqvp, id);
 
 relookup:
+	listlockval = dq_list_lock_val();
+
 	/*
 	 * Check the cache first.
 	 */
@@ -509,6 +522,11 @@ relookup:
 			continue;
 
 		dq_lock_internal(dq);
+		if (dq_list_lock_changed(listlockval)) {
+			dq_unlock_internal(dq);
+			goto relookup;
+		}
+
 		/*
 		 * dq_lock_internal may drop the quota_list_lock to msleep, so
 		 * we need to re-evaluate the identity of this dq
@@ -527,6 +545,13 @@ relookup:
 				TAILQ_REMOVE(&dqdirtylist, dq, dq_freelist);
 			else
 				TAILQ_REMOVE(&dqfreelist, dq, dq_freelist);
+		} else if (dq->dq_cnt == 0) {
+			/* We've overflowed */
+			--dq->dq_cnt;
+			dq_unlock_internal(dq);
+			dq_list_unlock();
+			*dqp = NODQUOT;
+			return (EINVAL);
 		}
 		dq_unlock_internal(dq);
 
@@ -579,7 +604,7 @@ relookup:
 			ndq = (struct dquot *)_MALLOC(sizeof *dq, M_DQUOT, M_WAITOK);
 			bzero((char *)ndq, sizeof *dq);
 
-		        dq_list_lock();
+		        listlockval = dq_list_lock();
 			/*
 			 * need to look for the entry again in the cache
 			 * since we dropped the quota list lock and
@@ -617,7 +642,7 @@ relookup:
 
 		dq_lock_internal(dq);
 
-		if (dq->dq_cnt || (dq->dq_flags & DQ_MOD)) {
+		if (dq_list_lock_changed(listlockval) || dq->dq_cnt || (dq->dq_flags & DQ_MOD)) {
 		        /*
 			 * we lost the race while we weren't holding
 			 * the quota list lock... dq_lock_internal
@@ -660,6 +685,10 @@ relookup:
 	 * one else can be trying to use this dq
 	 */
 	dq_lock_internal(dq);
+	if (dq_list_lock_changed(listlockval)) {
+		dq_unlock_internal(dq);
+		goto relookup;
+	}
 
 	/*
 	 * Initialize the contents of the dquot structure.
@@ -790,7 +819,7 @@ dqlookup(qfp, id, dqb, index)
 		 */
 		if (dqb->dqb_id == 0) {
 			bzero(dqb, sizeof(struct dqblk));
-			dqb->dqb_id = id;
+			dqb->dqb_id = OSSwapHostToBigInt32(id);
 			/*
 			 * Write back to reserve entry for this id
 			 */
@@ -801,11 +830,22 @@ dqlookup(qfp, id, dqb, index)
 				error = EIO;
 			if (error == 0)
 				++qfp->qf_entrycnt;
+			dqb->dqb_id = id;
 			break;
 		}
 		/* An id match means an entry was found. */
-		if (dqb->dqb_id == id)
+		if (OSSwapBigToHostInt32(dqb->dqb_id) == id) {
+			dqb->dqb_bhardlimit = OSSwapBigToHostInt64(dqb->dqb_bhardlimit);
+			dqb->dqb_bsoftlimit = OSSwapBigToHostInt64(dqb->dqb_bsoftlimit);
+			dqb->dqb_curbytes   = OSSwapBigToHostInt64(dqb->dqb_curbytes);
+			dqb->dqb_ihardlimit = OSSwapBigToHostInt32(dqb->dqb_ihardlimit);
+			dqb->dqb_isoftlimit = OSSwapBigToHostInt32(dqb->dqb_isoftlimit);
+			dqb->dqb_curinodes  = OSSwapBigToHostInt32(dqb->dqb_curinodes);
+			dqb->dqb_btime      = OSSwapBigToHostInt32(dqb->dqb_btime);
+			dqb->dqb_itime      = OSSwapBigToHostInt32(dqb->dqb_itime);
+			dqb->dqb_id         = OSSwapBigToHostInt32(dqb->dqb_id);
 			break;
+		}
 	}
 	qf_unlock(qfp);
 
@@ -877,14 +917,21 @@ dqsync_orphans(qfp)
 	struct quotafile *qfp;
 {
 	struct dquot *dq;
-	
+	int listlockval = 0;
+
 	dq_list_lock();
   loop:
+	listlockval = dq_list_lock_val();
+
 	TAILQ_FOREACH(dq, &dqdirtylist, dq_freelist) {
 		if (dq->dq_qfile != qfp)
 		        continue;
 
 		dq_lock_internal(dq);
+		if (dq_list_lock_changed(listlockval)) {
+			dq_unlock_internal(dq);
+			goto loop;
+		}
 
 		if (dq->dq_qfile != qfp) {
 		        /*
@@ -954,6 +1001,7 @@ dqsync_locked(struct dquot *dq)
 	struct proc *p = current_proc();		/* XXX */
 	struct vfs_context context;
 	struct vnode *dqvp;
+	struct dqblk dqb, *dqblkp;
 	uio_t auio;
 	int error;
 	char uio_buf[ UIO_SIZEOF(1) ];
@@ -969,11 +1017,26 @@ dqsync_locked(struct dquot *dq)
 
 	auio = uio_createwithbuffer(1, dqoffset(dq->dq_index), UIO_SYSSPACE, 
 								  UIO_WRITE, &uio_buf[0], sizeof(uio_buf));
-	uio_addiov(auio, CAST_USER_ADDR_T(&dq->dq_dqb), sizeof (struct dqblk));
+	uio_addiov(auio, CAST_USER_ADDR_T(&dqb), sizeof (struct dqblk));
 
 	context.vc_proc = p;
 	context.vc_ucred = dq->dq_qfile->qf_cred;
 
+	dqblkp = &dq->dq_dqb;
+	dqb.dqb_bhardlimit = OSSwapHostToBigInt64(dqblkp->dqb_bhardlimit);
+	dqb.dqb_bsoftlimit = OSSwapHostToBigInt64(dqblkp->dqb_bsoftlimit);
+	dqb.dqb_curbytes   = OSSwapHostToBigInt64(dqblkp->dqb_curbytes);
+	dqb.dqb_ihardlimit = OSSwapHostToBigInt32(dqblkp->dqb_ihardlimit);
+	dqb.dqb_isoftlimit = OSSwapHostToBigInt32(dqblkp->dqb_isoftlimit);
+	dqb.dqb_curinodes  = OSSwapHostToBigInt32(dqblkp->dqb_curinodes);
+	dqb.dqb_btime      = OSSwapHostToBigInt32(dqblkp->dqb_btime);
+	dqb.dqb_itime      = OSSwapHostToBigInt32(dqblkp->dqb_itime);
+	dqb.dqb_id         = OSSwapHostToBigInt32(dqblkp->dqb_id);
+	dqb.dqb_spare[0]   = 0;
+	dqb.dqb_spare[1]   = 0;
+	dqb.dqb_spare[2]   = 0;
+	dqb.dqb_spare[3]   = 0;
+
 	error = VNOP_WRITE(dqvp, auio, 0, &context);
 	if (uio_resid(auio) && error == 0)
 		error = EIO;
diff --git a/bsd/vfs/vfs_subr.c b/bsd/vfs/vfs_subr.c
index ce35a0fdc..c5dbcd1e3 100644
--- a/bsd/vfs/vfs_subr.c
+++ b/bsd/vfs/vfs_subr.c
@@ -1711,12 +1711,12 @@ vclean(vnode_t vp, int flags, proc_t p)
 	insmntque(vp, (struct mount *)0);
 
 	ucred = vp->v_cred;
-	vp->v_cred = NULL;
+	vp->v_cred = NOCRED;
 
 	vnode_unlock(vp);
 
-	if (ucred)
-	        kauth_cred_rele(ucred);
+	if (IS_VALID_CRED(ucred))
+	        kauth_cred_unref(&ucred);
 
 	OSAddAtomic(1, &num_recycledvnodes);
 	/*
@@ -2394,6 +2394,7 @@ vfs_unmountall()
 	struct mount *mp;
 	struct proc *p = current_proc();
 	int error;
+	int skip_listremove;
 
 	/*
 	 * Since this only runs when rebooting, it is not interlocked.
@@ -2402,11 +2403,15 @@ vfs_unmountall()
 	while(!TAILQ_EMPTY(&mountlist)) {
 		mp = TAILQ_LAST(&mountlist, mntlist);
 		mount_list_unlock();
-		error = dounmount(mp, MNT_FORCE, p);
+		skip_listremove = 0;
+		error = dounmount(mp, MNT_FORCE, &skip_listremove, p);
 		if (error) {
 			mount_list_lock();
-			TAILQ_REMOVE(&mountlist, mp, mnt_list);
-			printf("unmount of %s failed (", mp->mnt_vfsstat.f_mntonname);
+			if (skip_listremove == 0) {
+				TAILQ_REMOVE(&mountlist, mp, mnt_list);
+				printf("unmount of %s failed (", mp->mnt_vfsstat.f_mntonname);
+			}
+				
 			if (error == EBUSY)
 				printf("BUSY)\n");
 			else
@@ -2965,7 +2970,7 @@ sysctl_vfs_noremotehang SYSCTL_HANDLER_ARGS
 		return (error);
 	}
 
-	/* XXX req->p->p_ucred -> kauth_cred_get() ??? */
+	/* XXX req->p->p_ucred -> kauth_cred_get() - current unsafe ??? */
 	/* cansignal offers us enough security. */
 	if (p != req->p && suser(req->p->p_ucred, &req->p->p_acflag) != 0)
 		return (EPERM);
@@ -3996,6 +4001,47 @@ vnode_setsize(vnode_t vp, off_t size, int ioflag, vfs_context_t ctx)
 	return(vnode_setattr(vp, &va, ctx));
 }
 
+/*
+ * Create a filesystem object of arbitrary type with arbitrary attributes in
+ * the spevied directory with the specified name.
+ *
+ * Parameters:	dvp			Pointer to the vnode of the directory
+ *					in which to create the object.
+ *		vpp			Pointer to the area into which to
+ *					return the vnode of the created object.
+ *		cnp			Component name pointer from the namei
+ *					data structure, containing the name to
+ *					use for the create object.
+ *		vap			Pointer to the vnode_attr structure
+ *					describing the object to be created,
+ *					including the type of object.
+ *		flags			VN_* flags controlling ACL inheritance
+ *					and whether or not authorization is to
+ *					be required for the operation.
+ *		
+ * Returns:	0			Success
+ *		!0			errno value
+ *
+ * Implicit:	*vpp			Contains the vnode of the object that
+ *					was created, if successful.
+ *		*cnp			May be modified by the underlying VFS.
+ *		*vap			May be modified by the underlying VFS.
+ *					modified by either ACL inheritance or
+ *		
+ *		
+ *					be modified, even if the operation is
+ *					
+ *
+ * Notes:	The kauth_filesec_t in 'vap', if any, is in host byte order.
+ *
+ *		Modification of '*cnp' and '*vap' by the underlying VFS is
+ *		strongly discouraged.
+ *
+ * XXX:		This function is a 'vn_*' function; it belongs in vfs_vnops.c
+ *
+ * XXX:		We should enummerate the possible errno values here, and where
+ *		in the code they originated.
+ */
 errno_t
 vn_create(vnode_t dvp, vnode_t *vpp, struct componentname *cnp, struct vnode_attr *vap, int flags, vfs_context_t ctx)
 {
diff --git a/bsd/vfs/vfs_support.c b/bsd/vfs/vfs_support.c
index 2fb73f444..d73ea4d8e 100644
--- a/bsd/vfs/vfs_support.c
+++ b/bsd/vfs/vfs_support.c
@@ -819,27 +819,6 @@ err_pageout(struct vnop_pageout_args *ap)
 }
 
 
-struct vnop_devblocksize_args /* {
-	struct vnode *a_vp;
-	register_t *a_retval;
-} */;
-
-int
-nop_devblocksize(struct vnop_devblocksize_args *ap)
-{
-	/* XXX default value because the call sites do not check error */
-	*ap->a_retval = 512;
-	return (0);
-}
-
-int
-err_devblocksize(struct vnop_devblocksize_args *ap)
-{
-	(void)nop_devblocksize(ap);
-	return (ENOTSUP);
-}
-
-
 struct vnop_searchfs /* {
 	struct vnode *a_vp;
 	void *a_searchparams1;
diff --git a/bsd/vfs/vfs_support.h b/bsd/vfs/vfs_support.h
index 4abfd0b3f..e50630cb6 100644
--- a/bsd/vfs/vfs_support.h
+++ b/bsd/vfs/vfs_support.h
@@ -46,7 +46,6 @@
 #include <sys/cdefs.h>
 #include <sys/param.h>
 #include <sys/systm.h>
-#include <sys/resourcevar.h>
 #include <sys/kernel.h>
 #include <sys/file.h>
 #include <sys/stat.h>
@@ -57,7 +56,6 @@
 #include <sys/vnode_if.h>
 #include <sys/malloc.h>
 #include <sys/dirent.h>
-#include <vm/vm_pageout.h>
 
 __BEGIN_DECLS
 extern int nop_create(struct vnop_create_args *ap);
@@ -170,9 +168,6 @@ extern int err_pagein(struct vnop_pagein_args *ap);
 extern int nop_pageout(struct vnop_pageout_args *ap);
 extern int err_pageout(struct vnop_pageout_args *ap);
 
-extern int nop_devblocksize(struct vnop_devblocksize_args *ap);
-extern int err_devblocksize(struct vnop_devblocksize_args *ap);
-
 extern int nop_searchfs(struct vnop_searchfs_args *ap);
 extern int err_searchfs(struct vnop_searchfs_args *ap);
 
diff --git a/bsd/vfs/vfs_syscalls.c b/bsd/vfs/vfs_syscalls.c
index 340c6ab9d..186ac2c99 100644
--- a/bsd/vfs/vfs_syscalls.c
+++ b/bsd/vfs/vfs_syscalls.c
@@ -102,7 +102,6 @@
 
 #include <vm/vm_pageout.h>
 
-#include <architecture/byte_order.h>
 #include <libkern/OSAtomic.h>
 
 
@@ -734,16 +733,17 @@ safedounmount(mp, flags, p)
 	if (mp->mnt_flag & MNT_ROOTFS)
 		return (EBUSY); /* the root is always busy */
 
-	return (dounmount(mp, flags, p));
+	return (dounmount(mp, flags, NULL, p));
 }
 
 /*
  * Do the actual file system unmount.
  */
 int
-dounmount(mp, flags, p)
+dounmount(mp, flags, skiplistrmp, p)
 	register struct mount *mp;
 	int flags;
+	int * skiplistrmp;
 	struct proc *p;
 {
 	struct vnode *coveredvp = (vnode_t)0;
@@ -771,6 +771,8 @@ dounmount(mp, flags, p)
 		 * The prior unmount attempt has probably succeeded.
 		 * Do not dereference mp here - returning EBUSY is safest.
 		 */
+		if (skiplistrmp != NULL)
+			*skiplistrmp = 1;
 		return (EBUSY);
 	}
 	mp->mnt_kern_flag |= MNTK_UNMOUNT;
@@ -1528,6 +1530,30 @@ bad:
 
 }
 
+/*
+ * An open system call using an extended argument list compared to the regular
+ * system call 'open'.
+ *
+ * Parameters:	p			Process requesting the open
+ *		uap			User argument descriptor (see below)
+ *		retval			Pointer to an area to receive the
+ *					return calue from the system call
+ *
+ * Indirect:	uap->path		Path to open (same as 'open')
+ *		uap->flags		Flags to open (same as 'open'
+ *		uap->uid		UID to set, if creating
+ *		uap->gid		GID to set, if creating
+ *		uap->mode		File mode, if creating (same as 'open')
+ *		uap->xsecurity		ACL to set, if creating
+ *
+ * Returns:	0			Success
+ *		!0			errno value
+ *
+ * Notes:	The kauth_filesec_t in 'va', if any, is in host byte order.
+ *
+ * XXX:		We should enummerate the possible errno values here, and where
+ *		in the code they originated.
+ */
 int
 open_extended(struct proc *p, struct open_extended_args *uap, register_t *retval)
 {
@@ -1729,6 +1755,29 @@ out:
 	return error;
 }
 
+
+/*
+ * A mkfifo system call using an extended argument list compared to the regular
+ * system call 'mkfifo'.
+ *
+ * Parameters:	p			Process requesting the open
+ *		uap			User argument descriptor (see below)
+ *		retval			(Ignored)
+ *
+ * Indirect:	uap->path		Path to fifo (same as 'mkfifo')
+ *		uap->uid		UID to set
+ *		uap->gid		GID to set
+ *		uap->mode		File mode to set (same as 'mkfifo')
+ *		uap->xsecurity		ACL to set, if creating
+ *
+ * Returns:	0			Success
+ *		!0			errno value
+ *
+ * Notes:	The kauth_filesec_t in 'va', if any, is in host byte order.
+ *
+ * XXX:		We should enummerate the possible errno values here, and where
+ *		in the code they originated.
+ */
 int
 mkfifo_extended(struct proc *p, struct mkfifo_extended_args *uap, __unused register_t *retval)
 {
@@ -2440,8 +2489,8 @@ out:
 		vnode_put(vp);
 	if (dvp)
 		vnode_put(dvp);
-	if (context.vc_ucred)
- 		kauth_cred_rele(context.vc_ucred);
+	if (IS_VALID_CRED(context.vc_ucred))
+ 		kauth_cred_unref(&context.vc_ucred);
 	return(error);
 }
 
@@ -2479,7 +2528,7 @@ access(__unused struct proc *p, register struct access_args *uap, __unused regis
   	nameidone(&nd);
   
 out:
- 	kauth_cred_rele(context.vc_ucred);
+ 	kauth_cred_unref(&context.vc_ucred);
  	return(error);
 }
 
@@ -2821,6 +2870,28 @@ chmod1(vfs_context_t ctx, user_addr_t path, struct vnode_attr *vap)
 	return(error);
 }
 
+/*
+ * A chmod system call using an extended argument list compared to the regular
+ * system call 'mkfifo'.
+ *
+ * Parameters:	p			Process requesting the open
+ *		uap			User argument descriptor (see below)
+ *		retval			(ignored)
+ *
+ * Indirect:	uap->path		Path to object (same as 'chmod')
+ *		uap->uid		UID to set
+ *		uap->gid		GID to set
+ *		uap->mode		File mode to set (same as 'chmod')
+ *		uap->xsecurity		ACL to set (or delete)
+ *
+ * Returns:	0			Success
+ *		!0			errno value
+ *
+ * Notes:	The kauth_filesec_t in 'va', if any, is in host byte order.
+ *
+ * XXX:		We should enummerate the possible errno values here, and where
+ *		in the code they originated.
+ */
 int
 chmod_extended(struct proc *p, struct chmod_extended_args *uap, __unused register_t *retval)
 {
@@ -4603,10 +4674,11 @@ checkuseraccess (struct proc *p, register struct checkuseraccess_args *uap, __un
 	register struct vnode *vp;
 	int error;
 	struct nameidata nd;
-	struct ucred cred;	/* XXX ILLEGAL */
+	struct ucred template_cred;
 	int flags;		/*what will actually get passed to access*/
 	u_long nameiflags;
 	struct vfs_context context;
+	kauth_cred_t my_cred;
 
 	/* Make sure that the number of groups is correct before we do anything */
 
@@ -4618,16 +4690,18 @@ checkuseraccess (struct proc *p, register struct checkuseraccess_args *uap, __un
 	if ((error = suser(kauth_cred_get(), &p->p_acflag)))
 		return(error);
 
-	/* Fill in the credential structure */
-
-	cred.cr_ref = 0;
-	cred.cr_uid = uap->userid;
-	cred.cr_ngroups = uap->ngroups;
-	if ((error = copyin(CAST_USER_ADDR_T(uap->groups), (caddr_t) &(cred.cr_groups), (sizeof(gid_t))*uap->ngroups)))
+	/*
+	 * Fill in the template credential structure; we use a template because
+	 * lookup can attempt to take a persistent reference.
+	 */
+	template_cred.cr_uid = uap->userid;
+	template_cred.cr_ngroups = uap->ngroups;
+	if ((error = copyin(CAST_USER_ADDR_T(uap->groups), (caddr_t) &(template_cred.cr_groups), (sizeof(gid_t))*uap->ngroups)))
 		return (error);
 
+	my_cred = kauth_cred_create(&template_cred);
 	context.vc_proc = p;
-	context.vc_ucred = &cred;
+	context.vc_ucred = my_cred;
 
 	/* Get our hands on the file */
 	nameiflags = 0;
@@ -4635,8 +4709,10 @@ checkuseraccess (struct proc *p, register struct checkuseraccess_args *uap, __un
 	NDINIT(&nd, LOOKUP, nameiflags | AUDITVNPATH1, 
 		UIO_USERSPACE, CAST_USER_ADDR_T(uap->path), &context);
 
-	if ((error = namei(&nd)))
+	if ((error = namei(&nd))) {
+		kauth_cred_unref(&context.vc_ucred);
        		 return (error);
+	}
 	nameidone(&nd);
 	vp = nd.ni_vp;
 	
@@ -4656,11 +4732,8 @@ checkuseraccess (struct proc *p, register struct checkuseraccess_args *uap, __un
 		    	
 	vnode_put(vp);
 
-	if (error) 
- 		return (error);
-	
-	return (0); 
-
+	kauth_cred_unref(&context.vc_ucred);
+	return (error);
 } /* end of checkuseraccess system call */
 
 /************************************************/
@@ -4707,7 +4780,8 @@ searchfs (struct proc *p, register struct searchfs_args *uap, __unused register_
         searchblock.returnbuffer = CAST_USER_ADDR_T(tmp_searchblock.returnbuffer);
         searchblock.returnbuffersize = tmp_searchblock.returnbuffersize;
         searchblock.maxmatches = tmp_searchblock.maxmatches;
-        searchblock.timelimit = tmp_searchblock.timelimit;
+        searchblock.timelimit.tv_sec = tmp_searchblock.timelimit.tv_sec;
+        searchblock.timelimit.tv_usec = tmp_searchblock.timelimit.tv_usec;
         searchblock.searchparams1 = CAST_USER_ADDR_T(tmp_searchblock.searchparams1);
         searchblock.sizeofsearchparams1 = tmp_searchblock.sizeofsearchparams1;
         searchblock.searchparams2 = CAST_USER_ADDR_T(tmp_searchblock.searchparams2);
@@ -5439,6 +5513,8 @@ munge_statfs(struct mount *mp, struct vfsstatfs *sfsp,
  */
 void munge_stat(struct stat *sbp, struct user_stat *usbp)
 {
+        bzero(usbp, sizeof(struct user_stat));
+
 	usbp->st_dev = sbp->st_dev;
 	usbp->st_ino = sbp->st_ino;
 	usbp->st_mode = sbp->st_mode;
diff --git a/bsd/vfs/vfs_utfconv.c b/bsd/vfs/vfs_utfconv.c
index 6651c4f38..7a363eb49 100644
--- a/bsd/vfs/vfs_utfconv.c
+++ b/bsd/vfs/vfs_utfconv.c
@@ -33,7 +33,7 @@
 #include <sys/param.h>
 #include <sys/utfconv.h>
 #include <sys/errno.h>
-#include <architecture/byte_order.h>
+#include <libkern/OSByteOrder.h>
 
 /*
  * UTF-8 (Unicode Transformation Format)
@@ -181,7 +181,7 @@ utf8_encodelen(const u_int16_t * ucsp, size_t ucslen, u_int16_t altslash,
 		ucs_ch = *ucsp++;
 
 		if (swapbytes)
-			ucs_ch = NXSwapShort(ucs_ch);
+			ucs_ch = OSSwapInt16(ucs_ch);
 		if (ucs_ch == '/')
 			ucs_ch = altslash ? altslash : '_';
 		else if (ucs_ch == '\0')
@@ -238,7 +238,7 @@ utf8_encodestr(const u_int16_t * ucsp, size_t ucslen, u_int8_t * utf8p,
 			--extra;
 			ucs_ch = *chp++;
 		} else {
-			ucs_ch = swapbytes ? NXSwapShort(*ucsp++) : *ucsp++;
+			ucs_ch = swapbytes ? OSSwapInt16(*ucsp++) : *ucsp++;
 
 			if (decompose && unicode_decomposeable(ucs_ch)) {
 				extra = unicode_decompose(ucs_ch, sequence) - 1;
@@ -282,7 +282,7 @@ utf8_encodestr(const u_int16_t * ucsp, size_t ucslen, u_int8_t * utf8p,
 				u_int16_t ch2;
 				u_int32_t pair;
 
-				ch2 = swapbytes ? NXSwapShort(*ucsp) : *ucsp;
+				ch2 = swapbytes ? OSSwapInt16(*ucsp) : *ucsp;
 				if (ch2 >= SP_LOW_FIRST && ch2 <= SP_LOW_LAST) {
 					pair = ((ucs_ch - SP_HIGH_FIRST) << SP_HALF_SHIFT)
 						+ (ch2 - SP_LOW_FIRST) + SP_HALF_BASE;
@@ -420,13 +420,13 @@ utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp,
 				ucs_ch = (ch >> SP_HALF_SHIFT) + SP_HIGH_FIRST;
 				if (ucs_ch < SP_HIGH_FIRST || ucs_ch > SP_HIGH_LAST)
 					goto invalid;
-				*ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
+				*ucsp++ = swapbytes ? OSSwapInt16(ucs_ch) : ucs_ch;
 				if (ucsp >= bufend)
 					goto toolong;
 				ucs_ch = (ch & SP_HALF_MASK) + SP_LOW_FIRST;
 				if (ucs_ch < SP_LOW_FIRST || ucs_ch > SP_LOW_LAST)
 					goto invalid;
-				*ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
+				*ucsp++ = swapbytes ? OSSwapInt16(ucs_ch) : ucs_ch;
 			        continue;
 			default:
 				goto invalid;
@@ -440,7 +440,7 @@ utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp,
 
 					for (i = 0; i < count; ++i) {
 						ucs_ch = sequence[i];
-						*ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
+						*ucsp++ = swapbytes ? OSSwapInt16(ucs_ch) : ucs_ch;
 						if (ucsp >= bufend)
 							goto toolong;
 					}
@@ -451,7 +451,7 @@ utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp,
 				u_int16_t composite, base;
 
 				if (unicode_combinable(ucs_ch)) {
-					base = swapbytes ? NXSwapShort(*(ucsp - 1)) : *(ucsp - 1);
+					base = swapbytes ? OSSwapInt16(*(ucsp - 1)) : *(ucsp - 1);
 					composite = unicode_combine(base, ucs_ch);
 					if (composite) {
 						--ucsp;
@@ -476,7 +476,7 @@ utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp,
 			}
 			combcharcnt = 0;  /* start over */
 		}
-		*ucsp++ = swapbytes ? NXSwapShort(ucs_ch) : ucs_ch;
+		*ucsp++ = swapbytes ? OSSwapInt16(ucs_ch) : ucs_ch;
 	}
 	/*
 	 * Make a previous combining sequence canonical
diff --git a/bsd/vfs/vfs_vnops.c b/bsd/vfs/vfs_vnops.c
index f0b5959e2..15ef33cec 100644
--- a/bsd/vfs/vfs_vnops.c
+++ b/bsd/vfs/vfs_vnops.c
@@ -88,6 +88,7 @@
 #include <sys/xattr.h>
 #include <sys/ubc_internal.h>
 #include <sys/uio_internal.h>
+#include <sys/resourcevar.h>
 
 #include <vm/vm_kern.h>
 #include <vm/vm_map.h>
@@ -134,6 +135,51 @@ vn_open_modflags(struct nameidata *ndp, int *fmodep, int cmode)
 	return(vn_open_auth(ndp, fmodep, &va));
 }
 
+/*
+ * Open a file with authorization, updating the contents of the structures
+ * pointed to by ndp, fmodep, and vap as necessary to perform the requested
+ * operation.  This function is used for both opens of existing files, and
+ * creation of new files.
+ *
+ * Parameters:	ndp			The nami data pointer describing the
+ *					file
+ *		fmodep			A pointer to an int containg the mode
+ *					information to be used for the open
+ *		vap			A pointer to the vnode attribute
+ *					descriptor to be used for the open
+ *
+ * Indirect:	*			Contents of the data structures pointed
+ *					to by the parameters are modified as
+ *					necessary to the requested operation.
+ *
+ * Returns:	0			Success
+ *		!0			errno value
+ *
+ * Notes:	The kauth_filesec_t in 'vap', if any, is in host byte order.
+ *
+ *		The contents of '*ndp' will be modified, based on the other
+ *		arguments to this function, and to return file and directory
+ *		data necessary to satisfy the requested operation.
+ *
+ *		If the file does not exist and we are creating it, then the
+ *		O_TRUNC flag will be cleared in '*fmodep' to indicate to the
+ *		caller that the file was not truncated.
+ *
+ *		If the file exists and the O_EXCL flag was not specified, then
+ *		the O_CREAT flag will be cleared in '*fmodep' to indicate to
+ *		the caller that the existing file was merely opened rather
+ *		than created.
+ *
+ *		The contents of '*vap' will be modified as necessary to
+ *		complete the operation, including setting of supported
+ *		attribute, clearing of fields containing unsupported attributes
+ *		in the request, if the request proceeds without them, etc..
+ *
+ * XXX:		This function is too complicated in actings on its arguments
+ *
+ * XXX:		We should enummerate the possible errno values here, and where
+ *		in the code they originated.
+ */
 int
 vn_open_auth(struct nameidata *ndp, int *fmodep, struct vnode_attr *vap)
 {
@@ -149,6 +195,10 @@ again:
 	dvp = NULL;
 	fmode = *fmodep;
 	if (fmode & O_CREAT) {
+	        if ( (fmode & O_DIRECTORY) ) {
+		        error = EINVAL;
+			goto out;
+		}
 		ndp->ni_cnd.cn_nameiop = CREATE;
 		ndp->ni_cnd.cn_flags = LOCKPARENT | LOCKLEAF | AUDITVNPATH1;
 
@@ -237,6 +287,11 @@ badcreate:
 		vp = ndp->ni_vp;
 		nameidone(ndp);
 		ndp->ni_dvp = NULL;
+
+		if ( (fmode & O_DIRECTORY) && vp->v_type != VDIR ) {
+		        error = ENOTDIR;
+			goto bad;
+		}
 	}
 	if (vp->v_type == VSOCK && vp->v_tag != VT_FDESC) {
 		error = EOPNOTSUPP;	/* Operation not supported on socket */
@@ -791,7 +846,7 @@ vn_ioctl(fp, com, data, p)
 		return(error);
 	}
 	context.vc_proc = p;
-	context.vc_ucred = p->p_ucred;	/* XXX kauth_cred_get() ??? */
+	context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? */
 
 	switch (vp->v_type) {
 
@@ -858,6 +913,7 @@ vn_ioctl(fp, com, data, p)
 	}
 out:
 	(void)vnode_put(vp);
+	kauth_cred_unref(&context.vc_ucred);
 	return(error);
 }
 
@@ -949,13 +1005,14 @@ vn_kqfilt_add(fp, kn, p)
 	
 	if ( (error = vnode_getwithref(vp)) == 0 ) {
 		context.vc_proc = p;
-		context.vc_ucred = p->p_ucred;	/* XXX kauth_cred_get() ??? */
+		context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? */
 
 	        funnel_state = thread_funnel_set(kernel_flock, TRUE);
 		error = VNOP_KQFILT_ADD(vp, kn, &context);
 		thread_funnel_set(kernel_flock, funnel_state);
 
 		(void)vnode_put(vp);
+		kauth_cred_unref(&context.vc_ucred);
 	}
 	return (error);
 }
@@ -974,13 +1031,14 @@ vn_kqfilt_remove(vp, ident, p)
 	
 	if ( (error = vnode_getwithref(vp)) == 0 ) {
 	        context.vc_proc = p;
-		context.vc_ucred = p->p_ucred;	/* XXX kauth_cred_get() ??? */
+		context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? */
 
 		funnel_state = thread_funnel_set(kernel_flock, TRUE);
 		error = VNOP_KQFILT_REMOVE(vp, ident, &context);
 		thread_funnel_set(kernel_flock, funnel_state);
 
 		(void)vnode_put(vp);
+		kauth_cred_unref(&context.vc_ucred);
 	}
 	return (error);
 }
diff --git a/bsd/vfs/vfs_xattr.c b/bsd/vfs/vfs_xattr.c
index 7530731b2..0ecdcd307 100644
--- a/bsd/vfs/vfs_xattr.c
+++ b/bsd/vfs/vfs_xattr.c
@@ -43,7 +43,7 @@
 
 #include <sys/xattr.h>
 
-#include <architecture/byte_order.h>
+#include <libkern/OSByteOrder.h>
 #include <vm/vm_kern.h>
 
 /*
@@ -479,9 +479,9 @@ typedef struct attr_info {
 	((u_int8_t *)ATTR_NEXT(ae) <= ((ai).rawdata + (ai).rawsize))
 
 
-#define SWAP16(x)  NXSwapBigShortToHost((x))
-#define SWAP32(x)  NXSwapBigIntToHost((x))
-#define SWAP64(x)  NXSwapBigLongLongToHost((x))
+#define SWAP16(x)  OSSwapBigToHostInt16((x))
+#define SWAP32(x)  OSSwapBigToHostInt32((x))
+#define SWAP64(x)  OSSwapBigToHostInt64((x))
 
 
 static u_int32_t emptyfinfo[8] = {0};
@@ -1788,11 +1788,13 @@ write_xattrinfo(attr_info_t *ainfop)
 	uio_addiov(auio, (uintptr_t)ainfop->filehdr, ainfop->iosize);
 
 	swap_adhdr(ainfop->filehdr);
+	if (ainfop->attrhdr != NULL)
 	swap_attrhdr(ainfop->attrhdr);
 
 	error = VNOP_WRITE(ainfop->filevp, auio, 0, ainfop->context);
 
 	swap_adhdr(ainfop->filehdr);
+	if (ainfop->attrhdr != NULL)
 	swap_attrhdr(ainfop->attrhdr);
 	return (error);
 }
diff --git a/bsd/vfs/vnode_if.c b/bsd/vfs/vnode_if.c
index ebf0b1f3d..bad48873e 100644
--- a/bsd/vfs/vnode_if.c
+++ b/bsd/vfs/vnode_if.c
@@ -742,22 +742,6 @@ struct vnodeop_desc vnop_pageout_desc = {
 	NULL,
 };
 
-int vnop_devblocksize_vp_offsets[] = {
-	VOPARG_OFFSETOF(struct vnop_devblocksize_args,a_vp),
-	VDESC_NO_OFFSET
-};
-struct vnodeop_desc vnop_devblocksize_desc = {
-	0,
-	"vnop_devblocksize",
-	0,
-	vnop_devblocksize_vp_offsets,
-	VDESC_NO_OFFSET,
-	VDESC_NO_OFFSET,
-	VDESC_NO_OFFSET,
-	VDESC_NO_OFFSET,
-	NULL,
-};
-
 int vnop_searchfs_vp_offsets[] = {
 	VOPARG_OFFSETOF(struct vnop_searchfs_args,a_vp),
 	VDESC_NO_OFFSET
@@ -985,7 +969,6 @@ struct vnodeop_desc *vfs_op_descs[] = {
 	&vnop_allocate_desc,
 	&vnop_pagein_desc,
 	&vnop_pageout_desc,
-	&vnop_devblocksize_desc,
 	&vnop_searchfs_desc,
 	&vnop_copyfile_desc,
 	&vnop_getxattr_desc,
diff --git a/bsd/vm/vm_unix.c b/bsd/vm/vm_unix.c
index 3f6ddcf79..d60fc526c 100644
--- a/bsd/vm/vm_unix.c
+++ b/bsd/vm/vm_unix.c
@@ -64,9 +64,11 @@
 #include <sys/kernel.h>
 #include <sys/ubc_internal.h>
 #include <sys/user.h>
+#include <sys/syslog.h>
 #include <sys/stat.h>
 #include <sys/sysproto.h>
 #include <sys/mman.h>
+#include <sys/sysctl.h>
 
 #include <bsm/audit_kernel.h>
 #include <bsm/audit_kevents.h>
@@ -82,6 +84,12 @@
 
 #include <vm/vm_protos.h>
 
+void
+log_nx_failure(addr64_t vaddr, vm_prot_t prot)
+{
+        printf("NX failure: %s  -  vaddr=%qx,  prot=%x\n", current_proc()->p_comm, vaddr, prot);
+}
+
 
 int
 useracc(
@@ -349,6 +357,19 @@ pftout:
  *
  * XXX This should be a BSD system call, not a Mach trap!!!
  */
+/* 
+ *
+ * tfp_policy = KERN_TFP_POLICY_DENY; Deny Mode: None allowed except for self
+ * tfp_policy = KERN_TFP_POLICY_PERMISSIVE; Permissive Mode: all permissive; related ones allowed or privileged
+ * tfp_policy = KERN_TFP_POLICY_RESTRICTED; Restricted Mode: self access allowed; setgid (to tfp_group) are allowed for other tasks
+ *
+ */
+static  int tfp_policy = KERN_TFP_POLICY_RESTRICTED;
+/* the groutp is inited to kmem group and is modifiable by sysctl */
+static int tfp_group_inited = 0; /* policy groups are loaded ... */
+static  gid_t tfp_group_ronly = 0; /* procview group */
+static  gid_t tfp_group_rw = 0; /* procmod group */
+
 kern_return_t
 task_for_pid(
 	struct task_for_pid_args *args)
@@ -363,7 +384,12 @@ task_for_pid(
 	mach_port_name_t	tret;
 	void * sright;
 	int error = 0;
+	int is_member = 0;
 	boolean_t funnel_state;
+	boolean_t ispermitted = FALSE;
+#if DIAGNOSTIC
+	char procname[MAXCOMLEN+1];
+#endif /* DIAGNOSTIC */
 
 	AUDIT_MACH_SYSCALL_ENTER(AUE_TASKFORPID);
 	AUDIT_ARG(pid, pid);
@@ -378,7 +404,7 @@ task_for_pid(
 
 	funnel_state = thread_funnel_set(kernel_flock, TRUE);
 
-	p1 = get_bsdtask_info(t1);	/* XXX current proc */
+	p1 = current_proc();
 
 	/*
 	 * Delayed binding of thread credential to process credential, if we
@@ -389,34 +415,84 @@ task_for_pid(
 	    (uthread->uu_flag & UT_SETUID) == 0) {
 		kauth_cred_t old = uthread->uu_ucred;
 		proc_lock(p1);
+		kauth_cred_ref(p1->p_ucred);
 		uthread->uu_ucred = p1->p_ucred;
-		kauth_cred_ref(uthread->uu_ucred);
 		proc_unlock(p1);
-		if (old != NOCRED)
-			kauth_cred_rele(old);
+		if (IS_VALID_CRED(old))
+			kauth_cred_unref(&old);
 	}
 
 	p = pfind(pid);
 	AUDIT_ARG(process, p);
 
-	if (
-		(p != (struct proc *) 0)
-		&& (p1 != (struct proc *) 0)
-		&& (
-			(p1 == p)
-			|| !(suser(kauth_cred_get(), 0))
-			 || ((kauth_cred_getuid(p->p_ucred) == kauth_cred_getuid(kauth_cred_get())) 
-				&& (p->p_ucred->cr_ruid == kauth_cred_get()->cr_ruid)
-				&& ((p->p_flag & P_SUGID) == 0))
-		  )
-		&& (p->p_stat != SZOMB)
-		) {
-			if (p->task != TASK_NULL) {
-				task_reference(p->task);
-				sright = (void *)convert_task_to_port(p->task);
-				tret = ipc_port_copyout_send(
-					sright, 
-					get_task_ipcspace(current_task()));
+	/*
+	 * XXX p_ucred check can be bogus in multithreaded processes,
+	 * XXX unless the funnel is held.
+	 */
+	switch (tfp_policy) {
+
+		case KERN_TFP_POLICY_PERMISSIVE:
+			/* self or suser or related ones */
+			if ((p != (struct proc *) 0)
+				&& (p->p_stat != SZOMB)
+				&& (p1 != (struct proc *) 0)
+				&& (
+					(p1 == p)
+					|| !(suser(kauth_cred_get(), 0))
+					|| ((kauth_cred_getuid(p->p_ucred) == kauth_cred_getuid(kauth_cred_get())) && 
+						((p->p_ucred->cr_ruid == kauth_cred_get()->cr_ruid))
+						&& ((p->p_flag & P_SUGID) == 0))
+					)
+				)
+					ispermitted = TRUE;
+			break;
+
+		case KERN_TFP_POLICY_RESTRICTED:
+			/* self or suser or  setgid and related ones only */
+			if ((p != (struct proc *) 0)
+				&& (p1 != (struct proc *) 0)
+				&& (p->p_stat != SZOMB)
+				&& (
+					(p1 == p)
+					|| !(suser(kauth_cred_get(), 0))
+					|| (((tfp_group_inited != 0) && 
+							(
+							((kauth_cred_ismember_gid(kauth_cred_get(), 
+									tfp_group_ronly, &is_member) == 0) && is_member)
+							||((kauth_cred_ismember_gid(kauth_cred_get(), 
+									tfp_group_rw, &is_member) == 0) && is_member)
+							)
+					   )
+					   && ((kauth_cred_getuid(p->p_ucred) == kauth_cred_getuid(kauth_cred_get())) && 
+							((p->p_ucred->cr_ruid == kauth_cred_get()->cr_ruid))
+							&& ((p->p_flag & P_SUGID) == 0))
+					  )
+					)
+				)
+					ispermitted = TRUE;
+
+			break;
+
+		case KERN_TFP_POLICY_DENY:
+			/* self or suser only */
+		default:
+			/* do not return task port of other task at all */
+			if ((p1 != (struct proc *) 0) && (p != (struct proc *) 0) && (p->p_stat != SZOMB)
+					&& ((p1 == p)  || !(suser(kauth_cred_get(), 0))))
+				ispermitted = TRUE;
+			else
+				ispermitted = FALSE;
+			break;
+	};
+
+
+	if (ispermitted == TRUE) {
+		if (p->task != TASK_NULL) {
+			task_reference(p->task);
+			sright = (void *)convert_task_to_port(p->task);
+			tret = ipc_port_copyout_send(
+						sright, 
+						get_task_ipcspace(current_task()));
 			} else
 				tret  = MACH_PORT_NULL;
 			AUDIT_ARG(mach_port2, tret);
@@ -425,6 +501,22 @@ task_for_pid(
 			error = KERN_SUCCESS;
 			goto tfpout;
 	}
+#if DIAGNOSTIC
+	else {
+		/* 
+		 * There is no guarantee that p_comm is null terminated and
+		 * kernel implementation of string functions are complete. So 
+		 * ensure stale info is not leaked out, bzero the  buffer
+		 */
+		bzero(&procname[0], MAXCOMLEN+1);
+		strncpy(&procname[0], &p1->p_comm[0], MAXCOMLEN);
+		if (tfp_policy != KERN_TFP_POLICY_PERMISSIVE)
+			log(LOG_NOTICE, "(%d: %s)tfp: failed on %d:\n",
+				((p1 != PROC_NULL)?(p1->p_pid):0), &procname[0],
+				((p != PROC_NULL)?(p->p_pid):0));
+	}
+#endif /* DIAGNOSTIC */
+
     task_deallocate(t1);
 	tret = MACH_PORT_NULL;
 	(void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
@@ -435,6 +527,186 @@ tfpout:
 	return(error);
 }
 
+/*
+ *	Routine:	task_name_for_pid
+ *	Purpose:
+ *		Get the task name port for another "process", named by its
+ *		process ID on the same host as "target_task".
+ *
+ *		Only permitted to privileged processes, or processes
+ *		with the same user ID.
+ *
+ * XXX This should be a BSD system call, not a Mach trap!!!
+ */
+
+kern_return_t
+task_name_for_pid(
+	struct task_name_for_pid_args *args)
+{
+	mach_port_name_t	target_tport = args->target_tport;
+	int			pid = args->pid;
+	user_addr_t		task_addr = args->t;
+	struct uthread		*uthread;
+	struct proc	*p;
+	struct proc *p1;
+	task_t		t1;
+	mach_port_name_t	tret;
+	void * sright;
+	int error = 0;
+	boolean_t funnel_state;
+
+	AUDIT_MACH_SYSCALL_ENTER(AUE_TASKNAMEFORPID);
+	AUDIT_ARG(pid, pid);
+	AUDIT_ARG(mach_port1, target_tport);
+
+	t1 = port_name_to_task(target_tport);
+	if (t1 == TASK_NULL) {
+		(void ) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
+		AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
+		return(KERN_FAILURE);
+	} 
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+
+	p1 = current_proc();
+
+	/*
+	 * Delayed binding of thread credential to process credential, if we
+	 * are not running with an explicitly set thread credential.
+	 */
+	/*
+	 * XXX p_ucred check can be bogus in multithreaded processes,
+	 * XXX unless the funnel is held.
+	 */
+	uthread = get_bsdthread_info(current_thread());
+	if (uthread->uu_ucred != p1->p_ucred &&
+	    (uthread->uu_flag & UT_SETUID) == 0) {
+		kauth_cred_t old = uthread->uu_ucred;
+		proc_lock(p1);
+		kauth_cred_ref(p1->p_ucred);
+		uthread->uu_ucred = p1->p_ucred;
+		proc_unlock(p1);
+		if (IS_VALID_CRED(old))
+			kauth_cred_unref(&old);
+	}
+
+	p = pfind(pid);
+	AUDIT_ARG(process, p);
+
+	if ((p != (struct proc *) 0)
+	    && (p->p_stat != SZOMB)
+	    && (p1 != (struct proc *) 0)
+	    && ((p1 == p)
+		|| !(suser(kauth_cred_get(), 0))
+		|| ((kauth_cred_getuid(p->p_ucred) == kauth_cred_getuid(kauth_cred_get())) && 
+		    ((p->p_ucred->cr_ruid == kauth_cred_get()->cr_ruid)))))
+	{
+		if (p->task != TASK_NULL)
+		{
+			task_reference(p->task);
+			sright = (void *)convert_task_name_to_port(p->task);
+			tret = ipc_port_copyout_send(
+						sright, 
+						get_task_ipcspace(current_task()));
+		} else
+			tret  = MACH_PORT_NULL;
+		AUDIT_ARG(mach_port2, tret);
+		(void ) copyout((char *)&tret, task_addr, sizeof(mach_port_name_t));
+	        task_deallocate(t1);
+		error = KERN_SUCCESS;
+		goto tnfpout;
+	}
+
+	task_deallocate(t1);
+	tret = MACH_PORT_NULL;
+	(void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
+	error = KERN_FAILURE;
+tnfpout:
+	thread_funnel_set(kernel_flock, funnel_state);
+	AUDIT_MACH_SYSCALL_EXIT(error);
+	return(error);
+}
+
+static int
+sysctl_settfp_policy(__unused struct sysctl_oid *oidp, void *arg1,
+    __unused int arg2, struct sysctl_req *req)
+{
+    int error = 0;
+	int new_value;
+
+    error = SYSCTL_OUT(req, arg1, sizeof(int));
+    if (error || req->newptr == USER_ADDR_NULL)
+        return(error);
+
+	if (!is_suser())
+		return(EPERM);
+
+	if ((error = SYSCTL_IN(req, &new_value, sizeof(int)))) {
+		goto out;
+	}
+	if ((new_value == KERN_TFP_POLICY_DENY) 
+		|| (new_value == KERN_TFP_POLICY_PERMISSIVE)
+		|| (new_value == KERN_TFP_POLICY_RESTRICTED))
+			tfp_policy = new_value;
+	else
+			error = EINVAL;		
+out:
+    return(error);
+
+}
+
+static int
+sysctl_settfp_groups(__unused struct sysctl_oid *oidp, void *arg1,
+    __unused int arg2, struct sysctl_req *req)
+{
+    int error = 0;
+	int new_value;
+
+    error = SYSCTL_OUT(req, arg1, sizeof(int));
+    if (error || req->newptr == USER_ADDR_NULL)
+        return(error);
+
+	if (!is_suser())
+		return(EPERM);
+
+	/* 
+	 * Once set; cannot be reset till next boot. Launchd will set this
+	 * in its pid 1 init and no one can set after that.
+	 */
+	if (tfp_group_inited != 0)
+		return(EPERM);
+		
+	if ((error = SYSCTL_IN(req, &new_value, sizeof(int)))) {
+		goto out;
+	}
+
+	if (new_value >= 100) 
+			error = EINVAL;		
+	else {
+		if (arg1 == &tfp_group_ronly) 
+			tfp_group_ronly = new_value;
+		else if (arg1 == &tfp_group_rw)
+			tfp_group_rw = new_value;
+		else
+			error = EINVAL;
+		if ((tfp_group_ronly != 0 ) && (tfp_group_rw != 0 ))
+			tfp_group_inited = 1;
+	}
+
+out:
+    return(error);
+}
+
+SYSCTL_NODE(_kern, KERN_TFP, tfp, CTLFLAG_RW, 0, "tfp");
+SYSCTL_PROC(_kern_tfp, KERN_TFP_POLICY, policy, CTLTYPE_INT | CTLFLAG_RW,
+    &tfp_policy, sizeof(uint32_t), &sysctl_settfp_policy ,"I","policy");
+SYSCTL_PROC(_kern_tfp, KERN_TFP_READ_GROUP, read_group, CTLTYPE_INT | CTLFLAG_RW,
+    &tfp_group_ronly, sizeof(uint32_t), &sysctl_settfp_groups ,"I","read_group");
+SYSCTL_PROC(_kern_tfp, KERN_TFP_RW_GROUP, rw_group, CTLTYPE_INT | CTLFLAG_RW,
+    &tfp_group_rw, sizeof(uint32_t), &sysctl_settfp_groups ,"I","rw_group");
+
+
+SYSCTL_INT(_vm, OID_AUTO, shared_region_trace_level, CTLFLAG_RW, &shared_region_trace_level, 0, "");
 
 /*
  * Try and cap the number of mappings the user might be trying to deal with,
@@ -483,6 +755,12 @@ shared_region_make_private_np(
 	user_ranges = uap->ranges;
 	ranges_size = (vm_size_t) (range_count * sizeof (ranges[0]));
 
+	SHARED_REGION_TRACE(
+		SHARED_REGION_TRACE_INFO,
+		("shared_region: %p [%d(%s)] "
+		 "make_private(rangecount=%d)\n",
+		 current_thread(), p->p_pid, p->p_comm, range_count));
+
 	/* allocate kernel space for the "ranges" */
 	if (range_count != 0) {
 		if (range_count > SHARED_REGION_MAX_MAPPINGS) {
@@ -573,7 +851,18 @@ done:
 			  ranges_size);
 		ranges = NULL;
 	}
-	
+
+	SHARED_REGION_TRACE(
+		SHARED_REGION_TRACE_INFO,
+		("shared_region: %p [%d(%s)] "
+		 "make_private(rangecount=%d) -> %d "
+		 "shared_region=%p[%x,%x,%x]\n",
+		 current_thread(), p->p_pid, p->p_comm,
+		 range_count, error, shared_region,
+		 task_mapping_info.fs_base,
+		 task_mapping_info.system,
+		 task_mapping_info.flags));
+
 	return error;
 }
 
@@ -637,17 +926,33 @@ shared_region_map_file_np(
 	/* get file structure from file descriptor */
 	error = fp_lookup(p, fd, &fp, 0);
 	if (error) {
+		SHARED_REGION_TRACE(
+			SHARED_REGION_TRACE_ERROR,
+			("shared_region: %p [%d(%s)] map_file: "
+			 "fd=%d lookup failed (error=%d)\n",
+			 current_thread(), p->p_pid, p->p_comm, fd, error));
 		goto done;
 	}
 
 	/* make sure we're attempting to map a vnode */
 	if (fp->f_fglob->fg_type != DTYPE_VNODE) {
+		SHARED_REGION_TRACE(
+			SHARED_REGION_TRACE_ERROR,
+			("shared_region: %p [%d(%s)] map_file: "
+			 "fd=%d not a vnode (type=%d)\n",
+			 current_thread(), p->p_pid, p->p_comm,
+			 fd, fp->f_fglob->fg_type));
 		error = EINVAL;
 		goto done;
 	}
 
 	/* we need at least read permission on the file */
 	if (! (fp->f_fglob->fg_flag & FREAD)) {
+		SHARED_REGION_TRACE(
+			SHARED_REGION_TRACE_ERROR,
+			("shared_region: %p [%d(%s)] map_file: "
+			 "fd=%d not readable\n",
+			 current_thread(), p->p_pid, p->p_comm, fd));
 		error = EPERM;
 		goto done;
 	}
@@ -655,12 +960,23 @@ shared_region_map_file_np(
 	/* get vnode from file structure */
 	error = vnode_getwithref((vnode_t)fp->f_fglob->fg_data);
 	if (error) {
+		SHARED_REGION_TRACE(
+			SHARED_REGION_TRACE_ERROR,
+			("shared_region: %p [%d(%s)] map_file: "
+			 "fd=%d getwithref failed (error=%d)\n",
+			 current_thread(), p->p_pid, p->p_comm, fd, error));
 		goto done;
 	}
 	vp = (struct vnode *) fp->f_fglob->fg_data;
 
 	/* make sure the vnode is a regular file */
 	if (vp->v_type != VREG) {
+		SHARED_REGION_TRACE(
+			SHARED_REGION_TRACE_ERROR,
+			("shared_region: %p [%d(%s)] map_file(%p:'%s'): "
+			 "not a file (type=%d)\n",
+			 current_thread(), p->p_pid, p->p_comm,
+			 vp, vp->v_name, vp->v_type));
 		error = EINVAL;
 		goto done;
 	}
@@ -671,8 +987,16 @@ shared_region_map_file_np(
 		
 		context.vc_proc = p;
 		context.vc_ucred = kauth_cred_get();
-		if ((error = vnode_size(vp, &fs, &context)) != 0)
+		if ((error = vnode_size(vp, &fs, &context)) != 0) {
+			SHARED_REGION_TRACE(
+				SHARED_REGION_TRACE_ERROR,
+				("shared_region: %p [%d(%s)] "
+				 "map_file(%p:'%s'): "
+				 "vnode_size(%p) failed (error=%d)\n",
+				 current_thread(), p->p_pid, p->p_comm,
+				 vp, vp->v_name, vp));
 			goto done;
+		}
 		file_size = fs;
 	}
 
@@ -682,6 +1006,12 @@ shared_region_map_file_np(
 	mapping_count = uap->mappingCount; /* the number of mappings */
 	mappings_size = (vm_size_t) (mapping_count * sizeof (mappings[0]));
 	if (mapping_count == 0) {
+		SHARED_REGION_TRACE(
+			SHARED_REGION_TRACE_INFO,
+			("shared_region: %p [%d(%s)] map_file(%p:'%s'): "
+			 "no mappings\n",
+			 current_thread(), p->p_pid, p->p_comm,
+			 vp, vp->v_name));
 		error = 0;	/* no mappings: we're done ! */
 		goto done;
 	} else if (mapping_count <= SFM_MAX_STACK) {
@@ -701,6 +1031,13 @@ shared_region_map_file_np(
 				(vm_offset_t *) &mappings,
 				mappings_size);
 		if (kr != KERN_SUCCESS) {
+			SHARED_REGION_TRACE(
+				SHARED_REGION_TRACE_ERROR,
+				("shared_region: %p [%d(%s)] "
+				 "map_file(%p:'%s'): "
+				 "failed to allocate %d mappings (kr=0x%x)\n",
+				 current_thread(), p->p_pid, p->p_comm,
+				 vp, vp->v_name, mapping_count, kr));
 			error = ENOMEM;
 			goto done;
 		}
@@ -711,6 +1048,12 @@ shared_region_map_file_np(
 		       mappings,
 		       mappings_size);
 	if (error != 0) {
+		SHARED_REGION_TRACE(
+			SHARED_REGION_TRACE_ERROR,
+			("shared_region: %p [%d(%s)] map_file(%p:'%s'): "
+			 "failed to copyin %d mappings (error=%d)\n",
+			 current_thread(), p->p_pid, p->p_comm,
+			 vp, vp->v_name, mapping_count, error));
 		goto done;
 	}
 
@@ -739,6 +1082,15 @@ shared_region_map_file_np(
 			/* this mapping is not in the shared region... */
 			if (user_slide_p == NULL) {
 				/* ... and we can't slide it in: fail */
+				SHARED_REGION_TRACE(
+					SHARED_REGION_TRACE_CONFLICT,
+					("shared_region: %p [%d(%s)] "
+					 "map_file(%p:'%s'): "
+					 "mapping %p not in shared segment & "
+					 "no sliding\n",
+					 current_thread(), p->p_pid, p->p_comm,
+					 vp, vp->v_name,
+					 mappings[j].sfm_address));
 				error = EINVAL;
 				goto done;
 			}
@@ -747,6 +1099,15 @@ shared_region_map_file_np(
 				mappings_in_segment = FALSE;
 			} else if (mappings_in_segment != FALSE) {
 				/* other mappings were not outside: fail */
+				SHARED_REGION_TRACE(
+					SHARED_REGION_TRACE_CONFLICT,
+					("shared_region: %p [%d(%s)] "
+					 "map_file(%p:'%s'): "
+					 "mapping %p not in shared segment & "
+					 "other mappings in shared segment\n",
+					 current_thread(), p->p_pid, p->p_comm,
+					 vp, vp->v_name,
+					 mappings[j].sfm_address));
 				error = EINVAL;
 				goto done;
 			}
@@ -757,6 +1118,15 @@ shared_region_map_file_np(
 				mappings_in_segment = TRUE;
 			} else if (mappings_in_segment != TRUE) {
 				/* other mappings were not inside: fail */
+				SHARED_REGION_TRACE(
+					SHARED_REGION_TRACE_CONFLICT,
+					("shared_region: %p [%d(%s)] "
+					 "map_file(%p:'%s'): "
+					 "mapping %p in shared segment & "
+					 "others in shared segment\n",
+					 current_thread(), p->p_pid, p->p_comm,
+					 vp, vp->v_name,
+					 mappings[j].sfm_address));
 				error = EINVAL;
 				goto done;
 			}
@@ -800,6 +1170,12 @@ shared_region_map_file_np(
 	UBCINFOCHECK("shared_region_map_file_np", vp);
 	file_control = ubc_getobject(vp, UBC_HOLDOBJECT);
 	if (file_control == MEMORY_OBJECT_CONTROL_NULL) {
+		SHARED_REGION_TRACE(
+			SHARED_REGION_TRACE_ERROR,
+			("shared_region: %p [%d(%s)] map_file(%p:'%s'): "
+			 "ubc_getobject() failed\n",
+			 current_thread(), p->p_pid, p->p_comm,
+			 vp, vp->v_name));
 		error = EINVAL;
 		goto done;
 	}
@@ -848,6 +1224,13 @@ shared_region_map_file_np(
 		 * (via shared_region_make_private()) the shared region and
 		 * try to establish the mapping privately for this process.
 		 */
+		SHARED_REGION_TRACE(
+			SHARED_REGION_TRACE_CONFLICT,
+			("shared_region: %p [%d(%s)] "
+			 "map_file(%p:'%s'): "
+			 "not on root volume\n",
+			 current_thread(), p->p_pid, p->p_comm,
+			 vp->v_name));
 		error = EXDEV;
 		goto done;
 	}
@@ -864,8 +1247,7 @@ shared_region_map_file_np(
 			     base_offset,
 			     (user_slide_p) ? &slide : NULL);
 
-	switch (kr) {
-	case KERN_SUCCESS:
+	if (kr == KERN_SUCCESS) {
 		/*
 		 * The mapping was successful.  Let the buffer cache know
 		 * that we've mapped that file with these protections.  This
@@ -873,21 +1255,30 @@ shared_region_map_file_np(
 		 */
 		(void) ubc_map(vp, max_prot);
 		error = 0;
-		break;
-	case KERN_INVALID_ADDRESS:
-		error = EFAULT;
-		goto done;
-	case KERN_PROTECTION_FAILURE:
-		error = EPERM;
-		goto done;
-	case KERN_NO_SPACE:
-		error = ENOMEM;
-		goto done;
-	case KERN_FAILURE:
-	case KERN_INVALID_ARGUMENT:
-	default:
-		error = EINVAL;
-		goto done;
+	} else {
+		SHARED_REGION_TRACE(
+			SHARED_REGION_TRACE_CONFLICT,
+			("shared_region: %p [%d(%s)] "
+			 "map_file(%p:'%s'): "
+			 "map_shared_file failed, kr=0x%x\n",
+			 current_thread(), p->p_pid, p->p_comm,
+			 vp, vp->v_name, kr));
+		switch (kr) {
+		case KERN_INVALID_ADDRESS:
+			error = EFAULT;
+			goto done;
+		case KERN_PROTECTION_FAILURE:
+			error = EPERM;
+			goto done;
+		case KERN_NO_SPACE:
+			error = ENOMEM;
+			goto done;
+		case KERN_FAILURE:
+		case KERN_INVALID_ARGUMENT:
+		default:
+			error = EINVAL;
+			goto done;
+		}
 	}
 
 	if (p->p_flag & P_NOSHLIB) {
@@ -916,7 +1307,16 @@ shared_region_map_file_np(
 		}
 		error = copyout(&slide,
 				user_slide_p,
-				sizeof (int64_t));
+				sizeof (slide));
+		if (slide != 0) {
+			SHARED_REGION_TRACE(
+				SHARED_REGION_TRACE_CONFLICT,
+				("shared_region: %p [%d(%s)] "
+				 "map_file(%p:'%s'): "
+				 "slid by 0x%llx\n",
+				 current_thread(), p->p_pid, p->p_comm,
+				 vp, vp->v_name, slide));
+		}
 	}
 
 done:
@@ -999,23 +1399,31 @@ clone_system_shared_regions(
 		&(old_info.fs_base),
 		&(old_info.system),
 		&(old_info.flags), &next);
-	if ((shared_regions_active) ||
-		(base_vnode == ENV_DEFAULT_ROOT)) {
-	   if (shared_file_create_system_region(&new_shared_region))
-		return (ENOMEM);
+
+	if (shared_regions_active ||
+	    base_vnode == ENV_DEFAULT_ROOT) {
+		if (shared_file_create_system_region(&new_shared_region,
+						     old_info.fs_base,
+						     old_info.system))
+			return ENOMEM;
 	} else {
-	   new_shared_region = 
-		lookup_default_shared_region(
-			base_vnode, old_info.system);
-	   if(new_shared_region == NULL) {
-		shared_file_boot_time_init(
-			base_vnode, old_info.system);
-		vm_get_shared_region(current_task(), &new_shared_region);
-	   } else {
-		vm_set_shared_region(current_task(), new_shared_region);
-	   }
-	   if(old_shared_region)
-		shared_region_mapping_dealloc(old_shared_region);
+		if (old_shared_region &&
+		    base_vnode == ENV_DEFAULT_ROOT) {
+			base_vnode = old_info.fs_base;
+		}
+		new_shared_region =
+			lookup_default_shared_region(base_vnode,
+						     old_info.system);
+		if (new_shared_region == NULL) {
+			shared_file_boot_time_init(base_vnode,
+						   old_info.system);
+			vm_get_shared_region(current_task(),
+					     &new_shared_region);
+		} else {
+			vm_set_shared_region(current_task(), new_shared_region);
+		}
+		if (old_shared_region)
+			shared_region_mapping_dealloc(old_shared_region);
 	}
 	new_info.self = (vm_offset_t)new_shared_region;
 	shared_region_mapping_info(new_shared_region,
@@ -1049,18 +1457,19 @@ clone_system_shared_regions(
 		    */
 		   shared_region_object_chain_attach(
 			   new_shared_region, old_shared_region);
-	   } else {
-		   /*
-		    * We want a completely detached clone with no link to
-		    * the old shared region.  We'll be removing some mappings
-		    * in our private, cloned, shared region, so the old mappings
-		    * will become irrelevant to us.  Since we have a private
-		    * "shared region" now, it isn't going to be shared with
-		    * anyone else and we won't need to maintain mappings info.
-		    */
-		   shared_region_object_chain_detached(new_shared_region);
 	   }
 	}
+	if (!chain_regions) {
+		/*
+		 * We want a completely detached clone with no link to
+		 * the old shared region.  We'll be removing some mappings
+		 * in our private, cloned, shared region, so the old mappings
+		 * will become irrelevant to us.  Since we have a private
+		 * "shared region" now, it isn't going to be shared with
+		 * anyone else and we won't need to maintain mappings info.
+		 */
+		shared_region_object_chain_detached(new_shared_region);
+	}
 	if (vm_map_region_replace(current_map(), old_info.text_region, 
 			new_info.text_region, old_info.client_base, 
 			old_info.client_base+old_info.text_size)) {
@@ -1084,6 +1493,22 @@ clone_system_shared_regions(
 	if (!shared_regions_active || !chain_regions)
 		shared_region_mapping_dealloc(old_shared_region);
 
+	SHARED_REGION_TRACE(
+		SHARED_REGION_TRACE_INFO,
+		("shared_region: %p task=%p "
+		 "clone(active=%d, base=0x%x,chain=%d) "
+		 "old=%p[%x,%x,%x] new=%p[%x,%x,%x]\n",
+		 current_thread(), current_task(), 
+		 shared_regions_active, base_vnode, chain_regions,
+		 old_shared_region,
+		 old_info.fs_base,
+		 old_info.system,
+		 old_info.flags,
+		 new_shared_region,
+		 new_info.fs_base,
+		 new_info.system,
+		 new_info.flags));
+
 	return(0);
 
 }
@@ -1440,9 +1865,12 @@ restart:
 	resid_off = 0;
 
 	while(size) {
+		int resid_int;
 		error = vn_rdwr(UIO_READ, names_vp, (caddr_t)buf_ptr, 
 			size, resid_off,
-			UIO_SYSSPACE32, IO_NODELOCKED, kauth_cred_get(), &resid, p);
+			UIO_SYSSPACE32, IO_NODELOCKED, kauth_cred_get(),
+			&resid_int, p);
+		resid = (vm_size_t) resid_int;
 		if((error) || (size == resid)) {
 			if(!error) {
 				error = EINVAL;
@@ -1584,10 +2012,12 @@ bsd_read_page_cache_file(
 		}
 		*bufsize = profile_size;
 		while(profile_size) {
+			int resid_int;
 			error = vn_rdwr(UIO_READ, data_vp, 
 				(caddr_t) *buffer, profile_size, 
 				profile, UIO_SYSSPACE32, IO_NODELOCKED, 
-				kauth_cred_get(), &resid, p);
+				kauth_cred_get(), &resid_int, p);
+			resid = (vm_size_t) resid_int;
 			if((error) || (profile_size == resid)) {
 				bsd_close_page_cache_files(uid_files);
 				kmem_free(kernel_map, (vm_offset_t)*buffer, profile_size);
@@ -1699,10 +2129,12 @@ bsd_search_page_cache_data_base(
 		}
 		resid_off = 0;
 		while(size) {
+			int resid_int;
 			error = vn_rdwr(UIO_READ, vp, 
 				CAST_DOWN(caddr_t, (local_buf + resid_off)),
 				size, file_off + resid_off, UIO_SYSSPACE32, 
-				IO_NODELOCKED, kauth_cred_get(), &resid, p);
+				IO_NODELOCKED, kauth_cred_get(), &resid_int, p);
+			resid = (vm_size_t) resid_int;
 			if((error) || (size == resid)) {
 				if(local_buf != 0) {
 					kmem_free(kernel_map, local_buf, 4 * PAGE_SIZE);
diff --git a/bsd/vm/vnode_pager.c b/bsd/vm/vnode_pager.c
index f6969d1bc..552c0afb0 100644
--- a/bsd/vm/vnode_pager.c
+++ b/bsd/vm/vnode_pager.c
@@ -88,6 +88,32 @@ vnode_pager_get_filesize(struct vnode *vp)
 	return (vm_object_offset_t) ubc_getsize(vp);
 }
 
+kern_return_t
+vnode_pager_get_pathname(
+	struct vnode	*vp,
+	char		*pathname,
+	vm_size_t	*length_p)
+{
+	int	error, len;
+
+	len = (int) *length_p;
+	error = vn_getpath(vp, pathname, &len);
+	if (error != 0) {
+		return KERN_FAILURE;
+	}
+	*length_p = (vm_size_t) len;
+	return KERN_SUCCESS;
+}
+
+kern_return_t
+vnode_pager_get_filename(
+	struct vnode	*vp,
+	char		**filename)
+{
+	*filename = vp->v_name;
+	return KERN_SUCCESS;
+}
+
 pager_return_t
 vnode_pageout(struct vnode *vp,
 	upl_t			upl,
diff --git a/bsd/vm/vnode_pager.h b/bsd/vm/vnode_pager.h
index a63c66a88..b41cc7d52 100644
--- a/bsd/vm/vnode_pager.h
+++ b/bsd/vm/vnode_pager.h
@@ -139,6 +139,15 @@ pager_return_t	vnode_pageout(struct vnode *, upl_t,
 extern vm_object_offset_t vnode_pager_get_filesize(
 	struct vnode *vp);
 
+extern kern_return_t vnode_pager_get_pathname(
+	struct vnode	*vp,
+	char		*pathname,
+	vm_size_t	*length_p);
+
+extern kern_return_t vnode_pager_get_filename(
+	struct vnode	*vp,
+	char		**filename);
+
 #endif	/* KERNEL */
 
 #endif	/* _VNODE_PAGER_ */
diff --git a/config/BSDKernel.exports b/config/BSDKernel.exports
index eea8f21d8..fbbf9ac5e 100644
--- a/config/BSDKernel.exports
+++ b/config/BSDKernel.exports
@@ -124,7 +124,6 @@ _err_bwrite
 _err_close
 _err_copyfile
 _err_create
-_err_devblocksize
 _err_exchange
 _err_fsync
 _err_getattr
@@ -304,6 +303,7 @@ _kauth_cred_ref
 _kauth_cred_rele
 _kauth_cred_uid2guid
 _kauth_cred_uid2ntsid
+_kauth_cred_unref
 _kauth_deregister_scope
 _kauth_getgid
 _kauth_getruid
@@ -408,7 +408,6 @@ _nop_bwrite
 _nop_close
 _nop_copyfile
 _nop_create
-_nop_devblocksize
 _nop_exchange
 _nop_fsync
 _nop_getattr
diff --git a/config/IOKit.exports b/config/IOKit.exports
index 4729bf597..6cfdf9312 100644
--- a/config/IOKit.exports
+++ b/config/IOKit.exports
@@ -164,9 +164,9 @@ __ZN10IOWorkLoop10wakeupGateEPvb
 __ZN10IOWorkLoop12tryCloseGateEv
 __ZN10IOWorkLoop13_maintRequestEPvS0_S0_S0_
 __ZN10IOWorkLoop14addEventSourceEP13IOEventSource
+__ZN10IOWorkLoop15runEventSourcesEv
 __ZN10IOWorkLoop17removeEventSourceEP13IOEventSource
 __ZN10IOWorkLoop19signalWorkAvailableEv
-__ZN10IOWorkLoop20_RESERVEDIOWorkLoop1Ev
 __ZN10IOWorkLoop20_RESERVEDIOWorkLoop2Ev
 __ZN10IOWorkLoop20_RESERVEDIOWorkLoop3Ev
 __ZN10IOWorkLoop20_RESERVEDIOWorkLoop4Ev
@@ -261,6 +261,49 @@ __ZN11IOResourcesC2EPK11OSMetaClass
 __ZN11IOResourcesC2Ev
 __ZN11IOResourcesD0Ev
 __ZN11IOResourcesD2Ev
+__ZN12IODMACommand10gMetaClassE
+__ZN12IODMACommand10superClassE
+__ZN12IODMACommand11OutputBig32EPS_NS_9Segment64EPvm
+__ZN12IODMACommand11OutputBig64EPS_NS_9Segment64EPvm
+__ZN12IODMACommand11synchronizeEm
+__ZN12IODMACommand12OutputHost32EPS_NS_9Segment64EPvm
+__ZN12IODMACommand12OutputHost64EPS_NS_9Segment64EPvm
+__ZN12IODMACommand12cloneCommandEPv
+__ZN12IODMACommand14OutputLittle32EPS_NS_9Segment64EPvm
+__ZN12IODMACommand14OutputLittle64EPS_NS_9Segment64EPvm
+__ZN12IODMACommand15genIOVMSegmentsEPyPvPm
+__ZN12IODMACommand17withSpecificationEPFbPS_NS_9Segment64EPvmEhyNS_14MappingOptionsEymP8IOMapperS2_
+__ZN12IODMACommand19setMemoryDescriptorEPK18IOMemoryDescriptorb
+__ZN12IODMACommand21clearMemoryDescriptorEb
+__ZN12IODMACommand21initWithSpecificationEPFbPS_NS_9Segment64EPvmEhyNS_14MappingOptionsEymP8IOMapperS2_
+__ZN12IODMACommand22_RESERVEDIODMACommand0Ev
+__ZN12IODMACommand22_RESERVEDIODMACommand1Ev
+__ZN12IODMACommand22_RESERVEDIODMACommand2Ev
+__ZN12IODMACommand22_RESERVEDIODMACommand3Ev
+__ZN12IODMACommand22_RESERVEDIODMACommand4Ev
+__ZN12IODMACommand22_RESERVEDIODMACommand5Ev
+__ZN12IODMACommand22_RESERVEDIODMACommand6Ev
+__ZN12IODMACommand22_RESERVEDIODMACommand7Ev
+__ZN12IODMACommand22_RESERVEDIODMACommand8Ev
+__ZN12IODMACommand22_RESERVEDIODMACommand9Ev
+__ZN12IODMACommand23_RESERVEDIODMACommand10Ev
+__ZN12IODMACommand23_RESERVEDIODMACommand11Ev
+__ZN12IODMACommand23_RESERVEDIODMACommand12Ev
+__ZN12IODMACommand23_RESERVEDIODMACommand13Ev
+__ZN12IODMACommand23_RESERVEDIODMACommand14Ev
+__ZN12IODMACommand23_RESERVEDIODMACommand15Ev
+__ZN12IODMACommand4freeEv
+__ZN12IODMACommand7prepareEyybb
+__ZN12IODMACommand8completeEbb
+__ZN12IODMACommand9MetaClassC1Ev
+__ZN12IODMACommand9MetaClassC2Ev
+__ZN12IODMACommand9metaClassE
+__ZN12IODMACommandC1EPK11OSMetaClass
+__ZN12IODMACommandC1Ev
+__ZN12IODMACommandC2EPK11OSMetaClass
+__ZN12IODMACommandC2Ev
+__ZN12IODMACommandD0Ev
+__ZN12IODMACommandD2Ev
 __ZN12IOPMinformee10gMetaClassE
 __ZN12IOPMinformee10initializeEP9IOService
 __ZN12IOPMinformee10superClassE
@@ -334,6 +377,8 @@ __ZN12IOUserClient26removeMappingForDescriptorEP18IOMemoryDescriptor
 __ZN12IOUserClient30getExternalAsyncMethodForIndexEm
 __ZN12IOUserClient31getAsyncTargetAndMethodForIndexEPP9IOServicem
 __ZN12IOUserClient4freeEv
+__ZN12IOUserClient4initEP12OSDictionary
+__ZN12IOUserClient4initEv
 __ZN12IOUserClient9MetaClassC1Ev
 __ZN12IOUserClient9MetaClassC2Ev
 __ZN12IOUserClient9metaClassE
@@ -369,6 +414,7 @@ __ZN13IOCommandGate10gMetaClassE
 __ZN13IOCommandGate10runCommandEPvS0_S0_S0_
 __ZN13IOCommandGate10superClassE
 __ZN13IOCommandGate11commandGateEP8OSObjectPFiS1_PvS2_S2_S2_E
+__ZN13IOCommandGate11setWorkLoopEP10IOWorkLoop
 __ZN13IOCommandGate12checkForWorkEv
 __ZN13IOCommandGate12commandSleepEPvm
 __ZN13IOCommandGate13attemptActionEPFiP8OSObjectPvS2_S2_S2_ES2_S2_S2_S2_
@@ -382,7 +428,10 @@ __ZN13IOCommandGate23_RESERVEDIOCommandGate4Ev
 __ZN13IOCommandGate23_RESERVEDIOCommandGate5Ev
 __ZN13IOCommandGate23_RESERVEDIOCommandGate6Ev
 __ZN13IOCommandGate23_RESERVEDIOCommandGate7Ev
+__ZN13IOCommandGate4freeEv
 __ZN13IOCommandGate4initEP8OSObjectPFiS1_PvS2_S2_S2_E
+__ZN13IOCommandGate6enableEv
+__ZN13IOCommandGate7disableEv
 __ZN13IOCommandGate9MetaClassC1Ev
 __ZN13IOCommandGate9MetaClassC2Ev
 __ZN13IOCommandGate9metaClassE
@@ -506,9 +555,11 @@ __ZN14IOPMrootDomain12tellChangeUpEm
 __ZN14IOPMrootDomain12unIdleDeviceEP9IOServicem
 __ZN14IOPMrootDomain12wakeFromDozeEv
 __ZN14IOPMrootDomain13askChangeDownEm
+__ZN14IOPMrootDomain13copyPMSettingEP8OSSymbol
 __ZN14IOPMrootDomain13restartSystemEv
 __ZN14IOPMrootDomain13setPropertiesEP8OSObject
 __ZN14IOPMrootDomain14publishFeatureEPKc
+__ZN14IOPMrootDomain14publishFeatureEPKcjPj
 __ZN14IOPMrootDomain14shutdownSystemEv
 __ZN14IOPMrootDomain14tellChangeDownEm
 __ZN14IOPMrootDomain15powerChangeDoneEm
@@ -522,6 +573,7 @@ __ZN14IOPMrootDomain17setSleepSupportedEm
 __ZN14IOPMrootDomain18changePowerStateToEm
 __ZN14IOPMrootDomain19sysPowerDownHandlerEPvS0_mP9IOServiceS0_j
 __ZN14IOPMrootDomain22changePowerStateToPrivEm
+__ZN14IOPMrootDomain22removePublishedFeatureEj
 __ZN14IOPMrootDomain23requestPowerDomainStateEmP17IOPowerConnectionm
 __ZN14IOPMrootDomain23setQuickSpinDownTimeoutEv
 __ZN14IOPMrootDomain24displayWranglerPublishedEPvS0_P9IOService
@@ -530,7 +582,8 @@ __ZN14IOPMrootDomain25announcePowerSourceChangeEv
 __ZN14IOPMrootDomain26handleSleepTimerExpirationEv
 __ZN14IOPMrootDomain26restoreUserSpinDownTimeoutEv
 __ZN14IOPMrootDomain27displayWranglerNotificationEPvS0_mP9IOServiceS0_j
-__ZN14IOPMrootDomain27registerPMSettingControllerEPFiiiPvES0_
+__ZN14IOPMrootDomain27registerPMSettingControllerEPPK8OSSymbolPFiP8OSObjectS2_S5_mES5_mPS5_
+__ZN14IOPMrootDomain27registerPMSettingControllerEPPK8OSSymboljPFiP8OSObjectS2_S5_mES5_mPS5_
 __ZN14IOPMrootDomain39stopIgnoringClamshellEventsDuringWakeupEv
 __ZN14IOPMrootDomain5startEP9IOService
 __ZN14IOPMrootDomain9MetaClassC1Ev
@@ -563,24 +616,54 @@ __ZN15IOConditionLockC2EPK11OSMetaClass
 __ZN15IOConditionLockC2Ev
 __ZN15IOConditionLockD0Ev
 __ZN15IOConditionLockD2Ev
+__ZN15IOPMPowerSource10cycleCountEv
 __ZN15IOPMPowerSource10gMetaClassE
 __ZN15IOPMPowerSource10isChargingEv
+__ZN15IOPMPowerSource10setVoltageEj
 __ZN15IOPMPowerSource10superClassE
-__ZN15IOPMPowerSource11acConnectedEv
+__ZN15IOPMPowerSource11adapterInfoEv
 __ZN15IOPMPowerSource11atWarnLevelEv
-__ZN15IOPMPowerSource11curCapacityEv
-__ZN15IOPMPowerSource11isInstalledEv
 __ZN15IOPMPowerSource11maxCapacityEv
-__ZN15IOPMPowerSource12currentDrawnEv
+__ZN15IOPMPowerSource11powerSourceEv
+__ZN15IOPMPowerSource11setAmperageEi
+__ZN15IOPMPowerSource11setLocationEi
+__ZN15IOPMPowerSource12manufacturerEv
 __ZN15IOPMPowerSource12updateStatusEv
+__ZN15IOPMPowerSource13setCycleCountEj
+__ZN15IOPMPowerSource13setIsChargingEb
 __ZN15IOPMPowerSource13timeRemainingEv
+__ZN15IOPMPowerSource14errorConditionEv
+__ZN15IOPMPowerSource14setAdapterInfoEi
+__ZN15IOPMPowerSource14setAtWarnLevelEb
+__ZN15IOPMPowerSource14setMaxCapacityEj
+__ZN15IOPMPowerSource15atCriticalLevelEv
+__ZN15IOPMPowerSource15currentCapacityEv
+__ZN15IOPMPowerSource15setManufacturerEP8OSSymbol
+__ZN15IOPMPowerSource16batteryInstalledEv
+__ZN15IOPMPowerSource16setTimeRemainingEi
+__ZN15IOPMPowerSource17externalConnectedEv
+__ZN15IOPMPowerSource17setErrorConditionEP8OSSymbol
+__ZN15IOPMPowerSource18setAtCriticalLevelEb
+__ZN15IOPMPowerSource18setCurrentCapacityEj
+__ZN15IOPMPowerSource19legacyIOBatteryInfoEv
+__ZN15IOPMPowerSource19setBatteryInstalledEb
+__ZN15IOPMPowerSource20setExternalConnectedEb
+__ZN15IOPMPowerSource21externalChargeCapableEv
+__ZN15IOPMPowerSource22setLegacyIOBatteryInfoEP12OSDictionary
 __ZN15IOPMPowerSource24capacityPercentRemainingEv
-__ZN15IOPMPowerSource4initEt
+__ZN15IOPMPowerSource24setExternalChargeCapableEb
+__ZN15IOPMPowerSource4freeEv
+__ZN15IOPMPowerSource4initEv
+__ZN15IOPMPowerSource5modelEv
+__ZN15IOPMPowerSource6serialEv
 __ZN15IOPMPowerSource7voltageEv
-__ZN15IOPMPowerSource8depletedEv
+__ZN15IOPMPowerSource8amperageEv
+__ZN15IOPMPowerSource8locationEv
+__ZN15IOPMPowerSource8setModelEP8OSSymbol
 __ZN15IOPMPowerSource9MetaClassC1Ev
 __ZN15IOPMPowerSource9MetaClassC2Ev
 __ZN15IOPMPowerSource9metaClassE
+__ZN15IOPMPowerSource9setSerialEP8OSSymbol
 __ZN15IOPMPowerSourceC1EPK11OSMetaClass
 __ZN15IOPMPowerSourceC1Ev
 __ZN15IOPMPowerSourceC2EPK11OSMetaClass
@@ -817,11 +900,12 @@ __ZN18IOMemoryDescriptor13removeMappingEP11IOMemoryMap
 __ZN18IOMemoryDescriptor15initWithOptionsEPvmmP4taskmP8IOMapper
 __ZN18IOMemoryDescriptor16getSourceSegmentEmPm
 __ZN18IOMemoryDescriptor16performOperationEmmm
+__ZN18IOMemoryDescriptor16withAddressRangeEyymP4task
+__ZN18IOMemoryDescriptor17withAddressRangesEP14IOAddressRangemmP4task
 __ZN18IOMemoryDescriptor18getPhysicalAddressEv
 __ZN18IOMemoryDescriptor18withPhysicalRangesEP15IOPhysicalRangem11IODirectionb
 __ZN18IOMemoryDescriptor19withPhysicalAddressEmm11IODirection
 __ZN18IOMemoryDescriptor20getPhysicalSegment64EmPm
-__ZN18IOMemoryDescriptor28_RESERVEDIOMemoryDescriptor5Ev
 __ZN18IOMemoryDescriptor28_RESERVEDIOMemoryDescriptor6Ev
 __ZN18IOMemoryDescriptor28_RESERVEDIOMemoryDescriptor7Ev
 __ZN18IOMemoryDescriptor28_RESERVEDIOMemoryDescriptor8Ev
@@ -1085,6 +1169,7 @@ __ZN21IOSubMemoryDescriptor16getSourceSegmentEmPm
 __ZN21IOSubMemoryDescriptor16performOperationEmmm
 __ZN21IOSubMemoryDescriptor17getVirtualSegmentEmPm
 __ZN21IOSubMemoryDescriptor18getPhysicalSegmentEmPm
+__ZN21IOSubMemoryDescriptor20getPhysicalSegment64EmPm
 __ZN21IOSubMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb
 __ZN21IOSubMemoryDescriptor23initWithPhysicalAddressEmm11IODirection
 __ZN21IOSubMemoryDescriptor4freeEv
@@ -1187,9 +1272,10 @@ __ZN24IOBufferMemoryDescriptor15initWithAddressEjm11IODirectionP4task
 __ZN24IOBufferMemoryDescriptor15initWithOptionsEmjj
 __ZN24IOBufferMemoryDescriptor15initWithOptionsEmjjP4task
 __ZN24IOBufferMemoryDescriptor17inTaskWithOptionsEP4taskmjj
+__ZN24IOBufferMemoryDescriptor20initWithPhysicalMaskEP4taskmyyy
+__ZN24IOBufferMemoryDescriptor22inTaskWithPhysicalMaskEP4taskmyy
 __ZN24IOBufferMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb
 __ZN24IOBufferMemoryDescriptor23initWithPhysicalAddressEmm11IODirection
-__ZN24IOBufferMemoryDescriptor34_RESERVEDIOBufferMemoryDescriptor1Ev
 __ZN24IOBufferMemoryDescriptor34_RESERVEDIOBufferMemoryDescriptor2Ev
 __ZN24IOBufferMemoryDescriptor34_RESERVEDIOBufferMemoryDescriptor3Ev
 __ZN24IOBufferMemoryDescriptor34_RESERVEDIOBufferMemoryDescriptor4Ev
@@ -1229,6 +1315,7 @@ __ZN25IOGeneralMemoryDescriptor15unmapFromKernelEv
 __ZN25IOGeneralMemoryDescriptor16getSourceSegmentEmPm
 __ZN25IOGeneralMemoryDescriptor17getVirtualSegmentEmPm
 __ZN25IOGeneralMemoryDescriptor18getPhysicalSegmentEmPm
+__ZN25IOGeneralMemoryDescriptor20getPhysicalSegment64EmPm
 __ZN25IOGeneralMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb
 __ZN25IOGeneralMemoryDescriptor23initWithPhysicalAddressEmm11IODirection
 __ZN25IOGeneralMemoryDescriptor4freeEv
@@ -1350,7 +1437,6 @@ __ZN8IOMapper10superClassE
 __ZN8IOMapper11NewARTTableEmPPvPj
 __ZN8IOMapper12FreeARTTableEP6OSDatam
 __ZN8IOMapper17setMapperRequiredEb
-__ZN8IOMapper18_RESERVEDIOMapper0Ev
 __ZN8IOMapper18_RESERVEDIOMapper1Ev
 __ZN8IOMapper18_RESERVEDIOMapper2Ev
 __ZN8IOMapper18_RESERVEDIOMapper3Ev
@@ -1499,6 +1585,7 @@ __ZN9IOService15getDeviceMemoryEv
 __ZN9IOService15getPMRootDomainEv
 __ZN9IOService15instruct_driverEm
 __ZN9IOService15lookupInterruptEibPP21IOInterruptController
+__ZN9IOService15nextIdleTimeoutE12UnsignedWideS0_j
 __ZN9IOService15powerChangeDoneEm
 __ZN9IOService15probeCandidatesEP12OSOrderedSet
 __ZN9IOService15publishResourceEPK8OSSymbolP8OSObject
@@ -1522,6 +1609,7 @@ __ZN9IOService16applyToProvidersEPFvPS_PvES1_
 __ZN9IOService16command_receivedEPvS0_S0_S0_
 __ZN9IOService16didYouWakeSystemEv
 __ZN9IOService16disableInterruptEi
+__ZN9IOService16getCPUSnoopDelayEv
 __ZN9IOService16getInterruptTypeEiPi
 __ZN9IOService16registerInterestEPK8OSSymbolPFiPvS3_mPS_S3_jES3_S3_
 __ZN9IOService16removePowerChildEP17IOPowerConnection
@@ -1530,6 +1618,7 @@ __ZN9IOService16resolveInterruptEPS_i
 __ZN9IOService16resourceMatchingEPK8OSStringP12OSDictionary
 __ZN9IOService16resourceMatchingEPKcP12OSDictionary
 __ZN9IOService16scheduleFinalizeEv
+__ZN9IOService16setCPUSnoopDelayEm
 __ZN9IOService16startSettleTimerEm
 __ZN9IOService16start_our_changeEm
 __ZN9IOService16stringFromReturnEi
@@ -1552,9 +1641,9 @@ __ZN9IOService18getResourceServiceEv
 __ZN9IOService18lockForArbitrationEb
 __ZN9IOService18matchPropertyTableEP12OSDictionary
 __ZN9IOService18matchPropertyTableEP12OSDictionaryPl
+__ZN9IOService18requireMaxBusStallEm
 __ZN9IOService18setIdleTimerPeriodEm
 __ZN9IOService18settleTimerExpiredEv
-__ZN9IOService15nextIdleTimeoutE12UnsignedWideS0_j
 __ZN9IOService19_RESERVEDIOService4Ev
 __ZN9IOService19_RESERVEDIOService5Ev
 __ZN9IOService19_RESERVEDIOService6Ev
@@ -1611,22 +1700,6 @@ __ZN9IOService20_RESERVEDIOService44Ev
 __ZN9IOService20_RESERVEDIOService45Ev
 __ZN9IOService20_RESERVEDIOService46Ev
 __ZN9IOService20_RESERVEDIOService47Ev
-__ZN9IOService20_RESERVEDIOService48Ev
-__ZN9IOService20_RESERVEDIOService49Ev
-__ZN9IOService20_RESERVEDIOService50Ev
-__ZN9IOService20_RESERVEDIOService51Ev
-__ZN9IOService20_RESERVEDIOService52Ev
-__ZN9IOService20_RESERVEDIOService53Ev
-__ZN9IOService20_RESERVEDIOService54Ev
-__ZN9IOService20_RESERVEDIOService55Ev
-__ZN9IOService20_RESERVEDIOService56Ev
-__ZN9IOService20_RESERVEDIOService57Ev
-__ZN9IOService20_RESERVEDIOService58Ev
-__ZN9IOService20_RESERVEDIOService59Ev
-__ZN9IOService20_RESERVEDIOService60Ev
-__ZN9IOService20_RESERVEDIOService61Ev
-__ZN9IOService20_RESERVEDIOService62Ev
-__ZN9IOService20_RESERVEDIOService63Ev
 __ZN9IOService20callPlatformFunctionEPK8OSSymbolbPvS3_S3_S3_
 __ZN9IOService20callPlatformFunctionEPKcbPvS2_S2_S2_
 __ZN9IOService20getDeviceMemoryCountEv
@@ -1745,6 +1818,9 @@ __ZNK11IOMemoryMap9MetaClass5allocEv
 __ZNK11IOResources11getWorkLoopEv
 __ZNK11IOResources12getMetaClassEv
 __ZNK11IOResources9MetaClass5allocEv
+__ZNK12IODMACommand12getMetaClassEv
+__ZNK12IODMACommand19getMemoryDescriptorEv
+__ZNK12IODMACommand9MetaClass5allocEv
 __ZNK12IOPMinformee12getMetaClassEv
 __ZNK12IOPMinformee9MetaClass5allocEv
 __ZNK12IORootParent12getMetaClassEv
@@ -1840,6 +1916,7 @@ __ZNK17IOPowerConnection12getMetaClassEv
 __ZNK17IOPowerConnection9MetaClass5allocEv
 __ZNK18IOMemoryDescriptor12getDirectionEv
 __ZNK18IOMemoryDescriptor12getMetaClassEv
+__ZNK18IOMemoryDescriptor19dmaCommandOperationEmPvj
 __ZNK18IOMemoryDescriptor9MetaClass5allocEv
 __ZNK18IOMemoryDescriptor9getLengthEv
 __ZNK18IOPMchangeNoteList12getMetaClassEv
@@ -1865,6 +1942,7 @@ __ZNK21IOInterruptController9MetaClass5allocEv
 __ZNK21IONaturalMemoryCursor12getMetaClassEv
 __ZNK21IONaturalMemoryCursor9MetaClass5allocEv
 __ZNK21IOSubMemoryDescriptor12getMetaClassEv
+__ZNK21IOSubMemoryDescriptor19dmaCommandOperationEmPvj
 __ZNK21IOSubMemoryDescriptor9MetaClass5allocEv
 __ZNK21IOSubMemoryDescriptor9serializeEP11OSSerialize
 __ZNK22IOInterruptEventSource11getIntIndexEv
@@ -1880,6 +1958,7 @@ __ZNK24IOBufferMemoryDescriptor11getCapacityEv
 __ZNK24IOBufferMemoryDescriptor12getMetaClassEv
 __ZNK24IOBufferMemoryDescriptor9MetaClass5allocEv
 __ZNK25IOGeneralMemoryDescriptor12getMetaClassEv
+__ZNK25IOGeneralMemoryDescriptor19dmaCommandOperationEmPvj
 __ZNK25IOGeneralMemoryDescriptor9MetaClass5allocEv
 __ZNK25IOGeneralMemoryDescriptor9serializeEP11OSSerialize
 __ZNK25IOServiceUserNotification12getMetaClassEv
@@ -1894,6 +1973,7 @@ __ZNK28IOFilterInterruptEventSource9MetaClass5allocEv
 __ZNK32IOServiceMessageUserNotification12getMetaClassEv
 __ZNK32IOServiceMessageUserNotification9MetaClass5allocEv
 __ZNK8IOMapper12getMetaClassEv
+__ZNK8IOMapper13getBypassMaskEPy
 __ZNK8IOMapper9MetaClass5allocEv
 __ZNK8IOPMpriv12getMetaClassEv
 __ZNK8IOPMpriv9MetaClass5allocEv
@@ -1926,6 +2006,7 @@ __ZTV11IOCatalogue
 __ZTV11IODataQueue
 __ZTV11IOMemoryMap
 __ZTV11IOResources
+__ZTV12IODMACommand
 __ZTV12IOPMinformee
 __ZTV12IORootParent
 __ZTV12IOUserClient
@@ -1984,6 +2065,7 @@ __ZTVN11IOCatalogue9MetaClassE
 __ZTVN11IODataQueue9MetaClassE
 __ZTVN11IOMemoryMap9MetaClassE
 __ZTVN11IOResources9MetaClassE
+__ZTVN12IODMACommand9MetaClassE
 __ZTVN12IOPMinformee9MetaClassE
 __ZTVN12IORootParent9MetaClassE
 __ZTVN12IOUserClient9MetaClassE
diff --git a/config/IOKit.ppc.exports b/config/IOKit.ppc.exports
index e69de29bb..42d90d845 100644
--- a/config/IOKit.ppc.exports
+++ b/config/IOKit.ppc.exports
@@ -0,0 +1,39 @@
+__Z11IODBDMAStopPV23IODBDMAChannelRegisters
+__Z12IODBDMAFlushPV23IODBDMAChannelRegisters
+__Z12IODBDMAPausePV23IODBDMAChannelRegisters
+__Z12IODBDMAResetPV23IODBDMAChannelRegisters
+__Z12IODBDMAStartPV23IODBDMAChannelRegistersPV17IODBDMADescriptor
+__Z15IODBDMAContinuePV23IODBDMAChannelRegisters
+__ZN19IODBDMAMemoryCursor10gMetaClassE
+__ZN19IODBDMAMemoryCursor10superClassE
+__ZN19IODBDMAMemoryCursor17withSpecificationEmmm
+__ZN19IODBDMAMemoryCursor21initWithSpecificationEmmm
+__ZN19IODBDMAMemoryCursor9MetaClassC1Ev
+__ZN19IODBDMAMemoryCursor9MetaClassC2Ev
+__ZN19IODBDMAMemoryCursor9metaClassE
+__ZN19IODBDMAMemoryCursorC1EPK11OSMetaClass
+__ZN19IODBDMAMemoryCursorC1Ev
+__ZN19IODBDMAMemoryCursorC2EPK11OSMetaClass
+__ZN19IODBDMAMemoryCursorC2Ev
+__ZN19IODBDMAMemoryCursorD0Ev
+__ZN19IODBDMAMemoryCursorD2Ev
+__ZN9IOService20_RESERVEDIOService48Ev
+__ZN9IOService20_RESERVEDIOService49Ev
+__ZN9IOService20_RESERVEDIOService50Ev
+__ZN9IOService20_RESERVEDIOService51Ev
+__ZN9IOService20_RESERVEDIOService52Ev
+__ZN9IOService20_RESERVEDIOService53Ev
+__ZN9IOService20_RESERVEDIOService54Ev
+__ZN9IOService20_RESERVEDIOService55Ev
+__ZN9IOService20_RESERVEDIOService56Ev
+__ZN9IOService20_RESERVEDIOService57Ev
+__ZN9IOService20_RESERVEDIOService58Ev
+__ZN9IOService20_RESERVEDIOService59Ev
+__ZN9IOService20_RESERVEDIOService60Ev
+__ZN9IOService20_RESERVEDIOService61Ev
+__ZN9IOService20_RESERVEDIOService62Ev
+__ZN9IOService20_RESERVEDIOService63Ev
+__ZNK19IODBDMAMemoryCursor12getMetaClassEv
+__ZNK19IODBDMAMemoryCursor9MetaClass5allocEv
+__ZTV19IODBDMAMemoryCursor
+__ZTVN19IODBDMAMemoryCursor9MetaClassE
diff --git a/config/Libkern.exports b/config/Libkern.exports
index eeb7bfb60..1fad54e93 100644
--- a/config/Libkern.exports
+++ b/config/Libkern.exports
@@ -16,8 +16,6 @@ _OSCompareAndSwap
 _OSDecrementAtomic
 _OSDecrementAtomic16
 _OSDecrementAtomic8
-_OSDequeueAtomic
-_OSEnqueueAtomic
 _OSFree
 _OSIncrementAtomic
 _OSIncrementAtomic16
@@ -417,22 +415,6 @@ __ZN8OSObject19_RESERVEDOSObject12Ev
 __ZN8OSObject19_RESERVEDOSObject13Ev
 __ZN8OSObject19_RESERVEDOSObject14Ev
 __ZN8OSObject19_RESERVEDOSObject15Ev
-__ZN8OSObject19_RESERVEDOSObject16Ev
-__ZN8OSObject19_RESERVEDOSObject17Ev
-__ZN8OSObject19_RESERVEDOSObject18Ev
-__ZN8OSObject19_RESERVEDOSObject19Ev
-__ZN8OSObject19_RESERVEDOSObject20Ev
-__ZN8OSObject19_RESERVEDOSObject21Ev
-__ZN8OSObject19_RESERVEDOSObject22Ev
-__ZN8OSObject19_RESERVEDOSObject23Ev
-__ZN8OSObject19_RESERVEDOSObject24Ev
-__ZN8OSObject19_RESERVEDOSObject25Ev
-__ZN8OSObject19_RESERVEDOSObject26Ev
-__ZN8OSObject19_RESERVEDOSObject27Ev
-__ZN8OSObject19_RESERVEDOSObject28Ev
-__ZN8OSObject19_RESERVEDOSObject29Ev
-__ZN8OSObject19_RESERVEDOSObject30Ev
-__ZN8OSObject19_RESERVEDOSObject31Ev
 __ZN8OSObject4freeEv
 __ZN8OSObject4initEv
 __ZN8OSObject9MetaClassC1Ev
diff --git a/config/Libkern.ppc.exports b/config/Libkern.ppc.exports
index 9b1bdcf7a..df175fdcc 100644
--- a/config/Libkern.ppc.exports
+++ b/config/Libkern.ppc.exports
@@ -1,2 +1,21 @@
+_OSDequeueAtomic
+_OSEnqueueAtomic
+__ZN8OSObject19_RESERVEDOSObject16Ev
+__ZN8OSObject19_RESERVEDOSObject17Ev
+__ZN8OSObject19_RESERVEDOSObject18Ev
+__ZN8OSObject19_RESERVEDOSObject19Ev
+__ZN8OSObject19_RESERVEDOSObject20Ev
+__ZN8OSObject19_RESERVEDOSObject21Ev
+__ZN8OSObject19_RESERVEDOSObject22Ev
+__ZN8OSObject19_RESERVEDOSObject23Ev
+__ZN8OSObject19_RESERVEDOSObject24Ev
+__ZN8OSObject19_RESERVEDOSObject25Ev
+__ZN8OSObject19_RESERVEDOSObject26Ev
+__ZN8OSObject19_RESERVEDOSObject27Ev
+__ZN8OSObject19_RESERVEDOSObject28Ev
+__ZN8OSObject19_RESERVEDOSObject29Ev
+__ZN8OSObject19_RESERVEDOSObject30Ev
+__ZN8OSObject19_RESERVEDOSObject31Ev
 _bcopy_nc
 _bzero_nc
+
diff --git a/config/MasterVersion b/config/MasterVersion
index 021ccaef0..92104ddb5 100644
--- a/config/MasterVersion
+++ b/config/MasterVersion
@@ -1,4 +1,4 @@
-8.10.0
+8.10.1
 
 # The first line of this file contains the master version number for the kernel.
 # All other instances of the kernel version in xnu are derived from this file.
diff --git a/config/System6.0.exports b/config/System6.0.exports
index 1815d9dfb..a3ad829d3 100644
--- a/config/System6.0.exports
+++ b/config/System6.0.exports
@@ -128,8 +128,6 @@ _OSCompareAndSwap
 _OSDecrementAtomic
 _OSDecrementAtomic16
 _OSDecrementAtomic8
-_OSDequeueAtomic
-_OSEnqueueAtomic
 _OSIncrementAtomic
 _OSIncrementAtomic16
 _OSIncrementAtomic8
@@ -250,9 +248,9 @@ __ZN10IOWorkLoop10wakeupGateEPvb
 __ZN10IOWorkLoop12tryCloseGateEv
 __ZN10IOWorkLoop13_maintRequestEPvS0_S0_S0_
 __ZN10IOWorkLoop14addEventSourceEP13IOEventSource
+__ZN10IOWorkLoop15runEventSourcesEv
 __ZN10IOWorkLoop17removeEventSourceEP13IOEventSource
 __ZN10IOWorkLoop19signalWorkAvailableEv
-__ZN10IOWorkLoop20_RESERVEDIOWorkLoop1Ev
 __ZN10IOWorkLoop20_RESERVEDIOWorkLoop2Ev
 __ZN10IOWorkLoop20_RESERVEDIOWorkLoop3Ev
 __ZN10IOWorkLoop20_RESERVEDIOWorkLoop4Ev
@@ -495,6 +493,8 @@ __ZN12IOUserClient26removeMappingForDescriptorEP18IOMemoryDescriptor
 __ZN12IOUserClient30getExternalAsyncMethodForIndexEm
 __ZN12IOUserClient31getAsyncTargetAndMethodForIndexEPP9IOServicem
 __ZN12IOUserClient4freeEv
+__ZN12IOUserClient4initEP12OSDictionary
+__ZN12IOUserClient4initEv
 __ZN12IOUserClient9MetaClassC1Ev
 __ZN12IOUserClient9MetaClassC2Ev
 __ZN12IOUserClient9metaClassE
@@ -654,6 +654,7 @@ __ZN13IOCommandGate10gMetaClassE
 __ZN13IOCommandGate10runCommandEPvS0_S0_S0_
 __ZN13IOCommandGate10superClassE
 __ZN13IOCommandGate11commandGateEP8OSObjectPFiS1_PvS2_S2_S2_E
+__ZN13IOCommandGate11setWorkLoopEP10IOWorkLoop
 __ZN13IOCommandGate12checkForWorkEv
 __ZN13IOCommandGate12commandSleepEPvm
 __ZN13IOCommandGate13attemptActionEPFiP8OSObjectPvS2_S2_S2_ES2_S2_S2_S2_
@@ -667,7 +668,10 @@ __ZN13IOCommandGate23_RESERVEDIOCommandGate4Ev
 __ZN13IOCommandGate23_RESERVEDIOCommandGate5Ev
 __ZN13IOCommandGate23_RESERVEDIOCommandGate6Ev
 __ZN13IOCommandGate23_RESERVEDIOCommandGate7Ev
+__ZN13IOCommandGate4freeEv
 __ZN13IOCommandGate4initEP8OSObjectPFiS1_PvS2_S2_S2_E
+__ZN13IOCommandGate6enableEv
+__ZN13IOCommandGate7disableEv
 __ZN13IOCommandGate9MetaClassC1Ev
 __ZN13IOCommandGate9MetaClassC2Ev
 __ZN13IOCommandGate9metaClassE
@@ -815,7 +819,6 @@ __ZN14IOPMrootDomain25announcePowerSourceChangeEv
 __ZN14IOPMrootDomain26handleSleepTimerExpirationEv
 __ZN14IOPMrootDomain26restoreUserSpinDownTimeoutEv
 __ZN14IOPMrootDomain27displayWranglerNotificationEPvS0_mP9IOServiceS0_j
-__ZN14IOPMrootDomain27registerPMSettingControllerEPFiiiPvES0_
 __ZN14IOPMrootDomain39stopIgnoringClamshellEventsDuringWakeupEv
 __ZN14IOPMrootDomain5startEP9IOService
 __ZN14IOPMrootDomain9MetaClassC1Ev
@@ -848,30 +851,6 @@ __ZN15IOConditionLockC2EPK11OSMetaClass
 __ZN15IOConditionLockC2Ev
 __ZN15IOConditionLockD0Ev
 __ZN15IOConditionLockD2Ev
-__ZN15IOPMPowerSource10gMetaClassE
-__ZN15IOPMPowerSource10isChargingEv
-__ZN15IOPMPowerSource10superClassE
-__ZN15IOPMPowerSource11acConnectedEv
-__ZN15IOPMPowerSource11atWarnLevelEv
-__ZN15IOPMPowerSource11curCapacityEv
-__ZN15IOPMPowerSource11isInstalledEv
-__ZN15IOPMPowerSource11maxCapacityEv
-__ZN15IOPMPowerSource12currentDrawnEv
-__ZN15IOPMPowerSource12updateStatusEv
-__ZN15IOPMPowerSource13timeRemainingEv
-__ZN15IOPMPowerSource24capacityPercentRemainingEv
-__ZN15IOPMPowerSource4initEt
-__ZN15IOPMPowerSource7voltageEv
-__ZN15IOPMPowerSource8depletedEv
-__ZN15IOPMPowerSource9MetaClassC1Ev
-__ZN15IOPMPowerSource9MetaClassC2Ev
-__ZN15IOPMPowerSource9metaClassE
-__ZN15IOPMPowerSourceC1EPK11OSMetaClass
-__ZN15IOPMPowerSourceC1Ev
-__ZN15IOPMPowerSourceC2EPK11OSMetaClass
-__ZN15IOPMPowerSourceC2Ev
-__ZN15IOPMPowerSourceD0Ev
-__ZN15IOPMPowerSourceD2Ev
 __ZN15IOPanicPlatform10gMetaClassE
 __ZN15IOPanicPlatform10superClassE
 __ZN15IOPanicPlatform5startEP9IOService
@@ -1258,7 +1237,6 @@ __ZN18IOMemoryDescriptor18getPhysicalAddressEv
 __ZN18IOMemoryDescriptor18withPhysicalRangesEP15IOPhysicalRangem11IODirectionb
 __ZN18IOMemoryDescriptor19withPhysicalAddressEmm11IODirection
 __ZN18IOMemoryDescriptor20getPhysicalSegment64EmPm
-__ZN18IOMemoryDescriptor28_RESERVEDIOMemoryDescriptor5Ev
 __ZN18IOMemoryDescriptor28_RESERVEDIOMemoryDescriptor6Ev
 __ZN18IOMemoryDescriptor28_RESERVEDIOMemoryDescriptor7Ev
 __ZN18IOMemoryDescriptor28_RESERVEDIOMemoryDescriptor8Ev
@@ -1539,6 +1517,7 @@ __ZN21IOSubMemoryDescriptor16getSourceSegmentEmPm
 __ZN21IOSubMemoryDescriptor16performOperationEmmm
 __ZN21IOSubMemoryDescriptor17getVirtualSegmentEmPm
 __ZN21IOSubMemoryDescriptor18getPhysicalSegmentEmPm
+__ZN21IOSubMemoryDescriptor20getPhysicalSegment64EmPm
 __ZN21IOSubMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb
 __ZN21IOSubMemoryDescriptor23initWithPhysicalAddressEmm11IODirection
 __ZN21IOSubMemoryDescriptor4freeEv
@@ -1658,9 +1637,10 @@ __ZN24IOBufferMemoryDescriptor15initWithAddressEjm11IODirectionP4task
 __ZN24IOBufferMemoryDescriptor15initWithOptionsEmjj
 __ZN24IOBufferMemoryDescriptor15initWithOptionsEmjjP4task
 __ZN24IOBufferMemoryDescriptor17inTaskWithOptionsEP4taskmjj
+__ZN24IOBufferMemoryDescriptor20initWithPhysicalMaskEP4taskmyyy
+__ZN24IOBufferMemoryDescriptor22inTaskWithPhysicalMaskEP4taskmyy
 __ZN24IOBufferMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb
 __ZN24IOBufferMemoryDescriptor23initWithPhysicalAddressEmm11IODirection
-__ZN24IOBufferMemoryDescriptor34_RESERVEDIOBufferMemoryDescriptor1Ev
 __ZN24IOBufferMemoryDescriptor34_RESERVEDIOBufferMemoryDescriptor2Ev
 __ZN24IOBufferMemoryDescriptor34_RESERVEDIOBufferMemoryDescriptor3Ev
 __ZN24IOBufferMemoryDescriptor34_RESERVEDIOBufferMemoryDescriptor4Ev
@@ -1727,6 +1707,7 @@ __ZN25IOGeneralMemoryDescriptor15unmapFromKernelEv
 __ZN25IOGeneralMemoryDescriptor16getSourceSegmentEmPm
 __ZN25IOGeneralMemoryDescriptor17getVirtualSegmentEmPm
 __ZN25IOGeneralMemoryDescriptor18getPhysicalSegmentEmPm
+__ZN25IOGeneralMemoryDescriptor20getPhysicalSegment64EmPm
 __ZN25IOGeneralMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb
 __ZN25IOGeneralMemoryDescriptor23initWithPhysicalAddressEmm11IODirection
 __ZN25IOGeneralMemoryDescriptor4freeEv
@@ -1950,8 +1931,8 @@ __ZN7OSArray12removeObjectEj
 __ZN7OSArray12withCapacityEj
 __ZN7OSArray13initWithArrayEPKS_j
 __ZN7OSArray13replaceObjectEjPK15OSMetaClassBase
-__ZN7OSArray14ensureCapacityEj
 __ZN7OSArray14copyCollectionEP12OSDictionary
+__ZN7OSArray14ensureCapacityEj
 __ZN7OSArray15flushCollectionEv
 __ZN7OSArray15initWithObjectsEPPK8OSObjectjj
 __ZN7OSArray16initWithCapacityEj
@@ -1986,7 +1967,6 @@ __ZN8IOMapper10superClassE
 __ZN8IOMapper11NewARTTableEmPPvPj
 __ZN8IOMapper12FreeARTTableEP6OSDatam
 __ZN8IOMapper17setMapperRequiredEb
-__ZN8IOMapper18_RESERVEDIOMapper0Ev
 __ZN8IOMapper18_RESERVEDIOMapper1Ev
 __ZN8IOMapper18_RESERVEDIOMapper2Ev
 __ZN8IOMapper18_RESERVEDIOMapper3Ev
@@ -2097,22 +2077,6 @@ __ZN8OSObject19_RESERVEDOSObject12Ev
 __ZN8OSObject19_RESERVEDOSObject13Ev
 __ZN8OSObject19_RESERVEDOSObject14Ev
 __ZN8OSObject19_RESERVEDOSObject15Ev
-__ZN8OSObject19_RESERVEDOSObject16Ev
-__ZN8OSObject19_RESERVEDOSObject17Ev
-__ZN8OSObject19_RESERVEDOSObject18Ev
-__ZN8OSObject19_RESERVEDOSObject19Ev
-__ZN8OSObject19_RESERVEDOSObject20Ev
-__ZN8OSObject19_RESERVEDOSObject21Ev
-__ZN8OSObject19_RESERVEDOSObject22Ev
-__ZN8OSObject19_RESERVEDOSObject23Ev
-__ZN8OSObject19_RESERVEDOSObject24Ev
-__ZN8OSObject19_RESERVEDOSObject25Ev
-__ZN8OSObject19_RESERVEDOSObject26Ev
-__ZN8OSObject19_RESERVEDOSObject27Ev
-__ZN8OSObject19_RESERVEDOSObject28Ev
-__ZN8OSObject19_RESERVEDOSObject29Ev
-__ZN8OSObject19_RESERVEDOSObject30Ev
-__ZN8OSObject19_RESERVEDOSObject31Ev
 __ZN8OSObject4freeEv
 __ZN8OSObject4initEv
 __ZN8OSObject9MetaClassC1Ev
@@ -2311,6 +2275,7 @@ __ZN9IOService15getDeviceMemoryEv
 __ZN9IOService15getPMRootDomainEv
 __ZN9IOService15instruct_driverEm
 __ZN9IOService15lookupInterruptEibPP21IOInterruptController
+__ZN9IOService15nextIdleTimeoutE12UnsignedWideS0_j
 __ZN9IOService15powerChangeDoneEm
 __ZN9IOService15probeCandidatesEP12OSOrderedSet
 __ZN9IOService15publishResourceEPK8OSSymbolP8OSObject
@@ -2366,7 +2331,6 @@ __ZN9IOService18matchPropertyTableEP12OSDictionary
 __ZN9IOService18matchPropertyTableEP12OSDictionaryPl
 __ZN9IOService18setIdleTimerPeriodEm
 __ZN9IOService18settleTimerExpiredEv
-__ZN9IOService15nextIdleTimeoutE12UnsignedWideS0_j
 __ZN9IOService19_RESERVEDIOService4Ev
 __ZN9IOService19_RESERVEDIOService5Ev
 __ZN9IOService19_RESERVEDIOService6Ev
@@ -2423,22 +2387,6 @@ __ZN9IOService20_RESERVEDIOService44Ev
 __ZN9IOService20_RESERVEDIOService45Ev
 __ZN9IOService20_RESERVEDIOService46Ev
 __ZN9IOService20_RESERVEDIOService47Ev
-__ZN9IOService20_RESERVEDIOService48Ev
-__ZN9IOService20_RESERVEDIOService49Ev
-__ZN9IOService20_RESERVEDIOService50Ev
-__ZN9IOService20_RESERVEDIOService51Ev
-__ZN9IOService20_RESERVEDIOService52Ev
-__ZN9IOService20_RESERVEDIOService53Ev
-__ZN9IOService20_RESERVEDIOService54Ev
-__ZN9IOService20_RESERVEDIOService55Ev
-__ZN9IOService20_RESERVEDIOService56Ev
-__ZN9IOService20_RESERVEDIOService57Ev
-__ZN9IOService20_RESERVEDIOService58Ev
-__ZN9IOService20_RESERVEDIOService59Ev
-__ZN9IOService20_RESERVEDIOService60Ev
-__ZN9IOService20_RESERVEDIOService61Ev
-__ZN9IOService20_RESERVEDIOService62Ev
-__ZN9IOService20_RESERVEDIOService63Ev
 __ZN9IOService20callPlatformFunctionEPK8OSSymbolbPvS3_S3_S3_
 __ZN9IOService20callPlatformFunctionEPKcbPvS2_S2_S2_
 __ZN9IOService20getDeviceMemoryCountEv
@@ -2754,6 +2702,7 @@ __ZNK18IODTPlatformExpert14compareNubNameEPK9IOServiceP8OSStringPS4_
 __ZNK18IODTPlatformExpert9MetaClass5allocEv
 __ZNK18IOMemoryDescriptor12getDirectionEv
 __ZNK18IOMemoryDescriptor12getMetaClassEv
+__ZNK18IOMemoryDescriptor19dmaCommandOperationEmPvj
 __ZNK18IOMemoryDescriptor9MetaClass5allocEv
 __ZNK18IOMemoryDescriptor9getLengthEv
 __ZNK18IOPMchangeNoteList12getMetaClassEv
@@ -2781,6 +2730,7 @@ __ZNK21IOInterruptController9MetaClass5allocEv
 __ZNK21IONaturalMemoryCursor12getMetaClassEv
 __ZNK21IONaturalMemoryCursor9MetaClass5allocEv
 __ZNK21IOSubMemoryDescriptor12getMetaClassEv
+__ZNK21IOSubMemoryDescriptor19dmaCommandOperationEmPvj
 __ZNK21IOSubMemoryDescriptor9MetaClass5allocEv
 __ZNK21IOSubMemoryDescriptor9serializeEP11OSSerialize
 __ZNK22IOInterruptEventSource11getIntIndexEv
@@ -2802,6 +2752,7 @@ __ZNK24IOBufferMemoryDescriptor9MetaClass5allocEv
 __ZNK24IOCPUInterruptController12getMetaClassEv
 __ZNK24IOCPUInterruptController9MetaClass5allocEv
 __ZNK25IOGeneralMemoryDescriptor12getMetaClassEv
+__ZNK25IOGeneralMemoryDescriptor19dmaCommandOperationEmPvj
 __ZNK25IOGeneralMemoryDescriptor9MetaClass5allocEv
 __ZNK25IOGeneralMemoryDescriptor9serializeEP11OSSerialize
 __ZNK25IOServiceUserNotification12getMetaClassEv
@@ -2860,6 +2811,7 @@ __ZNK7OSArray9isEqualToEPK15OSMetaClassBase
 __ZNK7OSArray9isEqualToEPKS_
 __ZNK7OSArray9serializeEP11OSSerialize
 __ZNK8IOMapper12getMetaClassEv
+__ZNK8IOMapper13getBypassMaskEPy
 __ZNK8IOMapper9MetaClass5allocEv
 __ZNK8IOPMpriv12getMetaClassEv
 __ZNK8IOPMpriv9MetaClass5allocEv
@@ -3474,6 +3426,11 @@ _pexpert_version_minor:_version_minor
 _pexpert_version_variant:_version_variant
 _pmap_extract
 _pmap_find_phys
+_pmsBuild
+_pmsPark
+_pmsRun
+_pmsRunLocal
+_pmsStart
 _print_vmpage_stat
 _printf
 _processor_exit
diff --git a/config/System6.0.i386.exports b/config/System6.0.i386.exports
index 5b8690450..b6968b6d1 100644
--- a/config/System6.0.i386.exports
+++ b/config/System6.0.i386.exports
@@ -1,332 +1,21 @@
-_Gdt
-_Load_context
-_PE_incoming_interrupt
+_Cstate_table_set
 _PE_install_interrupt_handler
 _PE_interrupt_handler
-_Thread_continue
-__ZN15AppleIntelClock10gMetaClassE
-__ZN15AppleIntelClock10superClassE
-__ZN15AppleIntelClock5startEP9IOService
-__ZN15AppleIntelClock9MetaClassC1Ev
-__ZN15AppleIntelClock9MetaClassC2Ev
-__ZN15AppleIntelClock9metaClassE
-__ZN15AppleIntelClockC1EPK11OSMetaClass
-__ZN15AppleIntelClockC1Ev
-__ZN15AppleIntelClockC2EPK11OSMetaClass
-__ZN15AppleIntelClockC2Ev
-__ZN15AppleIntelClockD0Ev
-__ZN15AppleIntelClockD2Ev
-__ZNK15AppleIntelClock12getMetaClassEv
-__ZNK15AppleIntelClock9MetaClass5allocEv
-__ZTV15AppleIntelClock
-__ZTVN15AppleIntelClock9MetaClassE
-___divsi3
-___udivsi3
-__clts
-__fldcw
-__fnclex
-__fninit
-__fnstsw
-__fprestore
-__fpsave
-__fstcw
-__mp_disable_preemption
-__mp_enable_preemption
-__mp_enable_preemption_no_check
-__setts
-_a_dbl_fault
-_a_fpu_over
-_a_inv_tss
-_acc_type
 _acpi_install_wake_handler
 _acpi_sleep_kernel
-_act_machine_return
-_all_intrs
-_alltraps
-_avail_end
-_avail_start
-_bbc_config
-_bbc_gettime
-_bbc_settime
-_bcopy16
-_bcopy_no_overwrite
-_bit_lock
-_bit_lock_try
-_bit_unlock
-_blkclr
-_boot_args_start
-_check_io_fault
-_clear_kdb_intr
-_cli_count
-_collect_ref
-_collect_unref
-_copyp2p
-_copypv
-_cpu_idle_handler
-_cpu_interrupt
 _cpu_number
-_cpu_shutdown
 _cpu_to_lapic
-_cpu_vendors
-_cpudata_desc_pattern
-_cpuid_cpu_display
-_cpuid_family
-_cpuid_feature
-_cpuid_feature_display
 _cpuid_features
-_cpuid_get_feature_names
-_cpuid_get_info
 _cpuid_info
-_cpus_active
-_cpus_idle
-_createdt
-_dectohexdec
-_dev_indirect_count
-_dev_indirect_list
-_div_scale
-_dr0
-_dr1
-_dr2
-_dr3
-_dr6
-_dr_addr
-_dump_act
-_dump_regs
-_eintstack
-_emulate_io
-_fakePPCBootArgs
-_fakePPCDeviceTree
-_fc_get
-_fix_desc
-_flush_tlb
-_fp_free
-_fp_kind
-_fp_load
-_fp_save
-_fp_state_alloc
-_fpexterrflt
-_fpextovrflt
-_fpflush
-_fpinit
-_fpintr
-_fpnoextflt
-_fpu_get_fxstate
-_fpu_get_state
-_fpu_module_init
-_fpu_set_fxstate
-_fpu_set_state
-_gDriversProp
-_gMemoryMapNode
-_gdt
-_gdtptr
-_gdtr
-_get_cr0
-_get_cr2
-_get_cr3
-_get_cr4
-_get_ldt
-_get_pc
-_get_tr
-_hexdectodec
-_htonl
-_htons
-_i386_astintr
-_i386_exception
-_i386_init
-_i386_preinit
-_i386_signal_cpu
-_i386_signal_cpus
-_i386_vm_init
-_i_bit_clear
-_i_bit_set
-_idt
-_idtptr
-_ifps_zone
-_indent
-_init_fpu
-_insb
-_insl
-_inst_fetch
-_insw
-_intel_read_fault
-_intel_startCPU
-_inuse_ptepages_count
-_iopb_destroy
-_iopb_init
-_jail
-_kd_slmscd
-_kd_slmscu
-_kd_slmwd
-_kdp_copy_kmem
-_kdp_getstate
-_kdp_i386_backtrace
-_kdp_i386_trap
-_kdp_setstate
 _kdreboot
-_kernel_preempt_check
-_kernel_trap
-_ktss
 _lapic_end_of_interrupt
 _lapic_smm_restore
-_last_addr
-_ldt
-_ldt_desc_pattern
-_linb
-_linl
-_linw
-_locore_end
-_loutb
-_loutl
-_loutw
-_mach25_syscall
-_mach_rpc
-_machdep_call_count
-_machdep_call_table
-_machdep_syscall
-_master_cpu
-_master_is_up
-_master_processor
-_master_up
-_minsecurity
 _ml_get_max_cpus
-_mp_boot_pde
-_mp_kdp_enter
-_mp_kdp_exit
-_mp_kdp_lock
-_mp_kdp_ncpus
-_mp_kdp_trap
+_mp_rendezvous_no_intrs
 _mtrr_range_add
 _mtrr_range_remove
-_mul_scale
-_nptr
-_ntohl
-_ntohs
-_outsb
-_outsl
-_outsw
-_panic_trap
-_phys_attribute_clear
-_phys_attribute_set
-_phys_attribute_test
-_pmap_alloc_chunk
-_pmap_cache_count
-_pmap_cache_list
-_pmap_cache_lock
-_pmap_cache_max
-_pmap_copy_part_lpage
-_pmap_copy_part_rpage
-_pmap_debug
-_pmap_expand
-_pmap_map_bd
-_pmap_movepage
-_pmap_phys_attributes
-_pmap_pte
-_pmap_remove_range
-_pmap_set_modify
-_pmap_system_lock
-_pmap_update_interrupt
-_pmap_valid_page
-_printdt
-_process_pmap_updates
-_pstart
-_pv_free_list
-_pv_free_list_lock
-_pv_head_table
-_pv_list_zone
-_pv_lock_table
-_real_to_prot
-_recover_table
-_recover_table_end
-_remote_kdb
-_reset_mem_on_reboot
-_retry_table
-_retry_table_end
-_return_to_iret
+_pmsCPUSetPStateLimit
+_pmsCPULoadVIDTable
 _rtc_clock_stepped
 _rtc_clock_stepping
-_rtc_cyc_per_sec
-_rtcget
-_rtclock
-_rtcput
-_sectOBJCB
-_sectSizeOBJC
-_serial_getc
-_serial_init
-_set_cpu_model
-_set_cr0
-_set_cr3
-_set_cr4
-_set_kbd_leds
-_set_ldt
-_set_tr
-_signal_cpus
-_slave_boot_base
-_slave_boot_end
-_slave_boot_init
-_slave_pstart
-_slave_start
-_smp_init
 _smp_initialized
-_start_lock
-_startprog
-_sti_count
-_t_bounds
-_t_debug
-_t_fpu_err
-_t_gen_prot
-_t_int3
-_t_into
-_t_invop
-_t_nofpu
-_t_page_fault
-_t_preempt
-_t_segnp
-_t_stack_fault
-_t_trap_0f
-_t_trap_11
-_t_trap_12
-_t_trap_13
-_t_trap_14
-_t_trap_15
-_t_trap_16
-_t_trap_17
-_t_trap_18
-_t_trap_19
-_t_trap_1a
-_t_trap_1b
-_t_trap_1c
-_t_trap_1d
-_t_trap_1e
-_t_trap_1f
-_t_zero_div
-_tc_clear_screen
-_tc_enable
-_tc_hide_cursor
-_tc_initialize
-_tc_paint_char
-_tc_scroll_down
-_tc_scroll_up
-_tc_show_cursor
-_tc_update_color
-_thread_bind
-_thread_compose_cthread_desc
-_thread_fast_set_cthread_self
-_thread_get_cthread_self
-_thread_set_cthread_self
-_trap_mach25_syscall
-_trap_machdep_syscall
-_trap_unix_syscall
-_tss_desc_pattern
-_user_ldt_free
-_user_page_fault_continue
-_user_trap
-_v86_assist
-_v86_assist_on
-_v86_do_sti_cli
-_v86_do_sti_immediate
-_v86_unsafe_ok
-_virtual_avail
-_virtual_end
-_vm_first_phys
-_vm_last_phys
-_yeartoday
diff --git a/config/System6.0.ppc.exports b/config/System6.0.ppc.exports
index 16e230227..3b367efcc 100644
--- a/config/System6.0.ppc.exports
+++ b/config/System6.0.ppc.exports
@@ -6,6 +6,8 @@ _PE_read_write_time_of_day
 _PE_write_IIC
 _PPCcalls
 _ResetHandler
+_OSEnqueueAtomic
+_OSDequeueAtomic
 __Z11IODBDMAStopPV23IODBDMAChannelRegisters
 __Z12IODBDMAFlushPV23IODBDMAChannelRegisters
 __Z12IODBDMAPausePV23IODBDMAChannelRegisters
@@ -149,6 +151,38 @@ __ZN8AppleNMIC2EPK11OSMetaClass
 __ZN8AppleNMIC2Ev
 __ZN8AppleNMID0Ev
 __ZN8AppleNMID2Ev
+__ZN8OSObject19_RESERVEDOSObject16Ev
+__ZN8OSObject19_RESERVEDOSObject17Ev
+__ZN8OSObject19_RESERVEDOSObject18Ev
+__ZN8OSObject19_RESERVEDOSObject19Ev
+__ZN8OSObject19_RESERVEDOSObject20Ev
+__ZN8OSObject19_RESERVEDOSObject21Ev
+__ZN8OSObject19_RESERVEDOSObject22Ev
+__ZN8OSObject19_RESERVEDOSObject23Ev
+__ZN8OSObject19_RESERVEDOSObject24Ev
+__ZN8OSObject19_RESERVEDOSObject25Ev
+__ZN8OSObject19_RESERVEDOSObject26Ev
+__ZN8OSObject19_RESERVEDOSObject27Ev
+__ZN8OSObject19_RESERVEDOSObject28Ev
+__ZN8OSObject19_RESERVEDOSObject29Ev
+__ZN8OSObject19_RESERVEDOSObject30Ev
+__ZN8OSObject19_RESERVEDOSObject31Ev
+__ZN9IOService20_RESERVEDIOService48Ev
+__ZN9IOService20_RESERVEDIOService49Ev
+__ZN9IOService20_RESERVEDIOService50Ev
+__ZN9IOService20_RESERVEDIOService51Ev
+__ZN9IOService20_RESERVEDIOService52Ev
+__ZN9IOService20_RESERVEDIOService53Ev
+__ZN9IOService20_RESERVEDIOService54Ev
+__ZN9IOService20_RESERVEDIOService55Ev
+__ZN9IOService20_RESERVEDIOService56Ev
+__ZN9IOService20_RESERVEDIOService57Ev
+__ZN9IOService20_RESERVEDIOService58Ev
+__ZN9IOService20_RESERVEDIOService59Ev
+__ZN9IOService20_RESERVEDIOService60Ev
+__ZN9IOService20_RESERVEDIOService61Ev
+__ZN9IOService20_RESERVEDIOService62Ev
+__ZN9IOService20_RESERVEDIOService63Ev
 __ZNK10AppleMacIO12getMetaClassEv
 __ZNK10AppleMacIO14compareNubNameEPK9IOServiceP8OSStringPS4_
 __ZNK10AppleMacIO9MetaClass5allocEv
@@ -208,8 +242,3 @@ _ml_mem_backoff
 _pe_do_clock_test
 _pe_run_clock_test
 _scc
-_pmsStart
-_pmsPark
-_pmsRun
-_pmsRunLocal
-_pmsBuild
diff --git a/config/Unsupported.exports b/config/Unsupported.exports
index 816e4e058..06b4312eb 100644
--- a/config/Unsupported.exports
+++ b/config/Unsupported.exports
@@ -8,18 +8,39 @@ _KUNCUserNotificationDisplayNotice
 _MD5Final
 _MD5Init
 _MD5Update
+__ZN15IOWatchDogTimer10gMetaClassE
+__ZN15IOWatchDogTimer10superClassE
+__ZN15IOWatchDogTimer13setPropertiesEP8OSObject
+__ZN15IOWatchDogTimer25_RESERVEDIOWatchDogTimer0Ev
+__ZN15IOWatchDogTimer25_RESERVEDIOWatchDogTimer1Ev
+__ZN15IOWatchDogTimer25_RESERVEDIOWatchDogTimer2Ev
+__ZN15IOWatchDogTimer25_RESERVEDIOWatchDogTimer3Ev
+__ZN15IOWatchDogTimer4stopEP9IOService
+__ZN15IOWatchDogTimer5startEP9IOService
+__ZN15IOWatchDogTimer9MetaClassC1Ev
+__ZN15IOWatchDogTimer9MetaClassC2Ev
+__ZN15IOWatchDogTimer9metaClassE
+__ZN15IOWatchDogTimerC1EPK11OSMetaClass
+__ZN15IOWatchDogTimerC2EPK11OSMetaClass
+__ZN15IOWatchDogTimerD0Ev
+__ZN15IOWatchDogTimerD2Ev
 __ZN16IOPlatformDevice10gMetaClassE
 __ZN16IOPlatformDevice26_RESERVEDIOPlatformDevice0Ev
 __ZN16IOPlatformDevice26_RESERVEDIOPlatformDevice1Ev
 __ZN16IOPlatformDevice26_RESERVEDIOPlatformDevice2Ev
 __ZN16IOPlatformDevice26_RESERVEDIOPlatformDevice3Ev
-__ZN16IOPlatformDeviceC2EPK11OSMetaClass
 __ZN16IOPlatformDevice9metaClassE
+__ZN16IOPlatformDeviceC2EPK11OSMetaClass
 __ZN16IOPlatformDeviceD2Ev
 __ZN18IODTPlatformExpert9metaClassE
+__ZNK15IOWatchDogTimer12getMetaClassEv
+__ZNK15IOWatchDogTimer9MetaClass5allocEv
+__ZTV15IOWatchDogTimer
 __ZTV16IOPlatformDevice
+__ZTVN15IOWatchDogTimer9MetaClassE
 __doprnt
 _aes_decrypt_cbc
+_aes_decrypt_key
 _aes_decrypt_key128
 _aes_encrypt_cbc
 _aes_encrypt_key128
@@ -130,6 +151,8 @@ _m_pullup
 _m_split
 _m_trailingspace
 _mach_make_memory_entry_64
+_mach_memory_entry_page_op
+_mach_memory_entry_range_op
 _max_mem
 _mcl_to_paddr
 _mem_size
@@ -203,6 +226,7 @@ _soconnect
 _socreate
 _sodisconnect
 _sofree
+_sofreelastref
 _soisconnected
 _soisconnecting
 _soisdisconnected
@@ -220,6 +244,8 @@ _stack_privilege
 _task_resume
 _task_suspend
 _tcbinfo
+_temp_patch_ptrace
+_temp_unpatch_ptrace
 _termioschars
 _thread_call_func
 _thread_call_func_cancel
@@ -259,8 +285,8 @@ _vm_map_wire
 _vm_protect
 _vm_region
 _vm_region_object_create
-_vnode_getparent
 _vnode_getname
+_vnode_getparent
 _vnode_putname
 _vnode_tag
 _vnode_update_identity
diff --git a/config/Unsupported.i386.exports b/config/Unsupported.i386.exports
index 72a1f46cf..a34b29222 100644
--- a/config/Unsupported.i386.exports
+++ b/config/Unsupported.i386.exports
@@ -1,8 +1,20 @@
-_copypv
-_cpu_number
-_master_cpu
-_master_processor
-_ml_get_max_cpus
-_mtrr_range_add
-_mtrr_range_remove
-_thread_bind
+_cpu_data_ptr
+_dsmos_page_transform_hook
+_gPEEFISystemTable
+_hpet_get_info
+_io_map_spec
+_lapic_start
+_ml_get_apicid
+_ml_get_maxbusdelay
+_ml_get_maxsnoop
+_ml_hpet_cfg
+_mp_rendezvous
+_pmRegister
+_pm_init_lock
+_rdHPET
+_real_ncpus
+_rtc_clock_napped
+_tmrCvt
+_tsc_get_info
+_hibernate_vm_lock
+_hibernate_vm_unlock
diff --git a/config/Unsupported.ppc.exports b/config/Unsupported.ppc.exports
index 1f785c1ee..350464074 100644
--- a/config/Unsupported.ppc.exports
+++ b/config/Unsupported.ppc.exports
@@ -21,8 +21,6 @@ _ml_ppc_sleep
 _ml_set_processor_speed
 _ml_set_processor_voltage
 _ml_throttle
-_temp_patch_ptrace
-_temp_unpatch_ptrace
 _pmsStart
 _pmsPark
 _pmsRun
diff --git a/iokit/Drivers/platform/drvAppleNMI/AppleNMI.cpp b/iokit/Drivers/platform/drvAppleNMI/AppleNMI.cpp
index 575d49673..07da66d56 100644
--- a/iokit/Drivers/platform/drvAppleNMI/AppleNMI.cpp
+++ b/iokit/Drivers/platform/drvAppleNMI/AppleNMI.cpp
@@ -73,7 +73,9 @@ bool AppleNMI::start(IOService *provider)
   addNotification( gIOPublishNotification, serviceMatching("IOPMrootDomain"), (IOServiceNotificationHandler)RootRegistered, this, 0 );
 
   // Register the interrupt.
-  provider->registerInterrupt(0, this, (IOInterruptAction) &AppleNMI::handleInterrupt, 0);
+  IOInterruptAction handler = OSMemberFunctionCast(IOInterruptAction,
+	  				this, &AppleNMI::handleInterrupt);
+  provider->registerInterrupt(0, this, handler, 0);
   provider->enableInterrupt(0);
 
   return true;
@@ -125,24 +127,24 @@ IOReturn AppleNMI::powerStateWillChangeTo ( IOPMPowerFlags theFlags, unsigned lo
         {
             // Mask NMI and change from edge to level whilst sleeping (copied directly from OS9 code)
             nmiIntSourceAddr = (volatile unsigned long *)kExtInt9_NMIIntSource;
-            nmiIntSource = ml_phys_read(nmiIntSourceAddr);
+            nmiIntSource = ml_phys_read((vm_address_t)nmiIntSourceAddr);
             nmiIntSource |= kNMIIntLevelMask;
-            ml_phys_write(nmiIntSourceAddr, nmiIntSource);
+            ml_phys_write((vm_address_t)nmiIntSourceAddr, nmiIntSource);
             eieio();
             nmiIntSource |= kNMIIntMask;
-            ml_phys_write(nmiIntSourceAddr, nmiIntSource);
+            ml_phys_write((vm_address_t)nmiIntSourceAddr, nmiIntSource);
             eieio();
         }
         else
         {
             // Unmask NMI and change back to edge (copied directly from OS9 code)
             nmiIntSourceAddr = (volatile unsigned long *)kExtInt9_NMIIntSource;
-            nmiIntSource = ml_phys_read(nmiIntSourceAddr);
+            nmiIntSource = ml_phys_read((vm_address_t)nmiIntSourceAddr);
             nmiIntSource &= ~kNMIIntLevelMask;
-            ml_phys_write(nmiIntSourceAddr, nmiIntSource);
+            ml_phys_write((vm_address_t)nmiIntSourceAddr, nmiIntSource);
             eieio();
             nmiIntSource &= ~kNMIIntMask;
-            ml_phys_write(nmiIntSourceAddr, nmiIntSource);
+            ml_phys_write((vm_address_t)nmiIntSourceAddr, nmiIntSource);
             eieio();
         }
     }
diff --git a/iokit/IOKit/IOBufferMemoryDescriptor.h b/iokit/IOKit/IOBufferMemoryDescriptor.h
index c3e7af343..4ce4565f0 100644
--- a/iokit/IOKit/IOBufferMemoryDescriptor.h
+++ b/iokit/IOKit/IOBufferMemoryDescriptor.h
@@ -75,9 +75,19 @@ private:
                                vm_size_t    capacity,
                                vm_offset_t  alignment,
 			       task_t	    inTask);
-
     OSMetaClassDeclareReservedUsed(IOBufferMemoryDescriptor, 0);
+
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+    virtual bool initWithPhysicalMask(
+				task_t		  inTask,
+				IOOptionBits      options,
+				mach_vm_size_t    capacity,
+				mach_vm_address_t alignment,
+				mach_vm_address_t physicalMask);
+    OSMetaClassDeclareReservedUsed(IOBufferMemoryDescriptor, 1);
+#else
     OSMetaClassDeclareReservedUnused(IOBufferMemoryDescriptor, 1);
+#endif
     OSMetaClassDeclareReservedUnused(IOBufferMemoryDescriptor, 2);
     OSMetaClassDeclareReservedUnused(IOBufferMemoryDescriptor, 3);
     OSMetaClassDeclareReservedUnused(IOBufferMemoryDescriptor, 4);
@@ -150,6 +160,7 @@ public:
     @discussion Added in Mac OS X 10.2, this method allocates a memory buffer with a given size and alignment in the task's address space specified, and returns a memory descriptor instance representing the memory. It is recommended that memory allocated for I/O or sharing via mapping be created via IOBufferMemoryDescriptor. Options passed with the request specify the kind of memory to be allocated - pageablity and sharing are specified with option bits. This function may block and so should not be called from interrupt level or while a simple lock is held.
     @param inTask The task the buffer will be allocated in.
     @param options Options for the allocation:<br>
+    kIODirectionOut, kIODirectionIn - set the direction of the I/O transfer.<br>
     kIOMemoryPhysicallyContiguous - pass to request memory be physically contiguous. This option is heavily discouraged. The request may fail if memory is fragmented, may cause large amounts of paging activity, and may take a very long time to execute.<br>
     kIOMemoryPageable - pass to request memory be non-wired - the default for kernel allocated memory is wired.<br>
     kIOMemoryPurgeable - pass to request memory that may later have its purgeable state set with IOMemoryDescriptor::setPurgeable. Only supported for kIOMemoryPageable allocations.<br>
@@ -164,6 +175,26 @@ public:
                                             vm_size_t    capacity,
                                             vm_offset_t  alignment = 1);
 
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+/*! @function inTaskWithPhysicalMask
+    @abstract Creates a memory buffer with memory descriptor for that buffer. 
+    @discussion Added in Mac OS X 10.5, this method allocates a memory buffer with a given size and alignment in the task's address space specified, and returns a memory descriptor instance representing the memory. It is recommended that memory allocated for I/O or sharing via mapping be created via IOBufferMemoryDescriptor. Options passed with the request specify the kind of memory to be allocated - pageablity and sharing are specified with option bits. This function may block and so should not be called from interrupt level or while a simple lock is held.
+    @param inTask The task the buffer will be mapped in. Pass NULL to create memory unmapped in any task (eg. for use as a DMA buffer).
+    @param options Options for the allocation:<br>
+    kIODirectionOut, kIODirectionIn - set the direction of the I/O transfer.<br>
+    kIOMemoryPhysicallyContiguous - pass to request memory be physically contiguous. This option is heavily discouraged. The request may fail if memory is fragmented, may cause large amounts of paging activity, and may take a very long time to execute.<br>
+    kIOMemoryKernelUserShared - pass to request memory that will be mapped into both the kernel and client applications.
+    @param capacity The number of bytes to allocate.
+    @param mask The buffer will be allocated with pages such that physical addresses will only have bits set present in physicalMask. For example, pass 0x00000000FFFFFFFFULL for a buffer to be accessed by hardware that has 32 address bits.
+    @result Returns an instance of class IOBufferMemoryDescriptor to be released by the caller, which will free the memory desriptor and associated buffer. */
+
+    static IOBufferMemoryDescriptor * inTaskWithPhysicalMask(
+					    task_t	      inTask,
+                                            IOOptionBits      options,
+                                            mach_vm_size_t    capacity,
+                                            mach_vm_address_t physicalMask);
+#endif
+
     /*
      * withCapacity:
      *
@@ -248,6 +279,11 @@ public:
      * will not copy past the end of the memory descriptor's current capacity.
      */
     virtual bool appendBytes(const void *bytes, vm_size_t withLength);
+
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+    /* DEPRECATED */ virtual void * getVirtualSegment(IOByteCount offset,
+    /* DEPRECATED */					IOByteCount * length);
+#endif
 };
 
 #endif /* !_IOBUFFERMEMORYDESCRIPTOR_H */
diff --git a/iokit/IOKit/IOCommandGate.h b/iokit/IOKit/IOCommandGate.h
index 67c16b59d..61f70f417 100644
--- a/iokit/IOKit/IOCommandGate.h
+++ b/iokit/IOKit/IOCommandGate.h
@@ -54,7 +54,7 @@ check if the hardware is active, if so it will add the request to a pending
 queue internal to the device or the device's family.  Otherwise if the hardware
 is inactive then this request can be acted upon immediately.
 <br><br>
-	CAUTION: The runAction and runCommand functions can not be called from an interrupt context.
+    CAUTION: The runAction and runCommand functions can not be called from an interrupt context. But attemptCommand can, though it may return an error
 
 */
 class IOCommandGate : public IOEventSource
@@ -116,6 +116,12 @@ compiler warning.  Defaults to zero, see $link IOEventSource::setAction.
     @result True if inherited classes initialise successfully. */
     virtual bool init(OSObject *owner, Action action = 0);
 
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+    // Superclass overrides
+    virtual void free();
+    virtual void setWorkLoop(IOWorkLoop *inWorkLoop);
+#endif
+
 /*! @function runCommand
     @abstract Single thread a command with the target work-loop.
     @discussion Client function that causes the current action to be called in
@@ -123,13 +129,12 @@ a single threaded manner.  Beware the work-loop's gate is recursive and command
 gates can cause direct or indirect re-entrancy.	 When the executing on a
 client's thread runCommand will sleep until the work-loop's gate opens for
 execution of client actions, the action is single threaded against all other
-work-loop event sources.
+work-loop event sources.  If the command is disabled the attempt to run a command will be stalled until enable is called.
     @param arg0 Parameter for action of command gate, defaults to 0.
     @param arg1 Parameter for action of command gate, defaults to 0.
     @param arg2 Parameter for action of command gate, defaults to 0.
     @param arg3 Parameter for action of command gate, defaults to 0.
-    @result kIOReturnSuccess if successful. kIOReturnNotPermitted if this
-event source is currently disabled, kIOReturnNoResources if no action available.
+    @result kIOReturnSuccess if successful. kIOReturnAborted if a disabled command gate is free()ed before being reenabled, kIOReturnNoResources if no action available.
 */
     virtual IOReturn runCommand(void *arg0 = 0, void *arg1 = 0,
 				void *arg2 = 0, void *arg3 = 0);
@@ -141,13 +146,13 @@ a single threaded manner.  Beware the work-loop's gate is recursive and command
 gates can cause direct or indirect re-entrancy.	 When the executing on a
 client's thread runAction will sleep until the work-loop's gate opens for
 execution of client actions, the action is single threaded against all other
-work-loop event sources.
+work-loop event sources.  If the command is disabled the attempt to run a command will be stalled until enable is called.
     @param action Pointer to function to be executed in work-loop context.
     @param arg0 Parameter for action parameter, defaults to 0.
     @param arg1 Parameter for action parameter, defaults to 0.
     @param arg2 Parameter for action parameter, defaults to 0.
     @param arg3 Parameter for action parameter, defaults to 0.
-    @result kIOReturnSuccess if successful. kIOReturnBadArgument if action is not defined, kIOReturnNotPermitted if this event source is currently disabled.
+    @result kIOReturnSuccess if successful. kIOReturnBadArgument if action is not defined, kIOReturnAborted if a disabled command gate is free()ed before being reenabled.
 */
     virtual IOReturn runAction(Action action,
 			       void *arg0 = 0, void *arg1 = 0,
@@ -156,9 +161,7 @@ work-loop event sources.
 /*! @function attemptCommand
     @abstract Single thread a command with the target work-loop.
     @discussion Client function that causes the current action to be called in
-a single threaded manner.  Beware the work-loop's gate is recursive and command
-gates can cause direct or indirect re-entrancy.	 When the executing on a
-client's thread attemptCommand will fail if the work-loop's gate is open.
+a single threaded manner.  When the executing on a client's thread attemptCommand will fail if the work-loop's gate is closed.
     @param arg0 Parameter for action of command gate, defaults to 0.
     @param arg1 Parameter for action of command gate, defaults to 0.
     @param arg2 Parameter for action of command gate, defaults to 0.
@@ -173,7 +176,7 @@ client's thread attemptCommand will fail if the work-loop's gate is open.
     @discussion Client function that causes the given action to be called in
 a single threaded manner.  Beware the work-loop's gate is recursive and command
 gates can cause direct or indirect re-entrancy.	 When the executing on a
-client's thread attemptCommand will fail if the work-loop's gate is open.
+client's thread attemptCommand will fail if the work-loop's gate is closed.
     @param action Pointer to function to be executed in work-loop context.
     @param arg0 Parameter for action parameter, defaults to 0.
     @param arg1 Parameter for action parameter, defaults to 0.
@@ -201,6 +204,18 @@ client's thread attemptCommand will fail if the work-loop's gate is open.
     @param onlyOneThread true to only wake up at most one thread, false otherwise. */
     virtual void commandWakeup(void *event, bool oneThread = false);
 
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+/*! @function disable
+    @abstract Disable the command gate
+    @discussion When a command gate is disabled all future calls to runAction and runCommand will stall until the gate is enable()d later.  This can be used to block client threads when a system sleep is requested.  The IOWorkLoop thread itself will never stall, even when making runAction/runCommand calls.  This call must be made from a gated context, to clear potential race conditions.  */
+    virtual void disable();
+
+/*! @function enable
+    @abstract Enable command gate, this will unblock any blocked Commands and Actions.
+    @discussion  Enable the command gate.  The attemptAction/attemptCommand calls will now be enabled and can succeeed.  Stalled runCommand/runAction calls will be woken up. */
+    virtual void enable();
+#endif
+
 private:
     OSMetaClassDeclareReservedUnused(IOCommandGate, 0);
     OSMetaClassDeclareReservedUnused(IOCommandGate, 1);
diff --git a/iokit/IOKit/IODMACommand.h b/iokit/IOKit/IODMACommand.h
new file mode 100644
index 000000000..56a36018f
--- /dev/null
+++ b/iokit/IOKit/IODMACommand.h
@@ -0,0 +1,474 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifndef _IODMACOMMAND_H
+#define _IODMACOMMAND_H
+
+#include <IOKit/IOCommand.h>
+#include <IOKit/IOMemoryDescriptor.h>
+class IOMapper;
+
+/**************************** class IODMACommand ***************************/
+
+/*!
+    @class IODMACommand
+    @abstract A mechanism to convert memory references to I/O bus addresses.
+    @discussion The IODMACommand is supersedes the IOMemoryCursor and greatly enhances the functionality and power of it.  The command can be specified to output 64 bit physical addresses and also allows driver writers bypass mapping hardware or get addresses suitable for non-snooped DMA.
+<br><br>
+    The command is designed to be very easily subclassable.  Most driver writers need to associate some DMA operations with their memory descriptor and usually use a C structure for that purpose.  This structure is often kept in a linked list.  This IODMACommand has built it <kern/queue.h> linkage and can be derived and 'public:' variables added, giving the developer a structure that can associate a memory descriptor with a particular dma command but will also allow the developer to generate that command and keep the state necessary for tracking it.
+<br><br>
+    It is envisaged that a pool of IODMACommands will be created at driver initialisation and each command will be kept in an IOCommandPool while not in use.  However if developers wishes to maintain their own free lists that is certainly possible.  See the <kern/queue.h> and <xnu/iokit/Kernel/IOCommandPool> for sample code on manipulating the command's doubly linked list entries.
+<br><br>
+    The IODMACommand can be used in a 'weak-linked' manner.  To do this you must avoid using any static member functions.  Use the, much slower but safe, weakWithSpecification function.  On success a dma command instance will be returned.  This instance can then be used to clone as many commands as is needed.  Remember deriving from this class can not be done weakly, that is no weak subclassing!
+*/
+
+class IODMACommand : public IOCommand
+{
+    OSDeclareDefaultStructors(IODMACommand);
+
+public:
+
+/*!
+    @typedef Segment32
+    @discussion A 32 bit I/O bus address/length pair
+*/
+    struct Segment32 {
+	UInt32 fIOVMAddr, fLength;
+    };
+
+/*!
+    @typedef Segment64
+    @discussion A 64 bit I/O bus address/length pair
+*/
+    struct Segment64 {
+	UInt64 fIOVMAddr, fLength;
+    };
+
+/*! @enum MappingOptions
+    @abstract Mapping types to indicate the desired mapper type for translating memory descriptors into I/O DMA Bus addresses.
+    @constant kNonCoherent	Used by drivers for non-coherent transfers, implies unmapped memmory
+    @constant kMapped		Allow a driver to define addressing size
+    @constant kBypassed		Allow drivers to bypass any mapper
+    @constant kMaxMappingOptions	Internal use only
+*/
+    enum MappingOptions {
+	kMapped       = 0x00000000,
+	kBypassed     = 0x00000001,
+	kNonCoherent  = 0x00000002,
+	kTypeMask     = 0x0000000f,
+
+	kNoCacheStore = 0x00000010,	// Memory in descriptor 
+	kOnChip       = 0x00000020,	// Indicates DMA is on South Bridge
+	kIterateOnly  = 0x00000040	// DMACommand will be used as a cursor only
+    };
+
+/*! @enum SynchronizeOptions
+    @abstract Options for the synchronize method.
+    @constant kForceDoubleBuffer Copy the entire prepared range to a new page aligned buffer.
+*/
+    enum SynchronizeOptions {
+	kForceDoubleBuffer = 0x01000000
+    };
+
+/*!
+    @typedef SegmentFunction
+    @discussion Pointer to a C function that translates a 64 segment and outputs a single desired segment to the array at the requested index. There are a group of pre-implemented SegmentFunctions that may be usefull to the developer below.
+    @param segment The 64Bit I/O bus address and length.
+    @param segments Base of the output vector of DMA address length pairs.
+    @param segmentIndex Index to output 'segment' in the 'segments' array.
+    @result Returns true if segment encoding succeeded.  false may be returned if the current segment does not fit in an output segment, i.e. a 38bit address wont fit into a 32 encoding.
+*/
+    typedef bool (*SegmentFunction)(IODMACommand *target,
+	    			    Segment64 segment,
+				    void     *segments,
+				    UInt32    segmentIndex);
+
+    // -------------- Preimplemented output functions ----------------
+
+/*! @function OutputHost32
+    @abstract Output host natural Segment32 output segment function.
+*/
+    static bool OutputHost32(IODMACommand *target,
+	    Segment64 seg, void *segs, UInt32 ind);
+
+/*! @defined kIODMACommandOutputHost32
+    @abstract Output host natural Segment32 output segment function.
+ */
+#define kIODMACommandOutputHost32   (IODMACommand::OutputHost32)
+
+/*! @function OutputBig32
+    @abstract Output big-endian Segment32 output segment function.
+*/
+    static bool OutputBig32(IODMACommand *target,
+	    Segment64 seg, void *segs, UInt32 ind);
+
+/*! @defined kIODMACommandOutputBig32
+    @abstract Output big-endian Segment32 output segment function.
+ */
+#define kIODMACommandOutputBig32    (IODMACommand::OutputBig32)
+
+/*! @function OutputLittle32
+    @abstract Output little-endian Segment32 output segment function.
+*/
+    static bool OutputLittle32(IODMACommand *target,
+	    Segment64 seg, void *segs, UInt32 ind);
+
+/*! @defined kIODMACommandOutputLittle32
+    @abstract Output little-endian Segment32 output segment function.
+*/
+#define kIODMACommandOutputLittle32 (IODMACommand::OutputLittle32)
+
+/*! @function OutputHost64
+    @abstract Output host natural Segment64 output segment function.
+*/
+    static bool OutputHost64(IODMACommand *target,
+	    Segment64 seg, void *segs, UInt32 ind);
+
+/*! @defined kIODMACommandOutputHost64
+    @abstract Output host natural Segment64 output segment function.
+*/
+#define kIODMACommandOutputHost64   (IODMACommand::OutputHost64)
+
+/*! @function OutputBig64
+    @abstract Output big-endian Segment64 output segment function.
+*/
+    static bool OutputBig64(IODMACommand *target,
+	    Segment64 seg, void *segs, UInt32 ind);
+
+/*! @defined kIODMACommandOutputLittle64
+    @abstract Output little-endian Segment64 output segment function.
+*/
+#define kIODMACommandOutputBig64    (IODMACommand::OutputBig64)
+
+/*! @function OutputLittle64
+    @abstract Output little-endian Segment64 output segment function.
+*/
+    static bool OutputLittle64(IODMACommand *target,
+	    Segment64 seg, void *segs, UInt32 ind);
+
+/*! @defined kIODMACommandOutputBig64
+    @abstract Output big-endian Segment64 output segment function.
+*/
+#define kIODMACommandOutputLittle64 (IODMACommand::OutputLittle64)
+
+/*! @function withSpecification
+    @abstract Creates and initializes an IODMACommand in one operation.
+    @discussion Factory function to create and initialize an IODMACommand in one operation.
+    @param outSegFunc SegmentFunction to call to output one physical segment. A set of nine commonly required segment functions are provided.
+    @param numAddressBits Number of bits that the hardware uses on its internal address bus. Typically 32 but may be more on modern hardware.  A 0 implies no-restriction other than that implied by the output segment function.
+    @param maxSegmentSize Maximum allowable size for one segment.  If 0 is passed the maximum segment size is unlimited.
+    @param mappingOptions is the type of mapping that is required to translate an IOMemoryDescriptor into the desired number of bits.  For instance if your hardware only supports 32 bits but must run on machines with > 4G of RAM some mapping will be required.  Number of bits will be specified in numAddressBits, see below.This parameter can take 3 values:- kNonCoherent - used for non-coherent hardware transfers, Mapped - Validate that all I/O bus generated addresses are within the number of addressing bits specified, Bypassed indicates that bypassed addressing is required, this is used when the hardware transferes are into coherent memory but no mapping is required.  See also prepare() for failure cases.
+    @param maxTransferSize Maximum size of an entire transfer.	Defaults to 0 indicating no maximum.
+    @param alignment Alignment restriction, in bytes, on I/O bus addresses.  Defaults to single byte alignment.
+    @param mapper For mapping types kMapped & kBypassed mapper is used to define the hardware that will perform the mapping, defaults to the system mapper.
+    @result Returns a new memory cursor if successfully created and initialized, 0 otherwise.
+*/
+    static IODMACommand *
+	withSpecification(SegmentFunction  outSegFunc,
+			  UInt8            numAddressBits,
+			  UInt64           maxSegmentSize,
+			  MappingOptions   mappingOptions = kMapped,
+			  UInt64           maxTransferSize = 0,
+			  UInt32           alignment = 1,
+			  IOMapper        *mapper = 0,
+			  void            *refCon = 0);
+
+/*! @function weakWithSpecification
+    @abstract Creates and initialises an IODMACommand in one operation if this version of the operating system supports it.
+    @discussion Factory function to create and initialise an IODMACommand in one operation.  The function allows a developer to 'weak' link with IODMACommand.  This function will return kIOReturnUnsupported if the IODMACommand is unavailable.  This function is actually fairly slow so it will be better to call it once then clone the successfully create command using cloneCommand (q.v.).
+    @param newCommand Output reference variable of the newly created IODMACommand.
+    @param outSegFunc SegmentFunction to call to output one physical segment. A set of nine commonly required segment functions are provided.
+    @param numAddressBits Number of bits that the hardware uses on its internal address bus. Typically 32 but may be more on modern hardware.  A 0 implies no-restriction other than that implied by the output segment function.
+    @param maxSegmentSize Maximum allowable size for one segment. Zero is treated as an unlimited segment size.
+    @param mapType is the type of mapping that is required to translate an IOMemoryDescriptor into the desired number of bits.  For instance if your hardware only supports 32 bits but must run on machines with > 4G of RAM some mapping will be required.  Number of bits will be specified in numAddressBits, see below.  This parameter can take 3 values:- kNonCoherent - used for non-coherent hardware transfers, Mapped - Validate that all I/O bus generated addresses are within the number of addressing bits specified, Bypassed indicates that bypassed addressing is required, this is used when the hardware transfers are into coherent memory but no mapping is required.  See also prepare() for failure cases.
+    @param maxTransferSize Maximum size of an entire transfer.  Defaults to 0 indicating no maximum.
+    @param alignment Alignment restriction, in bytes, on I/O bus addresses.  Defaults to single byte alignment.
+    @param mapper For mapping types kMapped & kBypassed mapper is used to define the hardware that will perform the mapping, defaults to the system mapper.
+    @result kIOReturnSuccess if everything is OK, otherwise kIOReturnBadArgument if newCommand is NULL, kIOReturnUnsupported if the kernel doesn't export IODMACommand or IOReturnError if the new command fails to init, q.v. initWithSpecification.
+*/
+    // Note that the function has the attribute always_inline.
+    // The point of this function is to make a call into the kernel
+    // without generating an undefined symbol.  If the client could call
+    // the code as a function then the goal of no undefined symbols
+    // would be lost thus defeating the purpose.
+    static inline IOReturn weakWithSpecification
+	    (IODMACommand   **newCommand,
+	     SegmentFunction outSegFunc,
+	     UInt8           numAddressBits,
+	     UInt64          maxSegmentSize,
+	     MappingOptions  mapType = kMapped,
+	     UInt64          maxTransferSize = 0,
+	     UInt32          alignment = 1,
+	     IOMapper       *mapper = 0,
+	     void           *refCon = 0) __attribute__((always_inline));
+
+/*!
+    @function cloneCommand
+    @abstract Creates a new command based on the specification of the current one.
+    @discussion Factory function to create and initialise an IODMACommand in one operation.  The current command's specification will be duplicated in the new object, but however none of its state will be duplicated.  This means that it is safe to clone a command even if it is currently active and running, however you must be certain that the command to be duplicated does have a valid reference for the duration.
+    @result Returns a new memory cursor if successfully created and initialised, 0 otherwise.
+*/
+    virtual IODMACommand *cloneCommand(void *refCon = 0);
+
+/*! @function initWithSpecification
+    @abstract Primary initializer for the IODMACommand class. 
+    @param outSegFunc SegmentFunction to call to output one physical segment. A set of nine commonly required segment functions are provided.
+    @param numAddressBits Number of bits that the hardware uses on its internal address bus. Typically 32 but may be more on modern hardware.  A 0 implies no-restriction other than that implied by the output segment function.
+    @param maxSegmentSize Maximum allowable size for one segment.  Defaults to 0 which means any size.
+    @param mappingOptions is the type of mapping that is required to translate an IOMemoryDescriptor into the desired number of bits.  For instance if your hardware only supports 32 bits but must run on machines with > 4G of RAM some mapping will be required.  Number of bits will be specified in numAddressBits, see below.This parameter can take 3 values:- kNonCoherent - used for non-coherent hardware transfers, Mapped - Validate that all I/O bus generated addresses are within the number of addressing bits specified, Bypassed indicates that bypassed addressing is required, this is used when the hardware transferes are into coherent memory but no mapping is required.  See also prepare() for failure cases.
+    @param maxTransferSize Maximum size of an entire transfer.	Defaults to 0 indicating no maximum.
+    @param alignment Alignment restriction, in bytes, on I/O bus addresses.  Defaults to single byte alignment.
+    @param mapper For mapping types kMapped & kBypassed mapper is used to define the hardware that will perform the mapping, defaults to the system mapper.
+    @result Can fail if the mapping type is not recognised, if one of the 3 mandatory parameters are set to 0, if a 32 bit output function is selected when more than 32 bits of address is required or, if kBypassed is requested on a machine that doesn't support bypassing.  Returns true otherwise.
+*/
+    virtual bool initWithSpecification( SegmentFunction  outSegFunc,
+					UInt8     numAddressBits,
+					UInt64    maxSegmentSize,
+					MappingOptions mappingOptions = kMapped,
+					UInt64    maxTransferSize = 0,
+					UInt32    alignment = 1,
+					IOMapper *mapper = 0,
+					void     *refCon = 0);
+
+/*! @function setMemoryDescriptor
+    @abstract Sets and resets the DMACommand's current memory descriptor
+    @discussion The DMA command will configure itself based on the information that it finds in the memory descriptor.  It looks for things like the direction of the memory descriptor and whether the current memory descriptor is already mapped into some IOMMU.  As a programmer convenience it can also prepare the memory descriptor immediately.  See prepare().  Note the IODMACommand is designed to used multiple times with a succession of memory descriptors, making the pooling of commands possible.  It is an error though to attempt to reset a currently prepared() DMA command.  Warning: This routine may block so never try to autoprepare an IODMACommand while in a gated context, i.e. one of the WorkLoops action call outs.
+    @param mem A pointer to the current I/Os memory descriptor.
+    @param autoPrepare An optional boolean variable that will call the prepare() function automatically after the memory descriptor is processed. Defaults to true.
+    @result Returns kIOReturnSuccess, kIOReturnBusy if currently prepared, kIOReturnNoSpace if the length(mem) >= Maximum Transfer Size or the error codes returned by prepare() (qv).
+*/
+    virtual IOReturn setMemoryDescriptor(const IOMemoryDescriptor *mem,
+				         bool autoPrepare = true);
+
+/*! @function clearMemoryDescriptor
+    @abstract Clears the DMACommand's current memory descriptor
+    @discussion completes and invalidates the cache if the DMA command is currently active, copies all data from bounce buffers if necessary and releases all resources acquired during setMemoryDescriptor.
+    @param autoComplete An optional boolean variable that will call the complete() function automatically before the memory descriptor is processed. Defaults to true.
+*/
+    virtual IOReturn clearMemoryDescriptor(bool autoComplete = true);
+
+/*! @function getMemoryDescriptor
+    @abstract Get the current memory descriptor
+*/
+    virtual const IOMemoryDescriptor *getMemoryDescriptor() const;
+
+/*! @function prepare
+    @abstract Prepare the memory for an I/O transfer.
+    @discussion Allocate the mapping resources neccessary for this transfer, specifying a sub range of the IOMemoryDescriptor that will be the target of the I/O.  The complete() method frees these resources.  Data may be copied to buffers for kIODirectionOut memory descriptors, depending on hardware mapping resource availabilty or alignment restrictions.  It should be noted that the this function may block and should only be called on the clients context, i.e never call this routine while gated; also the call itself is not thread safe though this should be an issue as each IODMACommand is independant.
+    @param offset defines the starting offset in the memory descriptor the DMA command will operate on. genIOVMSegments will produce its results based on the offset and length passed to the prepare method.
+    @param length defines the ending position in the memory descriptor the DMA command will operate on. genIOVMSegments will produce its results based on the offset and length passed to the prepare method.
+    @param flushCache Flush the caches for the memory descriptor and make certain that the memory cycles are complete.  Defaults to true for kNonCoherent and is ignored by the other types.
+    @param synchronize Copy any buffered data back from the target IOMemoryDescriptor.  Defaults to true, if synchronize() is being used to explicitly copy data, passing false may avoid an unneeded copy.
+    @result An IOReturn code. */
+
+    virtual IOReturn prepare(UInt64 offset = 0, UInt64 length = 0, bool flushCache = true, bool synchronize = true);
+
+/*! @function complete
+    @abstract Complete processing of DMA mappings after an I/O transfer is finished.
+    @discussion This method should not be called unless a prepare was previously issued; the prepare() and complete() must occur in pairs, before and after an I/O transfer
+    @param invalidCache Invalidate the caches for the memory descriptor.  Defaults to true for kNonCoherent and is ignored by the other types.
+    @param synchronize Copy any buffered data back to the target IOMemoryDescriptor.  Defaults to true, if synchronize() is being used to explicitly copy data, passing false may avoid an unneeded copy.
+    @result kIOReturnNotReady if not prepared, kIOReturnSuccess otherwise. */
+
+    virtual IOReturn complete(bool invalidateCache = true, bool synchronize = true);
+
+/*! @function synchronize
+    @abstract Bring IOMemoryDescriptor and IODMACommand buffers into sync.
+    @discussion This method should not be called unless a prepare was previously issued. If needed a caller may synchronize any IODMACommand buffers with the original IOMemoryDescriptor buffers.
+    @param options Specifies the direction of the copy:
+	kIODirectionOut copy IOMemoryDesciptor memory to any IODMACommand buffers. By default this action takes place automatically at prepare().
+	kIODirectionIn copy any IODMACommand buffers back to the IOMemoryDescriptor. By default this action takes place automatically at complete().
+        kForceDoubleBuffer copy the entire prepared range to a new page aligned buffer.
+    @result kIOReturnNotReady if not prepared, kIOReturnBadArgument if invalid options are passed, kIOReturnSuccess otherwise. */
+
+    virtual IOReturn synchronize(IOOptionBits options);
+
+/*! @function genIOVMSegments
+    @abstract Generates a physical scatter/gather for the current DMA command
+    @discussion Generates a list of physical segments from the given memory descriptor, relative to the current position of the descriptor.  The constraints that are set during initialisation will be respected. This function maintains the state across multiple calls for efficiency.  However the state is discarded if the new offset is not the expected one.
+    @param offset input/output parameter, defines the starting and ending offset in the memory descriptor, relative to any offset passed to the prepare() method.
+    @param segments Void pointer to base of output physical scatter/gather list.  Always passed directly onto the SegmentFunction.
+    @param numSegments Input/output parameter Number of segments that can fit in the segment array and returns number of segments generated.
+    @result kIOReturnSuccess on success, kIOReturnOverrun if the memory descriptor is exhausted, kIOReturnMessageTooLarge if the output segment function's address bits has insufficient resolution for a segment, kIOReturnNotReady if the DMA command has not be prepared, kIOReturnBadArgument if the DMA command doesn't have a memory descriptor yet or some of the parameters are NULL and kIOReturnNotReady if the DMA command is not prepared.
+*/
+    virtual IOReturn genIOVMSegments(UInt64 *offset,
+				     void   *segments,
+				     UInt32 *numSegments);
+
+/*! @function gen32IOVMSegments
+    @abstract Helper function for a type checked call to genIOVMSegments(qv), for use with an IODMACommand set up with the output function kIODMACommandOutputHost32, kIODMACommandOutputBig32, or kIODMACommandOutputLittle32. If the output function of the IODMACommand is not a 32 bit function, results will be incorrect.
+*/
+    inline IOReturn gen32IOVMSegments(UInt64   *offset,
+				      Segment32 *segments,
+				      UInt32     *numSegments)
+    { return genIOVMSegments(offset, segments, numSegments); };
+
+/*! @function gen64IOVMSegments
+    @abstract Helper function for a type checked call to genIOVMSegments(qv), for use with an IODMACommand set up with the output function kIODMACommandOutputHost64, kIODMACommandOutputBig64, or kIODMACommandOutputLittle64. If the output function of the IODMACommand is not a 64 bit function, results will be incorrect.
+*/
+    inline IOReturn gen64IOVMSegments(UInt64    *offset,
+				      Segment64 *segments,
+				      UInt32    *numSegments)
+    { return genIOVMSegments(offset, segments, numSegments); };
+
+    virtual void free();
+
+private:
+
+    typedef IOReturn (*InternalSegmentFunction)(
+				    void         *reference,
+				    IODMACommand *target,
+				    Segment64     segment,
+				    void         *segments,
+				    UInt32        segmentIndex);
+
+    IOReturn genIOVMSegments(InternalSegmentFunction outSegFunc,
+				  void   *reference,
+				  UInt64 *offsetP,
+				  void   *segmentsP,
+				  UInt32 *numSegmentsP);
+
+    static IOReturn clientOutputSegment(
+	    void *reference, IODMACommand *target,
+	    Segment64 segment, void *vSegList, UInt32 outSegIndex);
+
+    static IOReturn segmentOp(
+			void         *reference,
+			IODMACommand *target,
+			Segment64     segment,
+			void         *segments,
+			UInt32        segmentIndex);
+    IOReturn IODMACommand::walkAll(UInt8 op);
+
+private:
+    OSMetaClassDeclareReservedUnused(IODMACommand,  0);
+    OSMetaClassDeclareReservedUnused(IODMACommand,  1);
+    OSMetaClassDeclareReservedUnused(IODMACommand,  2);
+    OSMetaClassDeclareReservedUnused(IODMACommand,  3);
+    OSMetaClassDeclareReservedUnused(IODMACommand,  4);
+    OSMetaClassDeclareReservedUnused(IODMACommand,  5);
+    OSMetaClassDeclareReservedUnused(IODMACommand,  6);
+    OSMetaClassDeclareReservedUnused(IODMACommand,  7);
+    OSMetaClassDeclareReservedUnused(IODMACommand,  8);
+    OSMetaClassDeclareReservedUnused(IODMACommand,  9);
+    OSMetaClassDeclareReservedUnused(IODMACommand, 10);
+    OSMetaClassDeclareReservedUnused(IODMACommand, 11);
+    OSMetaClassDeclareReservedUnused(IODMACommand, 12);
+    OSMetaClassDeclareReservedUnused(IODMACommand, 13);
+    OSMetaClassDeclareReservedUnused(IODMACommand, 14);
+    OSMetaClassDeclareReservedUnused(IODMACommand, 15);
+
+public:
+/*! @var fRefCon Reference Constant, client defined publicly avialable */
+    void *fRefCon;
+
+protected:
+
+/*! @var fMaxSegmentSize Maximum size of one segment in a scatter/gather list */
+    UInt64  fMaxSegmentSize;
+
+/*! @var fMaxTransferSize
+    Maximum size of a transfer that this memory cursor is allowed to generate */
+    UInt64  fMaxTransferSize;
+
+/*! @var fBypassMask
+    Mask to be ored into the address to bypass the given iommu's mapping. */
+    UInt64  fBypassMask;
+
+/*! @var fMapper
+    Client defined mapper. */
+    IOMapper *fMapper;
+
+/*! @var fMemory
+    memory descriptor for current I/O. */
+    const IOMemoryDescriptor *fMemory;
+
+/*! @var fOutSeg The action method called when an event has been delivered */
+    SegmentFunction fOutSeg;
+
+/*! @var fAlignMask
+    Alignment restriction mask. */
+    UInt32  fAlignMask;
+
+/*! @var fNumAddressBits
+    Number of bits that the hardware can address */
+    UInt32 fNumAddressBits;
+
+/*! @var fNumSegments
+    Number of contiguous segments required for the current memory descriptor and desired mapping */
+    UInt32  fNumSegments;
+
+/*! @var fMappingOptions
+    What type of I/O virtual address mapping is required for this command */
+    MappingOptions  fMappingOptions;
+
+/*! @var fActive
+    fActive indicates that this DMA command is currently prepared and ready to go */
+    UInt32 fActive;
+
+/*! @var reserved
+    Reserved for future use.  (Internal use only)  */
+    struct ExpansionData * reserved;
+};
+
+IOReturn IODMACommand::
+weakWithSpecification(IODMACommand **newCommand,
+		      SegmentFunction  outSegFunc,
+		      UInt8     numAddressBits,
+		      UInt64    maxSegmentSize,
+		      MappingOptions mapType,
+		      UInt64    maxTransferSize,
+		      UInt32    alignment,
+		      IOMapper *mapper,
+		      void     *refCon) 
+{
+    if (!newCommand)
+	return kIOReturnBadArgument;
+
+    IODMACommand *self = (IODMACommand *)
+	OSMetaClass::allocClassWithName("IODMACommand");
+    if (!self)
+	return kIOReturnUnsupported;
+
+    IOReturn ret;
+    bool inited = self->
+	initWithSpecification(outSegFunc,
+			      numAddressBits, maxSegmentSize, mapType,
+			      maxTransferSize, alignment, mapper, refCon);
+    if (inited)
+	ret =  kIOReturnSuccess;
+    else {
+	self->release();
+	self = 0;
+	ret = kIOReturnError;
+    }
+
+    *newCommand = self;
+    return ret;
+};
+#endif /* !_IODMACOMMAND_H */
+
diff --git a/iokit/IOKit/IOHibernatePrivate.h b/iokit/IOKit/IOHibernatePrivate.h
index d923dc564..db2c447e5 100644
--- a/iokit/IOKit/IOHibernatePrivate.h
+++ b/iokit/IOKit/IOHibernatePrivate.h
@@ -71,7 +71,10 @@ struct IOHibernateImageHeader
     uint32_t	signature;
     uint32_t	processorFlags;
 
-    uint8_t	reserved2[24];
+    uint32_t    runtimePages;
+    uint32_t    runtimePageCount;
+
+    uint8_t	reserved2[16];
     
     uint64_t	encryptStart;
     uint64_t	machineSignature;
@@ -81,7 +84,13 @@ struct IOHibernateImageHeader
 
     uint32_t	diag[4];
 
-    uint32_t	reserved[82];		// make sizeof == 512
+    int32_t	graphicsInfoOffset;
+    int32_t	cryptVarsOffset;
+    int32_t	memoryMapOffset;
+    uint32_t    memoryMapSize;
+    uint32_t    systemTableOffset;
+
+    uint32_t	reserved[77];		// make sizeof == 512
 
     uint32_t		fileExtentMapSize;
     IOPolledFileExtent	fileExtentMap[2];
@@ -146,14 +155,14 @@ enum
 
 struct hibernate_graphics_t
 {
-    unsigned long physicalAddress;	// Base address of video memory
-    unsigned long mode;    		// 
-    unsigned long rowBytes;   		// Number of bytes per pixel row
-    unsigned long width;      		// Width
-    unsigned long height;     		// Height
-    unsigned long depth;      		// Pixel Depth
-
-    uint8_t	  progressSaveUnder[kIOHibernateProgressCount][kIOHibernateProgressSaveUnderSize];
+    uint32_t physicalAddress;		// Base address of video memory
+    uint32_t mode;			// 
+    uint32_t rowBytes;   		// Number of bytes per pixel row
+    uint32_t width;      		// Width
+    uint32_t height;     		// Height
+    uint32_t depth;      		// Pixel Depth
+
+    uint8_t progressSaveUnder[kIOHibernateProgressCount][kIOHibernateProgressSaveUnderSize];
 };
 typedef struct hibernate_graphics_t hibernate_graphics_t;
 
@@ -231,15 +240,25 @@ hibernate_vm_lock(void);
 void
 hibernate_vm_unlock(void);
 
+// mark pages not to be saved, based on VM system accounting
 void
 hibernate_page_list_setall(hibernate_page_list_t * page_list,
 			   hibernate_page_list_t * page_list_wired,
 			   uint32_t * pagesOut);
 
+// mark pages to be saved, or pages not to be saved but available 
+// for scratch usage during restore
 void
 hibernate_page_list_setall_machine(hibernate_page_list_t * page_list,
                                     hibernate_page_list_t * page_list_wired,
                                     uint32_t * pagesOut);
+
+// mark pages not to be saved and not for scratch usage during restore
+void
+hibernate_page_list_set_volatile( hibernate_page_list_t * page_list,
+				  hibernate_page_list_t * page_list_wired,
+				  uint32_t * pagesOut);
+
 void
 hibernate_page_list_discard(hibernate_page_list_t * page_list);
 
@@ -249,11 +268,15 @@ hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_
 
 void 
 hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page);
+
 boolean_t 
 hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page);
 
+hibernate_bitmap_t *
+hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * page);
+
 uint32_t
-hibernate_page_list_count(hibernate_page_list_t *list, uint32_t set, uint32_t page);
+hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page);
 
 void 
 hibernate_restore_phys_page(uint64_t src, uint64_t dst, uint32_t len, uint32_t procFlags);
@@ -267,6 +290,10 @@ long
 hibernate_machine_entrypoint(IOHibernateImageHeader * header, void * p2, void * p3, void * p4);
 long
 hibernate_kernel_entrypoint(IOHibernateImageHeader * header, void * p2, void * p3, void * p4);
+void
+hibernate_newruntime_map(void * map, vm_size_t map_size, 
+			    uint32_t system_table_offset);
+
 
 extern uint32_t    gIOHibernateState;
 extern uint32_t    gIOHibernateMode;
@@ -336,6 +363,8 @@ enum
 #define kIOHibernateMemorySignatureEnvKey "mem-sig"
 #define kIOHibernateMachineSignatureKey	  "machine-signature"
 
+#define kIOHibernateRTCVariablesKey	"IOHibernateRTCVariables"
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/iokit/IOKit/IOKitKeys.h b/iokit/IOKit/IOKitKeys.h
index babf6dbc9..39519446a 100644
--- a/iokit/IOKit/IOKitKeys.h
+++ b/iokit/IOKit/IOKitKeys.h
@@ -82,6 +82,10 @@
 // IOService default user client class, for loadable user clients
 #define kIOUserClientClassKey		"IOUserClientClass"
 
+#define kIOUserClientCrossEndianKey		"IOUserClientCrossEndian"
+#define kIOUserClientCrossEndianCompatibleKey	"IOUserClientCrossEndianCompatible"
+#define kIOUserClientSharedInstanceKey		"IOUserClientSharedInstance"
+
 // IOService notification types
 #define kIOPublishNotification		"IOServicePublish"
 #define kIOFirstPublishNotification	"IOServiceFirstPublish"
@@ -95,6 +99,8 @@
 #define kIOAppPowerStateInterest	"IOAppPowerStateInterest"
 #define kIOPriorityPowerStateInterest	"IOPriorityPowerStateInterest"
 
+#define kIOPlatformDeviceMessageKey     "IOPlatformDeviceMessage"
+
 // IOService interest notification types
 #define kIOCFPlugInTypesKey		"IOCFPlugInTypes"
 
@@ -102,14 +108,16 @@
 #define kIOCommandPoolSizeKey	       "IOCommandPoolSize"          // (OSNumber)
 
 // properties found in services that have transfer constraints
-#define kIOMaximumBlockCountReadKey        "IOMaximumBlockCountRead"        // (OSNumber)
-#define kIOMaximumBlockCountWriteKey       "IOMaximumBlockCountWrite"       // (OSNumber)
-#define kIOMaximumByteCountReadKey         "IOMaximumByteCountRead"         // (OSNumber)
-#define kIOMaximumByteCountWriteKey        "IOMaximumByteCountWrite"        // (OSNumber)
-#define kIOMaximumSegmentCountReadKey      "IOMaximumSegmentCountRead"      // (OSNumber)
-#define kIOMaximumSegmentCountWriteKey     "IOMaximumSegmentCountWrite"     // (OSNumber)
-#define kIOMaximumSegmentByteCountReadKey  "IOMaximumSegmentByteCountRead"  // (OSNumber)
-#define kIOMaximumSegmentByteCountWriteKey "IOMaximumSegmentByteCountWrite" // (OSNumber)
+#define kIOMaximumBlockCountReadKey             "IOMaximumBlockCountRead"             // (OSNumber)
+#define kIOMaximumBlockCountWriteKey            "IOMaximumBlockCountWrite"            // (OSNumber)
+#define kIOMaximumByteCountReadKey              "IOMaximumByteCountRead"              // (OSNumber)
+#define kIOMaximumByteCountWriteKey             "IOMaximumByteCountWrite"             // (OSNumber)
+#define kIOMaximumSegmentCountReadKey           "IOMaximumSegmentCountRead"           // (OSNumber)
+#define kIOMaximumSegmentCountWriteKey          "IOMaximumSegmentCountWrite"          // (OSNumber)
+#define kIOMaximumSegmentByteCountReadKey       "IOMaximumSegmentByteCountRead"       // (OSNumber)
+#define kIOMaximumSegmentByteCountWriteKey      "IOMaximumSegmentByteCountWrite"      // (OSNumber)
+#define kIOMinimumSegmentAlignmentByteCountKey  "IOMinimumSegmentAlignmentByteCount"  // (OSNumber)
+#define kIOMaximumSegmentAddressableBitCountKey "IOMaximumSegmentAddressableBitCount" // (OSNumber)
 
 // properties found in services that wish to describe an icon
 //
diff --git a/iokit/IOKit/IOKitKeysPrivate.h b/iokit/IOKit/IOKitKeysPrivate.h
index 9a3a23df2..42159479d 100644
--- a/iokit/IOKit/IOKitKeysPrivate.h
+++ b/iokit/IOKit/IOKitKeysPrivate.h
@@ -34,6 +34,7 @@
 // properties found in the registry root
 #define kIOConsoleUsersKey		"IOConsoleUsers"		/* value is OSArray */
 #define kIOMaximumMappedIOByteCountKey  "IOMaximumMappedIOByteCount"    /* value is OSNumber */
+#define kIOStartupMkextCRC		"IOStartupMkextCRC"		/* value is 32-bit OSNumber */
 
 // properties found in the console user dict
 
@@ -60,4 +61,8 @@ typedef struct _IOUCProcessToken {
 
 #define kIOKernelHasSafeSleep		1
 
+enum { kIOPrepareToPhys32 = 0x04 };
+
+#define kIODirectionPrepareToPhys32 ((IODirection) kIOPrepareToPhys32)
+
 #endif /* ! _IOKIT_IOKITKEYSPRIVATE_H */
diff --git a/iokit/IOKit/IOLib.h b/iokit/IOKit/IOLib.h
index 2fbf9326a..76d990688 100644
--- a/iokit/IOKit/IOLib.h
+++ b/iokit/IOKit/IOLib.h
@@ -111,7 +111,7 @@ void   IOFreeAligned(void * address, vm_size_t size);
     @discussion This is a utility to allocate memory in the kernel, with an alignment restriction which is specified as a byte count, and will allocate only physically contiguous memory. The request may fail if memory is fragmented, and may cause large amounts of paging activity. This function may block and so should not be called from interrupt level or while a simple lock is held.
     @param size Size of the memory requested.
     @param alignment Byte count of the alignment for the memory. For example, pass 256 to get memory allocated at an address with bits 0-7 zero.
-    @param physicalAddress IOMallocContiguous returns the physical address of the allocated memory here, if physicalAddress is a non-zero pointer.
+    @param physicalAddress IOMallocContiguous returns the physical address of the allocated memory here, if physicalAddress is a non-zero pointer. The physicalAddress argument is deprecated and should be passed as NULL. To obtain the physical address for a memory buffer, use the IODMACommand class in conjunction with the IOMemoryDescriptor or IOBufferMemoryDescriptor classes.
     @result Virtual address of the allocated memory, or zero on failure. */
 
 void * IOMallocContiguous(vm_size_t size, vm_size_t alignment,
@@ -265,7 +265,7 @@ IOThread IOCreateThread(IOThreadFunc function, void *argument);
     @abstract Terminate exceution of current thread.
     @discussion This function destroys the currently running thread, and does not return. */
 
-volatile void IOExitThread(void);
+void IOExitThread(void) __dead2;
 
 /*! @function IOSleep
     @abstract Sleep the calling thread for a number of milliseconds.
diff --git a/iokit/IOKit/IOMapper.h b/iokit/IOKit/IOMapper.h
index f59b9b053..7834c7483 100644
--- a/iokit/IOKit/IOMapper.h
+++ b/iokit/IOKit/IOMapper.h
@@ -44,6 +44,11 @@ void IOMapperInsertPPNPages(ppnum_t addr, unsigned offset,
                             ppnum_t *pageList, unsigned pageCount);
 void IOMapperInsertUPLPages(ppnum_t addr, unsigned offset,
                             upl_page_info_t *pageList, unsigned pageCount);
+
+mach_vm_address_t IOMallocPhysical(mach_vm_size_t size, mach_vm_address_t mask);
+
+void IOFreePhysical(mach_vm_address_t address, mach_vm_size_t size);
+
 __END_DECLS
 
 #if __cplusplus
@@ -74,8 +79,6 @@ protected:
     OSData *fTableHandle;
     bool fIsSystem;
 
-    virtual bool start(IOService *provider);
-    virtual void free();
 
     static void setMapperRequired(bool hasMapper);
     static void waitForSystemMapper();
@@ -85,6 +88,11 @@ protected:
     virtual bool allocTable(IOByteCount size);
 
 public:
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+    virtual bool start(IOService *provider);
+    virtual void free();
+#endif
+
     // Static routines capable of allocating tables that are physically
     // contiguous in real memory space.
     static OSData * NewARTTable(IOByteCount size,
@@ -110,8 +118,16 @@ public:
     // iovm mapping.
     virtual addr64_t mapAddr(IOPhysicalAddress addr) = 0;
 
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+    // Get the address mask to or into an address to bypass this mapper
+    virtual bool getBypassMask(addr64_t *maskP) const
+    OSMetaClassDeclareReservedUsed(IOMapper, 0);
+#endif
+
 private:
+#if (defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
     OSMetaClassDeclareReservedUnused(IOMapper, 0);
+#endif
     OSMetaClassDeclareReservedUnused(IOMapper, 1);
     OSMetaClassDeclareReservedUnused(IOMapper, 2);
     OSMetaClassDeclareReservedUnused(IOMapper, 3);
diff --git a/iokit/IOKit/IOMemoryCursor.h b/iokit/IOKit/IOMemoryCursor.h
index 7a762f57c..dfe9eed8c 100644
--- a/iokit/IOKit/IOMemoryCursor.h
+++ b/iokit/IOKit/IOMemoryCursor.h
@@ -212,7 +212,7 @@ public:
     @param fromPosition Starting location of the I/O within a memory descriptor. 
     @param segments Pointer to an array of IOMemoryCursor::PhysicalSegments for the output physical scatter/gather list.
     @param maxSegments Maximum number of segments that can be written to segments array.
-    @param maxTransferSize Maximum transfer size is limited to that many bytes, otherwise it defaults to the maximum transfer size specified when the memory cursor was initialized.
+    @param inMaxTransferSize Maximum transfer size is limited to that many bytes, otherwise it defaults to the maximum transfer size specified when the memory cursor was initialized.
     @param transferSize Pointer to an IOByteCount variable that can contain the total size of the transfer being described.  Defaults to 0 indicating that no transfer size need be returned. 
     @result If the descriptor is exhausted of memory, a zero is returned, otherwise the number of segments that were filled in is returned.
 */
@@ -220,11 +220,11 @@ public:
 				       IOByteCount	   fromPosition,
 				       PhysicalSegment	  *segments,
 				       UInt32		   maxSegments,
-				       UInt32		   maxTransferSize = 0,
+				       UInt32		   inMaxTransferSize = 0,
 				       IOByteCount	  *transferSize = 0)
     {
 	return genPhysicalSegments(descriptor, fromPosition, segments,
-				maxSegments, maxTransferSize, transferSize);
+				maxSegments, inMaxTransferSize, transferSize);
     }
 };
 
@@ -288,7 +288,7 @@ successfully.
     @param fromPosition Starting location of the I/O within a memory descriptor. 
     @param segments Pointer to an array of IOMemoryCursor::PhysicalSegments for the output physical scatter/gather list.
     @param maxSegments Maximum number of segments that can be written to segments array.
-    @param maxTransferSize Maximum transfer size is limited to that many bytes, otherwise it defaults to the maximum transfer size specified when the memory cursor was initialized.
+    @param inMaxTransferSize Maximum transfer size is limited to that many bytes, otherwise it defaults to the maximum transfer size specified when the memory cursor was initialized.
     @param transferSize Pointer to an IOByteCount variable that can contain the total size of the transfer being described.  Defaults to 0 indicating that no transfer size need be returned. 
     @result If the descriptor is exhausted of memory, a zero is returned, otherwise the number of segments that were filled in is returned.
 */
@@ -296,11 +296,11 @@ successfully.
 				       IOByteCount	    fromPosition,
 				       PhysicalSegment *    segments,
 				       UInt32		    maxSegments,
-				       UInt32		    maxTransferSize = 0,
+				       UInt32		    inMaxTransferSize = 0,
 				       IOByteCount	 *  transferSize = 0)
     {
 	return genPhysicalSegments(descriptor, fromPosition, segments,
-				maxSegments, maxTransferSize, transferSize);
+				maxSegments, inMaxTransferSize, transferSize);
     }
 };
 
@@ -362,7 +362,7 @@ public:
     @param fromPosition Starting location of the I/O within a memory descriptor. 
     @param segments Pointer to an array of IOMemoryCursor::PhysicalSegments for the output physical scatter/gather list.
     @param maxSegments Maximum number of segments that can be written to segments array.
-    @param maxTransferSize Maximum transfer size is limited to that many bytes, otherwise it defaults to the maximum transfer size specified when the memory cursor was initialized.
+    @param inMaxTransferSize Maximum transfer size is limited to that many bytes, otherwise it defaults to the maximum transfer size specified when the memory cursor was initialized.
     @param transferSize Pointer to an IOByteCount variable that can contain the total size of the transfer being described.  Defaults to 0 indicating that no transfer size need be returned. 
     @result If the descriptor is exhausted of memory, a zero is returned, otherwise the number of segments that were filled in is returned.
 */
@@ -370,11 +370,11 @@ public:
 				       IOByteCount	    fromPosition,
 				       PhysicalSegment *    segments,
 				       UInt32		    maxSegments,
-				       UInt32		    maxTransferSize = 0,
+				       UInt32		    inMaxTransferSize = 0,
 				       IOByteCount	 *  transferSize = 0)
     {
 	return genPhysicalSegments(descriptor, fromPosition, segments,
-				maxSegments, maxTransferSize, transferSize);
+				maxSegments, inMaxTransferSize, transferSize);
     }
 };
 
@@ -440,7 +440,7 @@ public:
     @param fromPosition Starting location of the I/O within a memory descriptor. 
     @param segments Pointer to an array of DBDMA descriptors for the output physical scatter/gather list.  Be warned no room is left for a preamble in the output array.  'segments' should point to the first memory description slot in a DBDMA command.
     @param maxSegments Maximum number of segments that can be written to the DBDMA descriptor table.
-    @param maxTransferSize Maximum transfer size is limited to that many bytes, otherwise it defaults to the maximum transfer size specified when the memory cursor was initialized.
+    @param inMaxTransferSize Maximum transfer size is limited to that many bytes, otherwise it defaults to the maximum transfer size specified when the memory cursor was initialized.
     @param transferSize Pointer to an IOByteCount variable that can contain the total size of the transfer being described.  Defaults to 0 indicating that no transfer size need be returned. 
     @result If the descriptor is exhausted of memory, a zero is returned, otherwise the number of segments that were filled in is returned.
 */
@@ -448,11 +448,11 @@ public:
 				       IOByteCount	    fromPosition,
 				       IODBDMADescriptor *  segments,
 				       UInt32		    maxSegments,
-				       UInt32		    maxTransferSize = 0,
+				       UInt32		    inMaxTransferSize = 0,
 				       IOByteCount	 *  transferSize = 0)
     {
 	return genPhysicalSegments(descriptor, fromPosition, segments,
-				maxSegments, maxTransferSize, transferSize);
+				maxSegments, inMaxTransferSize, transferSize);
     }
 };
 
diff --git a/iokit/IOKit/IOMemoryDescriptor.h b/iokit/IOKit/IOMemoryDescriptor.h
index 551ecfff6..b154cbe1d 100644
--- a/iokit/IOKit/IOMemoryDescriptor.h
+++ b/iokit/IOKit/IOMemoryDescriptor.h
@@ -70,6 +70,8 @@ enum {
     kIOMemoryTypeUPL		= 0x00000030,
     kIOMemoryTypePersistentMD	= 0x00000040,	// Persistent Memory Descriptor
     kIOMemoryTypeUIO		= 0x00000050,
+    kIOMemoryTypeVirtual64	= 0x00000060,
+    kIOMemoryTypePhysical64	= 0x00000070,
     kIOMemoryTypeMask		= 0x000000f0,
 
     kIOMemoryAsReference	= 0x00000100,
@@ -94,6 +96,9 @@ enum
     kIOMemoryIncoherentIOStore	 = 2,
 };
 
+#define	IOMEMORYDESCRIPTOR_SUPPORTS_DMACOMMAND	1
+
+
 /*! @class IOMemoryDescriptor : public OSObject
     @abstract An abstract base class defining common methods for describing physical or virtual memory.
     @discussion The IOMemoryDescriptor object represents a buffer or range of memory, specified as one or more physical or virtual address ranges. It contains methods to return the memory's physically contiguous segments (fragments), for use with the IOMemoryCursor, and methods to map the memory into any address space with caching and placed mapping options. */
@@ -130,7 +135,7 @@ protected:
     IOOptionBits 	_tag;
 
 public:
-
+typedef IOOptionBits DMACommandOps;
     virtual IOPhysicalAddress getSourceSegment( IOByteCount offset,
 						IOByteCount * length );
     OSMetaClassDeclareReservedUsed(IOMemoryDescriptor, 0);
@@ -144,7 +149,7 @@ public:
                                  UInt32		offset,
                                  task_t		task,
                                  IOOptionBits	options,
-                                 IOMapper *	mapper = 0);
+                                 IOMapper *	mapper = kIOMapperSystem);
     OSMetaClassDeclareReservedUsed(IOMemoryDescriptor, 1);
 
     virtual addr64_t getPhysicalSegment64( IOByteCount offset,
@@ -184,9 +189,16 @@ public:
                                         IOByteCount offset, IOByteCount length );
     OSMetaClassDeclareReservedUsed(IOMemoryDescriptor, 4);
 
-private:
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+    // Used for dedicated communications for IODMACommand
+    virtual IOReturn dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const;
+    OSMetaClassDeclareReservedUsed(IOMemoryDescriptor, 5);
+#endif
 
+private:
+#if (defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
     OSMetaClassDeclareReservedUnused(IOMemoryDescriptor, 5);
+#endif
     OSMetaClassDeclareReservedUnused(IOMemoryDescriptor, 6);
     OSMetaClassDeclareReservedUnused(IOMemoryDescriptor, 7);
     OSMetaClassDeclareReservedUnused(IOMemoryDescriptor, 8);
@@ -259,6 +271,43 @@ public:
                                             task_t           withTask,
                                             bool             asReference = false);
 
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+/*! @function withAddressRange
+    @abstract Create an IOMemoryDescriptor to describe one virtual range of the specified map.
+    @discussion This method creates and initializes an IOMemoryDescriptor for memory consisting of a single virtual memory range mapped into the specified map.
+    @param address The virtual address of the first byte in the memory.
+    @param withLength The length of memory.
+    @param options
+        kIOMemoryDirectionMask (options:direction)     This nibble indicates the I/O direction to be associated with the descriptor, which may affect the operation of the prepare and complete methods on some architectures. 
+        kIOMemoryNoAutoPrepare Indicates that the temporary AutoPrepare of kernel_task memory should not be performed.
+    @param task The task the virtual ranges are mapped into.
+    @result The created IOMemoryDescriptor on success, to be released by the caller, or zero on failure. */
+
+    static IOMemoryDescriptor * withAddressRange(
+                                       mach_vm_address_t address,
+                                       mach_vm_size_t    length,
+                                       IOOptionBits      options,
+                               task_t            task);
+
+/*! @function withAddressRanges
+    @abstract Create an IOMemoryDescriptor to describe one or more virtual ranges.
+    @discussion This method creates and initializes an IOMemoryDescriptor for memory consisting of an array of virtual memory ranges each mapped into a specified source task.
+    @param ranges An array of IOAddressRange structures which specify the virtual ranges in the specified map which make up the memory to be described. IOAddressRange is the 64bit version of IOVirtualRange.
+    @param rangeCount The member count of the ranges array.
+    @param options
+        kIOMemoryDirectionMask (options:direction)     This nibble indicates the I/O direction to be associated with the descriptor, which may affect the operation of the prepare and complete methods on some architectures. 
+        kIOMemoryAsReference   For options:type = Virtual or Physical this indicate that the memory descriptor need not copy the ranges array into local memory.  This is an optimisation to try to minimise unnecessary allocations.
+        kIOMemoryNoAutoPrepare Indicates that the temporary AutoPrepare of kernel_task memory should not be performed.
+    @param task The task each of the virtual ranges are mapped into.
+    @result The created IOMemoryDescriptor on success, to be released by the caller, or zero on failure. */
+
+    static IOMemoryDescriptor * withAddressRanges(
+                                       IOAddressRange * ranges,
+                                       UInt32           rangeCount,
+                                       IOOptionBits     options,
+                                       task_t           withTask);
+#endif
+
 /*! @function withOptions
     @abstract Master initialiser for all variants of memory descriptors.
     @discussion This method creates and initializes an IOMemoryDescriptor for memory it has three main variants: Virtual, Physical & mach UPL.  These variants are selected with the options parameter, see below.  This memory descriptor needs to be prepared before it can be used to extract data from the memory described.  However we temporarily have setup a mechanism that automatically prepares kernel_task memory descriptors at creation time.
@@ -288,7 +337,7 @@ public:
                                            UInt32	offset,
                                            task_t	task,
                                            IOOptionBits	options,
-                                           IOMapper *	mapper = 0);
+                                           IOMapper *	mapper = kIOMapperSystem);
 
 /*! @function withPhysicalRanges
     @abstract Create an IOMemoryDescriptor to describe one or more physical ranges.
@@ -682,8 +731,9 @@ class IOGeneralMemoryDescriptor : public IOMemoryDescriptor
 
 public:
     union Ranges {
-        IOVirtualRange *  v;
-        IOPhysicalRange * p;
+        IOVirtualRange   *v;
+        IOAddressRange   *v64;
+        IOPhysicalRange  *p;
 	void 		 *uio;
     };
 protected:
@@ -708,36 +758,28 @@ protected:
 
     virtual void free();
 
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+    virtual IOReturn dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const;
+#endif
 
 private:
-    // Internal APIs may be made virtual at some time in the future.
-    IOReturn wireVirtual(IODirection forDirection);
-    void *createNamedEntry();	
 
-
-    /* DEPRECATED */ IOByteCount _position; /* absolute position over all ranges */
     /* DEPRECATED */ virtual void setPosition(IOByteCount position);
-
-/*
- * DEPRECATED IOByteCount _positionAtIndex; // relative position within range #n
- *
- * Re-use the _positionAtIndex as a count of the number of pages in
- * this memory descriptor.  Convieniently vm_address_t is an unsigned integer
- * type so I can get away without having to change the type.
- */
-    unsigned int		_pages;
-
-/* DEPRECATED */ unsigned    _positionAtOffset;  //range #n in which position is now
-
-    OSData *_memoryEntries;
-
-    /* DEPRECATED */ vm_offset_t _kernPtrAligned;
-    /* DEPRECATED */ unsigned    _kernPtrAtIndex;
-    /* DEPRECATED */ IOByteCount  _kernSize;
-
     /* DEPRECATED */ virtual void mapIntoKernel(unsigned rangeIndex);
     /* DEPRECATED */ virtual void unmapFromKernel();
 
+    // Internal APIs may be made virtual at some time in the future.
+    IOReturn wireVirtual(IODirection forDirection);
+    void *createNamedEntry();
+
+    // Internal
+    OSData *	    _memoryEntries;
+    unsigned int    _pages;
+    ppnum_t	    _highestPage;
+    uint32_t	    __iomd_reservedA;
+    uint32_t	    __iomd_reservedB;
+    uint32_t	    __iomd_reservedC;
+
 public:
     /*
      * IOMemoryDescriptor required methods
@@ -749,7 +791,7 @@ public:
                                  UInt32		offset,
                                  task_t		task,
                                  IOOptionBits	options,
-                                 IOMapper *	mapper = 0);
+                                 IOMapper *	mapper = kIOMapperSystem);
 
     // Secondary initialisers
     virtual bool initWithAddress(void *		address,
@@ -777,6 +819,11 @@ public:
                                         IODirection      withDirection,
                                         bool             asReference = false);
 
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+    virtual addr64_t getPhysicalSegment64( IOByteCount offset,
+                                            IOByteCount * length );
+#endif
+
     virtual IOPhysicalAddress getPhysicalSegment(IOByteCount offset,
 						 IOByteCount * length);
 
@@ -853,6 +900,11 @@ protected:
     IOMemoryDescriptor::withRanges;
     IOMemoryDescriptor::withSubRange;
 
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+    // used by IODMACommand
+    virtual IOReturn dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const;
+#endif
+
 public:
     /*
      * Initialize or reinitialize an IOSubMemoryDescriptor to describe
@@ -870,6 +922,11 @@ public:
      * IOMemoryDescriptor required methods
      */
 
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+    virtual addr64_t getPhysicalSegment64( IOByteCount offset,
+                                            IOByteCount * length );
+#endif
+
     virtual IOPhysicalAddress getPhysicalSegment(IOByteCount offset,
 						 IOByteCount * length);
 
diff --git a/iokit/IOKit/IOMultiMemoryDescriptor.h b/iokit/IOKit/IOMultiMemoryDescriptor.h
index 212145cff..390fbc1cf 100644
--- a/iokit/IOKit/IOMultiMemoryDescriptor.h
+++ b/iokit/IOKit/IOMultiMemoryDescriptor.h
@@ -126,6 +126,11 @@ public:
     virtual IOPhysicalAddress getPhysicalSegment( IOByteCount   offset,
                                                   IOByteCount * length );
 
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+    virtual addr64_t getPhysicalSegment64(        IOByteCount   offset,
+                                                  IOByteCount * length );
+#endif
+
 /*! @function prepare
     @abstract Prepare the memory for an I/O transfer.
     @discussion This involves paging in the memory, if necessary, and wiring it down for the duration of the transfer.  The complete() method completes the processing of the memory after the I/O transfer finishes.  This method needn't called for non-pageable memory.
diff --git a/iokit/IOKit/IOPlatformExpert.h b/iokit/IOKit/IOPlatformExpert.h
index 0bb1ff3e5..ff951f4d6 100644
--- a/iokit/IOKit/IOPlatformExpert.h
+++ b/iokit/IOKit/IOPlatformExpert.h
@@ -52,7 +52,8 @@ extern int PEGetPlatformEpoch( void );
 enum {
   kPEHaltCPU,
   kPERestartCPU,
-  kPEHangCPU
+  kPEHangCPU,
+  kPEUPSDelayHaltCPU
 };
 extern int (*PE_halt_restart)(unsigned int type);
 extern int PEHaltRestart(unsigned int type);
@@ -273,6 +274,9 @@ public:
 
     virtual void free();
 
+    virtual bool attachToChild( IORegistryEntry * child,
+                                const IORegistryPlane * plane );
+
     OSMetaClassDeclareReservedUnused(IOPlatformExpertDevice,  0);
     OSMetaClassDeclareReservedUnused(IOPlatformExpertDevice,  1);
     OSMetaClassDeclareReservedUnused(IOPlatformExpertDevice,  2);
diff --git a/iokit/IOKit/IOReturn.h b/iokit/IOKit/IOReturn.h
index a603184ad..ae24a19d6 100644
--- a/iokit/IOKit/IOReturn.h
+++ b/iokit/IOKit/IOReturn.h
@@ -52,8 +52,16 @@ typedef	kern_return_t		IOReturn;
 #define sub_iokit_firewire                err_sub(2)
 #define sub_iokit_block_storage           err_sub(4)
 #define sub_iokit_graphics                err_sub(5)
+#define sub_iokit_networking		  err_sub(6)
 #define sub_iokit_bluetooth               err_sub(8)
 #define sub_iokit_pmu                     err_sub(9)
+#define sub_iokit_acpi                    err_sub(10)
+#define sub_iokit_smbus                   err_sub(11)
+#define sub_iokit_ahci                    err_sub(12)
+#define sub_iokit_powermanagement         err_sub(13)
+//#define sub_iokit_hidsystem             err_sub(14)
+//#define sub_iokit_pccard                err_sub(21)
+
 #define sub_iokit_vendor_specific         err_sub(-2)
 #define sub_iokit_reserved                err_sub(-1)
 
diff --git a/iokit/IOKit/IOService.h b/iokit/IOKit/IOService.h
index 57612a440..4cb1724a3 100644
--- a/iokit/IOKit/IOService.h
+++ b/iokit/IOKit/IOService.h
@@ -382,6 +382,8 @@ private:
     OSMetaClassDeclareReservedUnused(IOService, 45);
     OSMetaClassDeclareReservedUnused(IOService, 46);
     OSMetaClassDeclareReservedUnused(IOService, 47);
+
+#ifdef __ppc__
     OSMetaClassDeclareReservedUnused(IOService, 48);
     OSMetaClassDeclareReservedUnused(IOService, 49);
     OSMetaClassDeclareReservedUnused(IOService, 50);
@@ -398,6 +400,7 @@ private:
     OSMetaClassDeclareReservedUnused(IOService, 61);
     OSMetaClassDeclareReservedUnused(IOService, 62);
     OSMetaClassDeclareReservedUnused(IOService, 63);
+#endif
 
 public:
 /*! @function getState
@@ -980,7 +983,7 @@ public:
 
 /*! @function disableInterrupt
     @abstract Disable a device interrupt.
-    @discussion Disable a device interrupt. It is the callers responsiblity to keep track of the enable state of the interrupt source.
+    @discussion Synchronously disable a device interrupt.  If the interrupt routine is running, the call will block until the routine completes.  It is the callers responsiblity to keep track of the enable state of the interrupt source.
     @param source The index of the interrupt source in the device.
     @result An IOReturn code.<br>kIOReturnNoInterrupt is returned if the source is not valid. */
 
@@ -1164,11 +1167,16 @@ public:
     static void actionFinalize( IOService * victim, IOOptionBits options );
     static void actionStop( IOService * client, IOService * provider );
 
-    void PMfree( void );
-
     virtual IOReturn resolveInterrupt(IOService *nub, int source);
     virtual IOReturn lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController);
 
+    // SPI to control CPU low power modes
+    void  setCPUSnoopDelay(UInt32 ns);
+    UInt32 getCPUSnoopDelay();
+    void   requireMaxBusStall(UInt32 ns);
+
+    void PMfree( void );
+
     /* power management */
     
 /*! @function PMinit
@@ -1847,6 +1855,7 @@ private:
     void driver_acked ( void );
     void start_ack_timer ( void );
     void stop_ack_timer ( void );
+
     unsigned long compute_settle_time ( void );
     IOReturn startSettleTimer ( unsigned long delay );
     IOReturn changeState ( void );
@@ -1862,6 +1871,7 @@ private:
     IOReturn allowCancelCommon ( void );
     void computeDesiredState ( void );
     void rebuildChildClampBits ( void );
+    IOReturn temporaryMakeUsable ( void );
 };
 
 #endif /* ! _IOKIT_IOSERVICE_H */
diff --git a/iokit/IOKit/IOTimeStamp.h b/iokit/IOKit/IOTimeStamp.h
index 7232fa959..b900cdfdd 100644
--- a/iokit/IOKit/IOTimeStamp.h
+++ b/iokit/IOKit/IOTimeStamp.h
@@ -82,71 +82,81 @@ IOTimeStamp(unsigned int csc,
 
 #endif /* KDEBUG */
 
-#define IODBG_SCSI(code)	(KDBG_CODE(DBG_IOKIT, DBG_IOSCSI, code))
-#define IODBG_DISK(code)	(KDBG_CODE(DBG_IOKIT, DBG_IODISK, code))
-#define IODBG_NETWORK(code)	(KDBG_CODE(DBG_IOKIT, DBG_IONETWORK, code))
-#define IODBG_KEYBOARD(code)	(KDBG_CODE(DBG_IOKIT, DBG_IOKEYBOARD, code))
-#define IODBG_POINTING(code)	(KDBG_CODE(DBG_IOKIT, DBG_IOPOINTING, code))
-#define IODBG_AUDIO(code)	(KDBG_CODE(DBG_IOKIT, DBG_IOAUDIO, code))
-#define IODBG_FLOPPY(code)	(KDBG_CODE(DBG_IOKIT, DBG_IOFLOPPY, code))
-#define IODBG_SERIAL(code)	(KDBG_CODE(DBG_IOKIT, DBG_IOSERIAL, code))
-#define IODBG_TTY(code)		(KDBG_CODE(DBG_IOKIT, DBG_IOTTY, code))
+#define IODBG_STORAGE(code)			(KDBG_CODE(DBG_IOKIT, DBG_IOSTORAGE, code))
+#define IODBG_NETWORK(code)			(KDBG_CODE(DBG_IOKIT, DBG_IONETWORK, code))
+#define IODBG_KEYBOARD(code)		(KDBG_CODE(DBG_IOKIT, DBG_IOKEYBOARD, code))
+#define IODBG_HID(code)				(KDBG_CODE(DBG_IOKIT, DBG_IOHID, code))
+#define IODBG_AUDIO(code)			(KDBG_CODE(DBG_IOKIT, DBG_IOAUDIO, code))
+#define IODBG_SERIAL(code)			(KDBG_CODE(DBG_IOKIT, DBG_IOSERIAL, code))
+#define IODBG_TTY(code)				(KDBG_CODE(DBG_IOKIT, DBG_IOTTY, code))
+#define IODBG_SAM(code)				(KDBG_CODE(DBG_IOKIT, DBG_IOSAM, code))
+#define IODBG_PARALLELATA(code)		(KDBG_CODE(DBG_IOKIT, DBG_IOPARALLELATA, code))
+#define IODBG_PARALLELSCSI(code)	(KDBG_CODE(DBG_IOKIT, DBG_IOPARALLELSCSI, code))
+#define IODBG_SATA(code)			(KDBG_CODE(DBG_IOKIT, DBG_IOSATA, code))
+#define IODBG_SAS(code)				(KDBG_CODE(DBG_IOKIT, DBG_IOSAS, code))
+#define IODBG_FIBRECHANNEL(code)	(KDBG_CODE(DBG_IOKIT, DBG_IOFIBRECHANNEL, code))
+#define IODBG_USB(code)				(KDBG_CODE(DBG_IOKIT, DBG_IOUSB, code))
+#define IODBG_BLUETOOTH(code)		(KDBG_CODE(DBG_IOKIT, DBG_IOBLUETOOTH, code))
+#define IODBG_FIREWIRE(code)		(KDBG_CODE(DBG_IOKIT, DBG_IOFIREWIRE, code))
+#define IODBG_INFINIBAND(code)		(KDBG_CODE(DBG_IOKIT, DBG_IOINFINIBAND, code))
+
+
+/* Backwards compatibility */
+#define IODBG_DISK(code)			IODBG_STORAGE(code)
+#define IODBG_POINTING(code)		IODBG_HID(code)
+
 
 /* IOKit infrastructure subclasses */
-#define IODBG_WORKLOOP(code)	(KDBG_CODE(DBG_IOKIT, DBG_IOWORKLOOP, code))
-#define IODBG_INTES(code)	(KDBG_CODE(DBG_IOKIT, DBG_IOINTES, code))
-#define IODBG_TIMES(code)	(KDBG_CODE(DBG_IOKIT, DBG_IOCLKES, code))
-#define IODBG_CMDQ(code)	(KDBG_CODE(DBG_IOKIT, DBG_IOCMDQ, code))
-#define IODBG_MCURS(code)	(KDBG_CODE(DBG_IOKIT, DBG_IOMCURS, code))
-#define IODBG_MDESC(code)	(KDBG_CODE(DBG_IOKIT, DBG_IOMDESC, code))
-#define IODBG_POWER(code)	(KDBG_CODE(DBG_IOKIT, DBG_IOPOWER, code))
+#define IODBG_WORKLOOP(code)		(KDBG_CODE(DBG_IOKIT, DBG_IOWORKLOOP, code))
+#define IODBG_INTES(code)			(KDBG_CODE(DBG_IOKIT, DBG_IOINTES, code))
+#define IODBG_TIMES(code)			(KDBG_CODE(DBG_IOKIT, DBG_IOCLKES, code))
+#define IODBG_CMDQ(code)			(KDBG_CODE(DBG_IOKIT, DBG_IOCMDQ, code))
+#define IODBG_MCURS(code)			(KDBG_CODE(DBG_IOKIT, DBG_IOMCURS, code))
+#define IODBG_MDESC(code)			(KDBG_CODE(DBG_IOKIT, DBG_IOMDESC, code))
+#define IODBG_POWER(code)			(KDBG_CODE(DBG_IOKIT, DBG_IOPOWER, code))
 
 /* IOKit specific codes - within each subclass */
 
-/* DBG_IOKIT/DBG_IOSCSI codes */
-
 /* DBG_IOKIT/DBG_IODISK codes */
 
 /* DBG_IOKIT/DBG_IONETWORK codes */
 
 /* DBG_IOKIT/DBG_IOKEYBOARD codes */
 
-/* DBG_IOKIT/DBG_IOPOINTING codes */
+/* DBG_IOKIT/DBG_IOHID codes */
 
 /* DBG_IOKIT/DBG_IOAUDIO codes */
 
-/* DBG_IOKIT/DBG_IOFLOPPY codes */
-
 /* DBG_IOKIT/DBG_IOSERIAL codes */
 
 /* DBG_IOKIT/DBG_IOTTY codes */
 
 /* DBG_IOKIT/DBG_IOWORKLOOP codes */
-#define IOWL_CLIENT	1	/* 0x050a0004 */
-#define IOWL_WORK	2	/* 0x050a0008 */
+#define IOWL_CLIENT		1	/* 0x05010004 */
+#define IOWL_WORK		2	/* 0x05010008 */
 
 /* DBG_IOKIT/DBG_IOINTES codes */
-#define IOINTES_CLIENT	1	/* 0x050b0004 */
-#define IOINTES_LAT	2	/* 0x050b0008 */
-#define IOINTES_SEMA	3	/* 0x050b000c */
-#define IOINTES_INTCTXT 4	/* 0x050b0010 */
-#define IOINTES_INTFLTR 5	/* 0x050b0014 */
-#define IOINTES_ACTION	6	/* 0x050b0018 */
-#define IOINTES_FILTER	7	/* 0x050b001c */
+#define IOINTES_CLIENT	1	/* 0x05020004 */
+#define IOINTES_LAT		2	/* 0x05020008 */
+#define IOINTES_SEMA	3	/* 0x0502000c */
+#define IOINTES_INTCTXT 4	/* 0x05020010 */
+#define IOINTES_INTFLTR 5	/* 0x05020014 */
+#define IOINTES_ACTION	6	/* 0x05020018 */
+#define IOINTES_FILTER	7	/* 0x0502001c */
 
 /* DBG_IOKIT/DBG_IOTIMES codes */
-#define IOTIMES_CLIENT	1	/* 0x050c0004 */
-#define IOTIMES_LAT	2	/* 0x050c0008 */
-#define IOTIMES_SEMA	3	/* 0x050c000c */
-#define IOTIMES_ACTION	4	/* 0x050c0010 */
+#define IOTIMES_CLIENT	1	/* 0x05030004 */
+#define IOTIMES_LAT		2	/* 0x05030008 */
+#define IOTIMES_SEMA	3	/* 0x0503000c */
+#define IOTIMES_ACTION	4	/* 0x05030010 */
 
 /* DBG_IOKIT/DBG_IOCMDQ codes */
-#define IOCMDQ_CLIENT	1	/* 0x050d0004 */
-#define IOCMDQ_LAT	2	/* 0x050d0008 */
-#define IOCMDQ_SEMA	3	/* 0x050d000c */
-#define IOCMDQ_PSEMA	4	/* 0x050d0010 */
-#define IOCMDQ_PLOCK	5	/* 0x050d0014 */
-#define IOCMDQ_ACTION	6	/* 0x050d0018 */
+#define IOCMDQ_CLIENT	1	/* 0x05040004 */
+#define IOCMDQ_LAT		2	/* 0x05040008 */
+#define IOCMDQ_SEMA		3	/* 0x0504000c */
+#define IOCMDQ_PSEMA	4	/* 0x05040010 */
+#define IOCMDQ_PLOCK	5	/* 0x05040014 */
+#define IOCMDQ_ACTION	6	/* 0x05040018 */
 
 /* DBG_IOKIT/DBG_IOMCURS codes */
 
diff --git a/iokit/IOKit/IOTypes.h b/iokit/IOKit/IOTypes.h
index d503e6a47..294f325b3 100644
--- a/iokit/IOKit/IOTypes.h
+++ b/iokit/IOKit/IOTypes.h
@@ -62,20 +62,9 @@ extern "C" {
 #ifndef __MACTYPES__	/* CF MacTypes.h */
 #ifndef __TYPES__	/* guess... Mac Types.h */
 
+#include <stdbool.h>
 #include <libkern/OSTypes.h>
 
-#ifndef __cplusplus
-#if !TYPE_BOOL
-#ifdef KERNEL
-typedef int	bool;
-enum {
-    false	= 0,
-    true	= 1
-};
-#endif
-#endif
-#endif
-
 #endif /* __TYPES__ */
 #endif /* __MACTYPES__ */
 
@@ -113,12 +102,23 @@ struct IOVirtualRange
     IOVirtualAddress	address;
     IOByteCount		length;
 };
+struct IOAddressRange
+{
+    mach_vm_address_t	address;
+    mach_vm_size_t	length;
+};
 #else
 typedef struct 
 {
     IOVirtualAddress	address;
     IOByteCount		length;
 } IOVirtualRange;
+
+struct IOAddressRange
+{
+    mach_vm_address_t	address;
+    mach_vm_size_t	length;
+};
 #endif
 
 /*
diff --git a/iokit/IOKit/IOUserClient.h b/iokit/IOKit/IOUserClient.h
index 17378e7bd..773145bcb 100644
--- a/iokit/IOKit/IOUserClient.h
+++ b/iokit/IOKit/IOUserClient.h
@@ -108,7 +108,10 @@ protected:
 
 public:
     OSSet * mappings;
-    void  * __reserved[8];
+    UInt8   sharedInstance;
+
+    UInt8   __reservedA[3];
+    void  * __reserved[7];
 
 private:
     OSMetaClassDeclareReservedUnused(IOUserClient, 0);
@@ -143,6 +146,10 @@ public:
     static IOReturn clientHasPrivilege( void * securityToken,
                                         const char * privilegeName );
 
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+    virtual bool init();
+    virtual bool init( OSDictionary * dictionary );
+#endif
     // Currently ignores the all args, just passes up to IOService::init()
     virtual bool initWithTask(
                     task_t owningTask, void * securityToken, UInt32 type,
diff --git a/iokit/IOKit/IOWorkLoop.h b/iokit/IOKit/IOWorkLoop.h
index eba08bbcb..d2d8222d3 100644
--- a/iokit/IOKit/IOWorkLoop.h
+++ b/iokit/IOKit/IOWorkLoop.h
@@ -154,7 +154,13 @@ protected:
     virtual void free();
 
 /*! @function threadMain
-    @discussion Work loop threads main function.  This function consists of 3 loops: the outermost loop is the semaphore clear and wait loop, the middle loop terminates when there is no more work, and the inside loop walks the event list calling the checkForWork method in each event source.  If an event source has more work to do, it can set the more flag and the middle loop will repeat.  When no more work is outstanding the outermost will sleep until an event is signalled or the least wakeupTime, whichever occurs first.  If the event source does not require the semaphore wait to time out, it must set the provided wakeupTime parameter to zero.
+    @discussion Work loop threads main function.  This function consists of 3
+    loops: the outermost loop is the semaphore clear and wait loop, the middle
+    loop terminates when there is no more work, and the inside loop walks the
+    event list calling the checkForWork method in each event source.  If an
+    event source has more work to do, it can set the more flag and the middle
+    loop will repeat.  When no more work is outstanding the outermost will
+    sleep until an event is signalled.
 */
     virtual void threadMain();
 
@@ -237,7 +243,7 @@ protected:
     virtual void openGate();
     virtual void closeGate();
     virtual bool tryCloseGate();
-    virtual int sleepGate(void *event, UInt32 interuptibleType);
+    virtual int  sleepGate(void *event, UInt32 interuptibleType);
     virtual void wakeupGate(void *event, bool oneThread);
 
 public:
@@ -245,9 +251,7 @@ public:
 
 /*! @function runAction
     @abstract Single thread a call to an action with the work-loop.
-    @discussion Client function that causes the given action to be called in
-a single threaded manner.  Beware: the work-loop's gate is recursive and runAction can cause direct or indirect re-entrancy.  When executing on a client's thread, runAction will sleep until the work-loop's gate opens for
-execution of client actions, the action is single threaded against all other work-loop event sources.
+    @discussion Client function that causes the given action to be called in a single threaded manner.  Beware: the work-loop's gate is recursive and runAction can cause direct or indirect re-entrancy.  When executing on a client's thread, runAction will sleep until the work-loop's gate opens for execution of client actions, the action is single threaded against all other work-loop event sources.
     @param action Pointer to function to be executed in work-loop context.
     @param arg0 Parameter for action parameter, defaults to 0.
     @param arg1 Parameter for action parameter, defaults to 0.
@@ -255,14 +259,40 @@ execution of client actions, the action is single threaded against all other wor
     @param arg3 Parameter for action parameter, defaults to 0.
     @result Returns the value of the Action callout.
 */
+    OSMetaClassDeclareReservedUsed(IOWorkLoop, 0);
     virtual IOReturn runAction(Action action, OSObject *target,
 			       void *arg0 = 0, void *arg1 = 0,
 			       void *arg2 = 0, void *arg3 = 0);
 
+#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
+/*! @function runEventSources
+    @discussion Consists of the inner 2 loops of the threadMain function(qv).
+    The outer loop terminates when there is no more work, and the inside loop
+    walks the event list calling the checkForWork method in each event source.
+    If an event source has more work to do, it can set the more flag and the
+    outer loop will repeat.
+<br><br>
+    This function can be used to clear a priority inversion between the normal
+    workloop thread and multimedia's real time threads.  The problem is that
+    the interrupt action routine is often held off by high priority threads.
+    So if they want to get their data now they will have to call us and ask if
+    any data is available.  The multi-media user client will arrange for this
+    function to be called, which causes any pending interrupts to be processed
+    and the completion routines called.  By the time the function returns all
+    outstanding work will have been completed at the real time threads
+    priority.
+
+    @result Return false if the work loop is shutting down, true otherwise.
+*/
+    OSMetaClassDeclareReservedUsed(IOWorkLoop, 1);
+    virtual bool runEventSources();
+#endif
+
 protected:
-    OSMetaClassDeclareReservedUsed(IOWorkLoop, 0);
 
+#if (defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
     OSMetaClassDeclareReservedUnused(IOWorkLoop, 1);
+#endif
     OSMetaClassDeclareReservedUnused(IOWorkLoop, 2);
     OSMetaClassDeclareReservedUnused(IOWorkLoop, 3);
     OSMetaClassDeclareReservedUnused(IOWorkLoop, 4);
diff --git a/iokit/IOKit/assert.h b/iokit/IOKit/assert.h
index 9b372d235..f7212675f 100644
--- a/iokit/IOKit/assert.h
+++ b/iokit/IOKit/assert.h
@@ -28,6 +28,7 @@
 
 #ifndef	_IO_ASSERT_H_
 #define	_IO_ASSERT_H_
+#include <sys/cdefs.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -45,5 +46,11 @@ extern "C" {
 }
 #endif
 
+
+#if( !defined( OSCompileAssert ) )
+#	define OSCompileAssert( TEST )     \
+	extern int OSCompileAssertFailed[ ( TEST ) ? 1 : -1 ] __unused;
+#endif
+
 #endif	/* _IO_ASSERT_H_ */
 
diff --git a/iokit/IOKit/nvram/IONVRAMController.h b/iokit/IOKit/nvram/IONVRAMController.h
index 3027c46ca..307d666a6 100644
--- a/iokit/IOKit/nvram/IONVRAMController.h
+++ b/iokit/IOKit/nvram/IONVRAMController.h
@@ -26,6 +26,9 @@
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
+#ifndef _IOKIT_IONVRAMCONTROLLER_H
+#define _IOKIT_IONVRAMCONTROLLER_H
+
 #include <IOKit/IOService.h>
 
 class IONVRAMController: public IOService
@@ -42,3 +45,5 @@ public:
   virtual IOReturn write(IOByteCount offset, UInt8 *buffer,
 			 IOByteCount length) = 0;
 };
+
+#endif /* !_IOKIT_IONVRAMCONTROLLER_H */
diff --git a/iokit/IOKit/pwr_mgt/IOPM.h b/iokit/IOKit/pwr_mgt/IOPM.h
index bf6902e6b..0e5ac04f4 100644
--- a/iokit/IOKit/pwr_mgt/IOPM.h
+++ b/iokit/IOKit/pwr_mgt/IOPM.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -28,246 +28,344 @@
 #ifndef _IOKIT_IOPM_H
 #define _IOKIT_IOPM_H
 
-#define IOPMMaxPowerStates 10
+#include <IOKit/IOTypes.h>
+#include <IOKit/IOMessage.h>
+#include <IOKit/IOReturn.h>
+
+#ifdef __ppc__
+#include <IOKit/pwr_mgt/IOPMDeprecated.h>
+#endif
+
+
+
+enum {
+    kIOPMMaxPowerStates = 10,
+    IOPMMaxPowerStates = kIOPMMaxPowerStates
+};
 
 typedef unsigned long IOPMPowerFlags;
 enum {
-                        // following  bits are used in the input and output power fields
-    kIOPMClockNormal		= 0x0004,
-    kIOPMClockRunning		= 0x0008,
-    kIOPMAuxPowerOn		= 0x0020,	// used only between root and root parent
-    kIOPMPagingAvailable	= 0x0020,	// used only between paging plexus and its children
-    kIOPMPassThrough		= 0x0100,
-    kIOPMDoze			= 0x0400,
-    kIOPMSoftSleep		= 0x0400,	// old usage, replaced by kIOPMDoze
-    kIOPMSleep			= 0x0001,
-    kIOPMRestart		= 0x0080,
-                        // following  bits are used in the capabilites field and the power fields
-    kIOPMPowerOn		= 0x0002,
-    kIOPMPreventSystemSleep	= 0x0010,
-    kIOPMPreventIdleSleep	= 0x0040,
-                        // following  bits are used in the capabilites field
-    kIOPMNotAttainable		= 0x0001,	// used between a driver and its policy-maker
-    kIOPMChildClamp		= 0x0080,	// used internally in a power domain parent
-    kIOPMChildClamp2		= 0x0200,	// used internally in a power domain parent
-    kIOPMDeviceUsable		= 0x8000,
-    kIOPMMaxPerformance	 	= 0x4000,
-    kIOPMContextRetained	= 0x2000,
-    kIOPMConfigRetained		= 0x1000,
-    kIOPMSleepCapability	= 0x0004,
-    kIOPMRestartCapability	= 0x0080,
-    kIOPMNotPowerManaged	= 0x0800,	// this is an error return rather than a bit
-    kIOPMStaticPowerValid	= 0x0800,	// therefore this bit safely overloads it
+    // The following  bits are used in the input and output power fields.
+    kIOPMClockNormal                = 0x0004,
+    kIOPMClockRunning               = 0x0008,
+    // Reserved - Used only between root and root parent.
+    kIOPMAuxPowerOn                 = 0x0020,
+    // Reserved - kIOPMPagingAvailable used only by now-defunct paging plexus
+    kIOPMPagingAvailable            = 0x0020,
+    kIOPMPassThrough                = 0x0100,
+    kIOPMDoze                       = 0x0400,
+    // Obsolete - use kIOPMDoze instead of kIOPMSoftSleep
+    kIOPMSoftSleep                  = 0x0400,
+    kIOPMSleep                      = 0x0001,
+    kIOPMRestart                    = 0x0080,
+
+    // The following bits are used in the capabilites field and the power fields
+    kIOPMPowerOn                    = 0x0002,
+    kIOPMPreventSystemSleep         = 0x0010,
+    kIOPMPreventIdleSleep           = 0x0040,
+
+    // The following  bits are used in the capabilites field only.
+    // Used between a driver and its policy-maker
+    kIOPMNotAttainable              = 0x0001,
+    // Used internally in a power domain parent
+    kIOPMChildClamp                 = 0x0080,
+    // Used internally in a power domain parent
+    kIOPMChildClamp2                = 0x0200,
+    // Marks device as usable in this state
+    kIOPMDeviceUsable               = 0x8000,
+    // Device runs at max performance in this state
+    kIOPMMaxPerformance             = 0x4000,
+    kIOPMContextRetained            = 0x2000,
+    kIOPMConfigRetained             = 0x1000,
+    // Device is capable of system sleep in this state
+    kIOPMSleepCapability            = 0x0004,
+    kIOPMRestartCapability          = 0x0080,
+
+    // Reserved - Error code. (this is an error return rather than a bit)
+    kIOPMNotPowerManaged            = 0x0800,
+    // Therefore this bit safely overloads it
+    kIOPMStaticPowerValid           = 0x0800,
     
-    kIOPMCapabilitiesMask =     kIOPMPowerOn | kIOPMDeviceUsable | kIOPMMaxPerformance |
-                                kIOPMContextRetained | kIOPMConfigRetained | kIOPMSleepCapability |
+    kIOPMCapabilitiesMask =     kIOPMPowerOn | kIOPMDeviceUsable | 
+                                kIOPMMaxPerformance | kIOPMContextRetained | 
+                                kIOPMConfigRetained | kIOPMSleepCapability |
                                 kIOPMRestartCapability
 };
 
 
 enum {
-    IOPMNotAttainable	= kIOPMNotAttainable,
-    IOPMPowerOn		= kIOPMPowerOn,
-    IOPMClockNormal	= kIOPMClockNormal,
-    IOPMClockRunning	= kIOPMClockRunning,
-    IOPMAuxPowerOn	= kIOPMAuxPowerOn,
-    IOPMDeviceUsable	= kIOPMDeviceUsable,
-    IOPMMaxPerformance	= kIOPMMaxPerformance,
-    IOPMContextRetained	= kIOPMContextRetained,
-    IOPMConfigRetained	= kIOPMConfigRetained,
-    IOPMNotPowerManaged	= kIOPMNotPowerManaged,
-    IOPMPagingAvailable	= kIOPMPagingAvailable,
-    IOPMSoftSleep	= kIOPMSoftSleep
+    IOPMNotAttainable           = kIOPMNotAttainable,
+    IOPMPowerOn                 = kIOPMPowerOn,
+    IOPMClockNormal             = kIOPMClockNormal,
+    IOPMClockRunning            = kIOPMClockRunning,
+    IOPMAuxPowerOn              = kIOPMAuxPowerOn,
+    IOPMDeviceUsable            = kIOPMDeviceUsable,
+    IOPMMaxPerformance          = kIOPMMaxPerformance,
+    IOPMContextRetained         = kIOPMContextRetained,
+    IOPMConfigRetained          = kIOPMConfigRetained,
+    IOPMNotPowerManaged         = kIOPMNotPowerManaged,
+    IOPMPagingAvailable         = kIOPMPagingAvailable,
+    IOPMSoftSleep               = kIOPMSoftSleep
 };
 
 
 enum {
-    kIOPMNextHigherState	= 1,
-    kIOPMHighestState		= 2,
-    kIOPMNextLowerState		= 3,
-    kIOPMLowestState		= 4
+    kIOPMNextHigherState        = 1,
+    kIOPMHighestState           = 2,
+    kIOPMNextLowerState         = 3,
+    kIOPMLowestState            = 4
 };
 
 enum {
-    IOPMNextHigherState		= kIOPMNextHigherState,
-    IOPMHighestState		= kIOPMHighestState,
-    IOPMNextLowerState		= kIOPMNextLowerState,
-    IOPMLowestState		= kIOPMLowestState
+    IOPMNextHigherState         = kIOPMNextHigherState,
+    IOPMHighestState            = kIOPMHighestState,
+    IOPMNextLowerState          = kIOPMNextLowerState,
+    IOPMLowestState             = kIOPMLowestState
 };
 
-
-
-enum {		// commands on power managment command queue
+// Internal commands used by power managment command queue
+enum {
     kIOPMBroadcastAggressiveness = 1,
     kIOPMUnidleDevice
 };
 
-enum {		// special value means "power consumption unknown"
+// Power consumption unknown value
+enum {
     kIOPMUnknown = 0xFFFF
 };
 
-// Power events
+/*******************************************************************************
+ *
+ * Root Domain property keys of interest
+ *
+ ******************************************************************************/
+
+/* AppleClamshellState 
+ * reflects the state of the clamshell (lid) on a portable.
+ * It has a boolean value.
+ *  true        == clamshell is closed
+ *  false       == clamshell is open
+ *  not present == no clamshell on this hardware
+ */
+#define kAppleClamshellStateKey             "AppleClamshellState"
+
+/* AppleClamshellCausesSleep 
+ * reflects the clamshell close behavior on a portable. 
+ * It has a boolean value.
+ *  true        == system will sleep when clamshell is closed
+ *  false       == system will not sleep on clamshell close 
+ *                  (typically external display mode)
+ *  not present == no clamshell on this hardware
+ */
+#define kAppleClamshellCausesSleepKey       "AppleClamshellCausesSleep"
+
+/*******************************************************************************
+ *
+ * Root Domain general interest messages
+ *
+ ******************************************************************************/
+
+/* kIOPMMessageClamshellStateChange
+ * Delivered as a general interest notification on the IOPMrootDomain
+ * IOPMrootDomain sends this message when state of either AppleClamshellState
+ * or AppleClamshellCausesSleep changes. If this clamshell change results in
+ * a sleep, the sleep will initiate soon AFTER delivery of this message.
+ * The state of both variables is encoded in a bitfield argument sent with
+ * the message. Check bits 0 and 1 using kClamshellStateBit & kClamshellSleepBit
+ */
+enum {
+    kClamshellStateBit = (1 << 0),
+    kClamshellSleepBit = (1 << 1)
+};
+
+#define kIOPMMessageClamshellStateChange   \
+                iokit_family_msg(sub_iokit_powermanagement, 0x100)
+
+/* kIOPMMessageFeatureChange
+ * Delivered when the set of supported features ("Supported Features" dictionary
+ * under IOPMrootDomain registry) changes in some way. Typically addition or
+ * removal of a supported feature.
+ * RootDomain passes no argument with this message.
+ */
+#define kIOPMMessageFeatureChange           \
+                iokit_family_msg(sub_iokit_powermanagement, 0x110)
+
+/* kIOPMMessageInflowDisableCancelled
+ * The battery has drained completely to its "Fully Discharged" state. 
+ * If a user process has disabled battery inflow for battery 
+ * calibration, we forcibly re-enable Inflow at this point.
+ * If inflow HAS been forcibly re-enabled, bit 0
+ * (kInflowForciblyEnabledBit) will be set.
+ */
 enum {
-  kClamshellClosedEventMask  = (1<<0),  // User closed lid
-  kDockingBarEventMask       = (1<<1),  // OBSOLETE
-  kACPlugEventMask           = (1<<2),  // User plugged or unplugged adapter
-  kFrontPanelButtonEventMask = (1<<3),  // User hit the front panel button
-  kBatteryStatusEventMask    = (1<<4)   // Battery status has changed
+    kInflowForciblyEnabledBit = (1 << 0)
 };
 
-// Power commands issued to root domain
+#define kIOPMMessageInternalBatteryFullyDischarged  \
+                iokit_family_msg(sub_iokit_powermanagement, 0x120)
+
+
+/*******************************************************************************
+ *
+ * Power commands issued to root domain
+ *
+ * These commands are issued from system drivers only:
+ *      ApplePMU, AppleSMU, IOGraphics, AppleACPIFamily
+ *
+ ******************************************************************************/
 enum {
-  kIOPMSleepNow                  = (1<<0),  // put machine to sleep now
-  kIOPMAllowSleep                = (1<<1),  // allow idle sleep
-  kIOPMPreventSleep              = (1<<2),  // do not allow idle sleep
-  kIOPMPowerButton		 = (1<<3),  // power button was pressed
-  kIOPMClamshellClosed		 = (1<<4),  // clamshell was closed
-  kIOPMPowerEmergency		 = (1<<5),  // battery dangerously low
-  kIOPMDisableClamshell		 = (1<<6),  // do not sleep on clamshell closure
-  kIOPMEnableClamshell		 = (1<<7),  // sleep on clamshell closure
-  kIOPMProcessorSpeedChange	 = (1<<8),  // change the processor speed
-  kIOPMOverTemp                  = (1<<9)   // system dangerously hot
+  kIOPMSleepNow                 = (1<<0),  // put machine to sleep now
+  kIOPMAllowSleep               = (1<<1),  // allow idle sleep
+  kIOPMPreventSleep             = (1<<2),  // do not allow idle sleep
+  kIOPMPowerButton              = (1<<3),  // power button was pressed
+  kIOPMClamshellClosed          = (1<<4),  // clamshell was closed
+  kIOPMPowerEmergency           = (1<<5),  // battery dangerously low
+  kIOPMDisableClamshell         = (1<<6),  // do not sleep on clamshell closure
+  kIOPMEnableClamshell          = (1<<7),  // sleep on clamshell closure
+  kIOPMProcessorSpeedChange     = (1<<8),  // change the processor speed
+  kIOPMOverTemp                 = (1<<9),  // system dangerously hot
+  kIOPMClamshellOpened          = (1<<10)  // clamshell was opened
 };
-                                        // Return codes
 
-// PUBLIC power management features
-// NOTE: this is a direct port from classic, some of these bits
-//       are obsolete but are included for completeness
+/*******************************************************************************
+ *
+ * Power Management Return Codes
+ *
+ ******************************************************************************/
 enum {
-  kPMHasWakeupTimerMask        = (1<<0),  // 1=wake timer is supported
-  kPMHasSharedModemPortMask    = (1<<1),  // Not used
-  kPMHasProcessorCyclingMask   = (1<<2),  // 1=processor cycling supported
-  kPMMustProcessorCycleMask    = (1<<3),  // Not used
-  kPMHasReducedSpeedMask       = (1<<4),  // 1=supports reduced processor speed
-  kPMDynamicSpeedChangeMask    = (1<<5),  // 1=supports changing processor speed on the fly
-  kPMHasSCSIDiskModeMask       = (1<<6),  // 1=supports using machine as SCSI drive
-  kPMCanGetBatteryTimeMask     = (1<<7),  // 1=battery time can be calculated
-  kPMCanWakeupOnRingMask       = (1<<8),  // 1=machine can wake on modem ring
-  kPMHasDimmingSupportMask     = (1<<9),  // 1=has monitor dimming support
-  kPMHasStartupTimerMask       = (1<<10), // 1=can program startup timer
-  kPMHasChargeNotificationMask = (1<<11), // 1=client can determine charger status/get notifications
-  kPMHasDimSuspendSupportMask  = (1<<12), // 1=can dim diplay to DPMS ('off') state
-  kPMHasWakeOnNetActivityMask  = (1<<13), // 1=supports waking upon receipt of net packet
-  kPMHasWakeOnLidMask          = (1<<14), // 1=can wake upon lid/case opening
-  kPMCanPowerOffPCIBusMask     = (1<<15), // 1=can remove power from PCI bus on sleep
-  kPMHasDeepSleepMask          = (1<<16), // 1=supports deep (hibernation) sleep
-  kPMHasSleepMask              = (1<<17), // 1=machine support low power sleep (ala powerbooks)
-  kPMSupportsServerModeAPIMask = (1<<18), // 1=supports reboot on AC resume for unexpected power loss
-  kPMHasUPSIntegrationMask     = (1<<19)  // 1=supports incorporating UPS devices into power source calcs
+    kIOPMNoErr                  = 0,
+    // Returned by powerStateWillChange and powerStateDidChange:
+    // Immediate acknowledgement of power state change
+    kIOPMAckImplied             = 0,
+    // Acknowledgement of power state change will come later 
+    kIOPMWillAckLater           = 1,
+    
+    // Returned by requestDomainState:
+    // Unrecognized specification parameter
+    kIOPMBadSpecification       = 4,
+    // No power state matches search specification
+    kIOPMNoSuchState            = 5,
+    
+    // Device cannot change its power for some reason
+    kIOPMCannotRaisePower       = 6,
+    
+  // Returned by changeStateTo:
+    // Requested state doesn't exist
+    kIOPMParameterError         = 7,
+    // Device not yet fully hooked into power management
+    kIOPMNotYetInitialized      = 8,
+
+    // And the old constants; deprecated
+    IOPMNoErr = kIOPMNoErr,
+    IOPMAckImplied = kIOPMAckImplied, 
+    IOPMWillAckLater = kIOPMWillAckLater,
+    IOPMBadSpecification = kIOPMBadSpecification,
+    IOPMNoSuchState = kIOPMNoSuchState,
+    IOPMCannotRaisePower = kIOPMCannotRaisePower,
+    IOPMParameterError = kIOPMParameterError,
+    IOPMNotYetInitialized = kIOPMNotYetInitialized
+};
+
+
+// IOPMPowerSource class descriptive strings
+// Power Source state is published as properties to the IORegistry under these 
+// keys.
+#define kIOPMPSExternalConnectedKey                 "ExternalConnected"
+#define kIOPMPSExternalChargeCapableKey             "ExternalChargeCapable"
+#define kIOPMPSBatteryInstalledKey                  "BatteryInstalled"
+#define kIOPMPSIsChargingKey                        "IsCharging"
+#define kIOPMFullyChargedKey                        "FullyCharged"
+#define kIOPMPSAtWarnLevelKey                       "AtWarnLevel"
+#define kIOPMPSAtCriticalLevelKey                   "AtCriticalLevel"
+#define kIOPMPSCurrentCapacityKey                   "CurrentCapacity"
+#define kIOPMPSMaxCapacityKey                       "MaxCapacity"
+#define kIOPMPSDesignCapacityKey                    "DesignCapacity"
+#define kIOPMPSTimeRemainingKey                     "TimeRemaining"
+#define kIOPMPSAmperageKey                          "Amperage"
+#define kIOPMPSVoltageKey                           "Voltage"
+#define kIOPMPSCycleCountKey                        "CycleCount"
+#define kIOPMPSMaxErrKey                            "MaxErr"
+#define kIOPMPSAdapterInfoKey                       "AdapterInfo"
+#define kIOPMPSLocationKey                          "Location"
+#define kIOPMPSErrorConditionKey                    "ErrorCondition"
+#define kIOPMPSManufacturerKey                      "Manufacturer"
+#define kIOPMPSManufactureDateKey                   "ManufactureDate"
+#define kIOPMPSModelKey                             "Model"
+#define kIOPMPSSerialKey                            "Serial"
+#define kIOPMDeviceNameKey                          "DeviceName"
+#define kIOPMPSLegacyBatteryInfoKey                 "LegacyBatteryInfo"
+#define kIOPMPSBatteryHealthKey                     "BatteryHealth"
+#define kIOPMPSHealthConfidenceKey                  "HealthConfidence"
+
+// Definitions for battery location, in case of multiple batteries.
+// A location of 0 is unspecified
+// Location is undefined for single battery systems
+enum {
+    kIOPMPSLocationLeft = 1001,
+    kIOPMPSLocationRight = 1002
 };
 
-// PRIVATE power management features
-// NOTE: this is a direct port from classic, some of these bits
-//       are obsolete but are included for completeness.
+// Battery quality health types, specified by BatteryHealth and HealthConfidence
+// properties in an IOPMPowerSource battery kext.
 enum {
-  kPMHasExtdBattInfoMask       = (1<<0),  // Not used
-  kPMHasBatteryIDMask          = (1<<1),  // Not used
-  kPMCanSwitchPowerMask        = (1<<2),  // Not used 
-  kPMHasCelsiusCyclingMask     = (1<<3),  // Not used
-  kPMHasBatteryPredictionMask  = (1<<4),  // Not used
-  kPMHasPowerLevelsMask        = (1<<5),  // Not used
-  kPMHasSleepCPUSpeedMask      = (1<<6),  // Not used
-  kPMHasBtnIntHandlersMask     = (1<<7),  // 1=supports individual button interrupt handlers
-  kPMHasSCSITermPowerMask      = (1<<8),  // 1=supports SCSI termination power switch
-  kPMHasADBButtonHandlersMask  = (1<<9),  // 1=supports button handlers via ADB
-  kPMHasICTControlMask         = (1<<10), // 1=supports ICT control
-  kPMHasLegacyDesktopSleepMask = (1<<11), // 1=supports 'doze' style sleep
-  kPMHasDeepIdleMask           = (1<<12), // 1=supports Idle2 in hardware
-  kPMOpenLidPreventsSleepMask  = (1<<13), // 1=open case prevent machine from sleeping
-  kPMClosedLidCausesSleepMask  = (1<<14), // 1=case closed (clamshell closed) causes sleep
-  kPMHasFanControlMask         = (1<<15), // 1=machine has software-programmable fan/thermostat controls
-  kPMHasThermalControlMask     = (1<<16), // 1=machine supports thermal monitoring
-  kPMHasVStepSpeedChangeMask   = (1<<17), // 1=machine supports processor voltage/clock change
-  kPMEnvironEventsPolledMask   = (1<<18)  // 1=machine doesn't generate pmu env ints, we must poll instead 
+    kIOPMUndefinedValue = 0,
+    kIOPMPoorValue      = 1,
+    kIOPMFairValue      = 2,
+    kIOPMGoodValue      = 3
 };
 
-// DEFAULT public and private features for machines whose device tree
-// does NOT contain this information (pre-Core99).
-
-// For Cuda-based Desktops
-
-#define kStdDesktopPMFeatures   kPMHasWakeupTimerMask         |\
-                                kPMHasProcessorCyclingMask    |\
-                                kPMHasDimmingSupportMask      |\
-                                kPMHasStartupTimerMask        |\
-                                kPMSupportsServerModeAPIMask  |\
-                                kPMHasUPSIntegrationMask
-
-#define kStdDesktopPrivPMFeatures  kPMHasExtdBattInfoMask     |\
-                                   kPMHasICTControlMask       |\
-                                   kPMHasLegacyDesktopSleepMask
-
-#define kStdDesktopNumBatteries 0
-
-// For Wallstreet (PowerBook G3 Series 1998)
-
-#define kWallstreetPMFeatures   kPMHasWakeupTimerMask         |\
-                                kPMHasProcessorCyclingMask    |\
-                                kPMHasReducedSpeedMask        |\
-                                kPMDynamicSpeedChangeMask     |\
-                                kPMHasSCSIDiskModeMask        |\
-                                kPMCanGetBatteryTimeMask      |\
-                                kPMHasDimmingSupportMask      |\
-                                kPMHasChargeNotificationMask  |\
-                                kPMHasDimSuspendSupportMask   |\
-                                kPMHasSleepMask
-
-#define kWallstreetPrivPMFeatures  kPMHasExtdBattInfoMask      |\
-                                   kPMHasBatteryIDMask         |\
-                                   kPMCanSwitchPowerMask       |\
-                                   kPMHasADBButtonHandlersMask |\
-                                   kPMHasSCSITermPowerMask     |\
-                                   kPMHasICTControlMask        |\
-                                   kPMClosedLidCausesSleepMask |\
-                                   kPMEnvironEventsPolledMask
-
-#define kStdPowerBookPMFeatures      kWallstreetPMFeatures
-#define kStdPowerBookPrivPMFeatures  kWallstreetPrivPMFeatures
-
-#define kStdPowerBookNumBatteries 2
-
-// For 101 (PowerBook G3 Series 1999)
-
-#define k101PMFeatures          kPMHasWakeupTimerMask         |\
-                                kPMHasProcessorCyclingMask    |\
-                                kPMHasReducedSpeedMask        |\
-                                kPMDynamicSpeedChangeMask     |\
-                                kPMHasSCSIDiskModeMask        |\
-                                kPMCanGetBatteryTimeMask      |\
-                                kPMHasDimmingSupportMask      |\
-                                kPMHasChargeNotificationMask  |\
-                                kPMHasDimSuspendSupportMask   |\
-                                kPMHasSleepMask               |\
-                                kPMHasUPSIntegrationMask
-
-#define k101PrivPMFeatures      kPMHasExtdBattInfoMask        |\
-                                kPMHasBatteryIDMask           |\
-                                kPMCanSwitchPowerMask         |\
-                                kPMHasADBButtonHandlersMask   |\
-                                kPMHasSCSITermPowerMask       |\
-                                kPMHasICTControlMask          |\
-                                kPMClosedLidCausesSleepMask   |\
-                                kPMEnvironEventsPolledMask
-
-#define IOPMNoErr		0	// normal return
-
-                        // returned by powerStateWillChange and powerStateDidChange:
-#define IOPMAckImplied		0	// acknowledgement of power state change is implied
-#define IOPMWillAckLater	1	// acknowledgement of power state change will come later
-
-                        // returned by requestDomainState
-#define IOPMBadSpecification	4	// unrecognized specification parameter
-#define IOPMNoSuchState		5	// no power state matches search specification
-
-#define IOPMCannotRaisePower	6	// a device cannot change its power for some reason
-
-                        // returned by changeStateTo
-#define IOPMParameterError	7	// requested state doesn't exist
-#define IOPMNotYetInitialized	8	// device not yet fully hooked into power management "graph"
-
-
-						// used by Root Domain UserClient
+// Battery's time remaining estimate is invalid this long (seconds) after a wake
+#define kIOPMPSInvalidWakeSecondsKey           "BatteryInvalidWakeSeconds"
+
+// Battery must wait this long (seconds) after being completely charged before
+// the battery is settled.
+#define kIOPMPSPostChargeWaitSecondsKey        "PostChargeWaitSeconds"
+
+// Battery must wait this long (seconds) after being completely discharged 
+// before the battery is settled.
+#define kIOPMPSPostDishargeWaitSecondsKey      "PostDischargeWaitSeconds"
+
+
+
+// PM Settings Controller setting types
+// Settings types used primarily with:
+//      IOPMrootDomain::registerPMSettingController
+// The values are identical to the similarly named keys for use in user space
+// PM settings work. Those keys are defined in IOPMLibPrivate.h.
+#define kIOPMSettingWakeOnRingKey                   "Wake On Modem Ring"
+#define kIOPMSettingRestartOnPowerLossKey           "Automatic Restart On Power Loss"
+#define kIOPMSettingWakeOnACChangeKey               "Wake On AC Change"
+#define kIOPMSettingSleepOnPowerButtonKey           "Sleep On Power Button"
+#define kIOPMSettingWakeOnClamshellKey              "Wake On Clamshell Open"
+#define kIOPMSettingReduceBrightnessKey             "ReduceBrightness"
+#define kIOPMSettingDisplaySleepUsesDimKey          "Display Sleep Uses Dim"
+#define kIOPMSettingTimeZoneOffsetKey               "TimeZoneOffsetSeconds"
+#define kIOPMSettingMobileMotionModuleKey           "MobileMotionModule"
+
+// Setting controlling drivers can register to receive scheduled wake data
+// Either in "CF seconds" type, or structured calendar data in a formatted
+// IOPMCalendarStruct defined below.
+#define kIOPMSettingAutoWakeSecondsKey              "wake"
+#define kIOPMSettingAutoWakeCalendarKey             "WakeByCalendarDate"
+#define kIOPMSettingAutoPowerSecondsKey             "poweron"
+#define kIOPMSettingAutoPowerCalendarKey            "PowerByCalendarDate"
+
+// Debug seconds auto wake
+// Used by sleep cycling debug tools
+#define kIOPMSettingDebugWakeRelativeKey            "WakeRelativeToSleep"
+#define kIOPMSettingDebugPowerRelativeKey           "PowerRelativeToShutdown"
+
+struct IOPMCalendarStruct {
+    UInt32      year;
+    UInt8       month;
+    UInt8       day;
+    UInt8       hour;
+    UInt8       minute;
+    UInt8       second;
+};
+typedef struct IOPMCalendarStruct IOPMCalendarStruct;
 
+// SetAggressiveness types
 enum {
     kPMGeneralAggressiveness = 0,
     kPMMinutesToDim,
@@ -287,21 +385,21 @@ enum {
     kIOPMExternalPower
 };
 
-#define kAppleClamshellStateKey           "AppleClamshellState"
-#define kIOREMSleepEnabledKey             "REMSleepEnabled"
+#define kIOREMSleepEnabledKey               "REMSleepEnabled"
 
 // Strings for deciphering the dictionary returned from IOPMCopyBatteryInfo
-#define kIOBatteryInfoKey		"IOBatteryInfo"
-#define kIOBatteryCurrentChargeKey	"Current"
-#define kIOBatteryCapacityKey		"Capacity"
-#define kIOBatteryFlagsKey		"Flags"
-#define kIOBatteryVoltageKey		"Voltage"
-#define kIOBatteryAmperageKey		"Amperage"
+#define kIOBatteryInfoKey                   "IOBatteryInfo"
+#define kIOBatteryCurrentChargeKey          "Current"
+#define kIOBatteryCapacityKey               "Capacity"
+#define kIOBatteryFlagsKey                  "Flags"
+#define kIOBatteryVoltageKey                "Voltage"
+#define kIOBatteryAmperageKey               "Amperage"
+#define kIOBatteryCycleCountKey             "Cycle Count"
 
 enum {
-    kIOBatteryInstalled		= (1 << 2),
-    kIOBatteryCharge		= (1 << 1),
-    kIOBatteryChargerConnect	= (1 << 0)
+    kIOBatteryInstalled         = (1 << 2),
+    kIOBatteryCharge            = (1 << 1),
+    kIOBatteryChargerConnect    = (1 << 0)
 };
 
 
@@ -312,20 +410,7 @@ enum {
 // Apple private Legacy messages for re-routing AutoWake and AutoPower messages to the PMU
 // through newer user space IOPMSchedulePowerEvent API
 #define kIOPMUMessageLegacyAutoWake                 iokit_family_msg(sub_iokit_pmu, 0x200)
-#define kIOPMUMessageLegacyAutoPower            	iokit_family_msg(sub_iokit_pmu, 0x210)
-
-// These flags are deprecated. Use the version with the kIOPM prefix below.
-enum {
-  kACInstalled      = kIOBatteryChargerConnect,
-  kBatteryCharging  = kIOBatteryCharge,
-  kBatteryInstalled = kIOBatteryInstalled,
-  kUPSInstalled     = (1<<3),
-  kBatteryAtWarn    = (1<<4),
-  kBatteryDepleted  = (1<<5),
-  kACnoChargeCapability = (1<<6),     // AC adapter cannot charge battery
-  kRawLowBattery    = (1<<7),         // used only by  Platform Expert
-  kForceLowSpeed    = (1<<8)         // set by Platfm Expert, chk'd by Pwr Plugin};
-};
+#define kIOPMUMessageLegacyAutoPower                iokit_family_msg(sub_iokit_pmu, 0x210)
 
 // For use with IOPMPowerSource bFlags
 #define IOPM_POWER_SOURCE_REV   2
@@ -361,17 +446,17 @@ enum {
 };
 
 struct stateChangeNote{
-    IOPMPowerFlags	stateFlags;
-    unsigned long	stateNum;
-    void * 		powerRef;
+    IOPMPowerFlags    stateFlags;
+    unsigned long    stateNum;
+    void *         powerRef;
 };
 typedef struct stateChangeNote stateChangeNote;
 
 struct IOPowerStateChangeNotification {
-    void *		powerRef;
-    unsigned long	returnValue;
-    unsigned long	stateNumber;
-    IOPMPowerFlags	stateFlags;
+    void *        powerRef;
+    unsigned long    returnValue;
+    unsigned long    stateNumber;
+    IOPMPowerFlags    stateFlags;
 };
 typedef struct IOPowerStateChangeNotification IOPowerStateChangeNotification;
 typedef IOPowerStateChangeNotification sleepWakeNote;
diff --git a/iokit/IOKit/pwr_mgt/IOPMDeprecated.h b/iokit/IOKit/pwr_mgt/IOPMDeprecated.h
new file mode 100644
index 000000000..3bee01a3b
--- /dev/null
+++ b/iokit/IOKit/pwr_mgt/IOPMDeprecated.h
@@ -0,0 +1,177 @@
+/*
+ * Copyright (c) 1998-2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _IOPMDeprecated_h_
+#define _IOPMDeprecated_h_
+
+#ifdef __ppc__
+
+// Power events
+enum {
+  kClamshellClosedEventMask  = (1<<0),  // User closed lid
+  kDockingBarEventMask       = (1<<1),  // OBSOLETE
+  kACPlugEventMask           = (1<<2),  // User plugged or unplugged adapter
+  kFrontPanelButtonEventMask = (1<<3),  // User hit the front panel button
+  kBatteryStatusEventMask    = (1<<4)   // Battery status has changed
+};
+
+// PUBLIC power management features
+// NOTE: this is a direct port from classic, some of these bits
+//       are obsolete but are included for completeness
+enum {
+  kPMHasWakeupTimerMask        = (1<<0),  // 1=wake timer is supported
+  kPMHasSharedModemPortMask    = (1<<1),  // Not used
+  kPMHasProcessorCyclingMask   = (1<<2),  // 1=processor cycling supported
+  kPMMustProcessorCycleMask    = (1<<3),  // Not used
+  kPMHasReducedSpeedMask       = (1<<4),  // 1=supports reduced processor speed
+  kPMDynamicSpeedChangeMask    = (1<<5),  // 1=supports changing processor speed on the fly
+  kPMHasSCSIDiskModeMask       = (1<<6),  // 1=supports using machine as SCSI drive
+  kPMCanGetBatteryTimeMask     = (1<<7),  // 1=battery time can be calculated
+  kPMCanWakeupOnRingMask       = (1<<8),  // 1=machine can wake on modem ring
+  kPMHasDimmingSupportMask     = (1<<9),  // 1=has monitor dimming support
+  kPMHasStartupTimerMask       = (1<<10), // 1=can program startup timer
+  kPMHasChargeNotificationMask = (1<<11), // 1=client can determine charger status/get notifications
+  kPMHasDimSuspendSupportMask  = (1<<12), // 1=can dim diplay to DPMS ('off') state
+  kPMHasWakeOnNetActivityMask  = (1<<13), // 1=supports waking upon receipt of net packet
+  kPMHasWakeOnLidMask          = (1<<14), // 1=can wake upon lid/case opening
+  kPMCanPowerOffPCIBusMask     = (1<<15), // 1=can remove power from PCI bus on sleep
+  kPMHasDeepSleepMask          = (1<<16), // 1=supports deep (hibernation) sleep
+  kPMHasSleepMask              = (1<<17), // 1=machine support low power sleep (ala powerbooks)
+  kPMSupportsServerModeAPIMask = (1<<18), // 1=supports reboot on AC resume for unexpected power loss
+  kPMHasUPSIntegrationMask     = (1<<19)  // 1=supports incorporating UPS devices into power source calcs
+};
+
+// PRIVATE power management features
+// NOTE: this is a direct port from classic, some of these bits
+//       are obsolete but are included for completeness.
+enum {
+  kPMHasExtdBattInfoMask       = (1<<0),  // Not used
+  kPMHasBatteryIDMask          = (1<<1),  // Not used
+  kPMCanSwitchPowerMask        = (1<<2),  // Not used 
+  kPMHasCelsiusCyclingMask     = (1<<3),  // Not used
+  kPMHasBatteryPredictionMask  = (1<<4),  // Not used
+  kPMHasPowerLevelsMask        = (1<<5),  // Not used
+  kPMHasSleepCPUSpeedMask      = (1<<6),  // Not used
+  kPMHasBtnIntHandlersMask     = (1<<7),  // 1=supports individual button interrupt handlers
+  kPMHasSCSITermPowerMask      = (1<<8),  // 1=supports SCSI termination power switch
+  kPMHasADBButtonHandlersMask  = (1<<9),  // 1=supports button handlers via ADB
+  kPMHasICTControlMask         = (1<<10), // 1=supports ICT control
+  kPMHasLegacyDesktopSleepMask = (1<<11), // 1=supports 'doze' style sleep
+  kPMHasDeepIdleMask           = (1<<12), // 1=supports Idle2 in hardware
+  kPMOpenLidPreventsSleepMask  = (1<<13), // 1=open case prevent machine from sleeping
+  kPMClosedLidCausesSleepMask  = (1<<14), // 1=case closed (clamshell closed) causes sleep
+  kPMHasFanControlMask         = (1<<15), // 1=machine has software-programmable fan/thermostat controls
+  kPMHasThermalControlMask     = (1<<16), // 1=machine supports thermal monitoring
+  kPMHasVStepSpeedChangeMask   = (1<<17), // 1=machine supports processor voltage/clock change
+  kPMEnvironEventsPolledMask   = (1<<18)  // 1=machine doesn't generate pmu env ints, we must poll instead 
+};
+
+// DEFAULT public and private features for machines whose device tree
+// does NOT contain this information (pre-Core99).
+
+// For Cuda-based Desktops
+
+#define kStdDesktopPMFeatures   kPMHasWakeupTimerMask         |\
+                                kPMHasProcessorCyclingMask    |\
+                                kPMHasDimmingSupportMask      |\
+                                kPMHasStartupTimerMask        |\
+                                kPMSupportsServerModeAPIMask  |\
+                                kPMHasUPSIntegrationMask
+
+#define kStdDesktopPrivPMFeatures  kPMHasExtdBattInfoMask     |\
+                                   kPMHasICTControlMask       |\
+                                   kPMHasLegacyDesktopSleepMask
+
+#define kStdDesktopNumBatteries 0
+
+// For Wallstreet (PowerBook G3 Series 1998)
+
+#define kWallstreetPMFeatures   kPMHasWakeupTimerMask         |\
+                                kPMHasProcessorCyclingMask    |\
+                                kPMHasReducedSpeedMask        |\
+                                kPMDynamicSpeedChangeMask     |\
+                                kPMHasSCSIDiskModeMask        |\
+                                kPMCanGetBatteryTimeMask      |\
+                                kPMHasDimmingSupportMask      |\
+                                kPMHasChargeNotificationMask  |\
+                                kPMHasDimSuspendSupportMask   |\
+                                kPMHasSleepMask
+
+#define kWallstreetPrivPMFeatures  kPMHasExtdBattInfoMask      |\
+                                   kPMHasBatteryIDMask         |\
+                                   kPMCanSwitchPowerMask       |\
+                                   kPMHasADBButtonHandlersMask |\
+                                   kPMHasSCSITermPowerMask     |\
+                                   kPMHasICTControlMask        |\
+                                   kPMClosedLidCausesSleepMask |\
+                                   kPMEnvironEventsPolledMask
+
+#define kStdPowerBookPMFeatures      kWallstreetPMFeatures
+#define kStdPowerBookPrivPMFeatures  kWallstreetPrivPMFeatures
+
+#define kStdPowerBookNumBatteries 2
+
+// For 101 (PowerBook G3 Series 1999)
+
+#define k101PMFeatures          kPMHasWakeupTimerMask         |\
+                                kPMHasProcessorCyclingMask    |\
+                                kPMHasReducedSpeedMask        |\
+                                kPMDynamicSpeedChangeMask     |\
+                                kPMHasSCSIDiskModeMask        |\
+                                kPMCanGetBatteryTimeMask      |\
+                                kPMHasDimmingSupportMask      |\
+                                kPMHasChargeNotificationMask  |\
+                                kPMHasDimSuspendSupportMask   |\
+                                kPMHasSleepMask               |\
+                                kPMHasUPSIntegrationMask
+
+#define k101PrivPMFeatures      kPMHasExtdBattInfoMask        |\
+                                kPMHasBatteryIDMask           |\
+                                kPMCanSwitchPowerMask         |\
+                                kPMHasADBButtonHandlersMask   |\
+                                kPMHasSCSITermPowerMask       |\
+                                kPMHasICTControlMask          |\
+                                kPMClosedLidCausesSleepMask   |\
+                                kPMEnvironEventsPolledMask
+
+
+// These flags are deprecated. Use the version with the kIOPM prefix in IOPM.h
+enum {
+  kACInstalled      = (1<<0),
+  kBatteryCharging  = (1<<1),
+  kBatteryInstalled = (1<<2),
+  kUPSInstalled     = (1<<3),
+  kBatteryAtWarn    = (1<<4),
+  kBatteryDepleted  = (1<<5),
+  kACnoChargeCapability = (1<<6),     // AC adapter cannot charge battery
+  kRawLowBattery    = (1<<7),         // used only by  Platform Expert
+  kForceLowSpeed    = (1<<8)         // set by Platfm Expert, chk'd by Pwr Plugin};
+};
+
+#endif /* __ppc32 */
+#endif /* _IOPMDeprecated_h_ */
diff --git a/iokit/IOKit/pwr_mgt/IOPMPowerSource.h b/iokit/IOKit/pwr_mgt/IOPMPowerSource.h
index 047eb7f39..b398ee81a 100644
--- a/iokit/IOKit/pwr_mgt/IOPMPowerSource.h
+++ b/iokit/IOKit/pwr_mgt/IOPMPowerSource.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -25,59 +25,244 @@
  * 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
+ 
+#ifndef _IOPMPowerSource_h_
+#define _IOPMPowerSource_h_
+
 #include <libkern/c++/OSObject.h>
+#include <IOKit/pwr_mgt/IOPM.h>
 #include <IOKit/IOTypes.h>
 #include <IOKit/IOReturn.h>
-#include "IOPM.h"
-
-class ApplePMU;
+#include <IOKit/IOService.h>
 
-const unsigned long kSecondsPerHour = (60*60);
-const unsigned long kTenMinutesInSeconds = (10 * 60);
+enum {
+    kSecondsPerHour = 3600,
+    kTenMinutesInSeconds = 600
+};
 
-// our battery (power source) object
+/* class IOPMPowerSource
+ *
+ * See IOKit/pwr_mgt/IOPM.h for power source keys relevant to this class. These
+ * report-type keys are required for calls to IOPMPowerSource::setReportables(),
+ * and they define the IORegistry interface through which data is passed back
+ * up to the rest of the system.
+ *
+ * A subclassing driver that doesn't want to do anything fancy should:
+ *   1. Subclass IOPMPowerSource
+ *   3. Install its own battery change notifications or polling routine that can
+ *          converse with actual battery hardware.
+ *   4. When battery state changes, change the relevant member variables
+ *          through setCurrentCapacity() style accessors.
+ *   5. Call updateStatus() on itself when all such settings have been updated.
+ *   
+ * The subclass driver should also initially populate its settings and call
+ * updateStatus() on launch.
+ *
+ *
+ * Settings
+ *
+ * ExternalConnected
+ * Type: bool
+ * IORegistry Key: kIOPMPSExternalConnectedKey
+ * True if computer is drawing external power 
+ *
+ * ExternalChargeCapable
+ * Type: bool
+ * IORegistry Key: kIOPMPSExternalChargeCapableKey
+ * True if external power is capable of charging internal battery
+ *
+ * BatteryInstalled
+ * Type: bool
+ * IORegistry Key: kIOPMPSBatteryInstalledKey
+ * True if a battery is present; false if removed
+ *
+ * IsCharging
+ * Type: bool
+ * IORegistry Key: kIOPMPSIsChargingKey
+ * True if battery is charging itself from external power
+ *
+ * AtWarnLevel
+ * Type: bool
+ * IORegistry Key: kIOPMPSAtWarnLevelKey
+ * True if draining battery capacity and past warn level
+ *
+ * AtCriticalLevel
+ * Type: bool
+ * IORegistry Key: kIOPMPSAtCriticalLevelKey
+ * True if draining battery capacity and past critical level
+ *
+ * CurrentCapacity
+ * MaxCapacity
+ * Type: unsigned int
+ * IORegistry Key: kIOPMPSCurrentCapacityKey, kIOPMPSMaxCapacityKey
+ * Capacity measured in mAh
+ *
+ * TimeRemaining
+ * Type: int
+ * IORegistry Key: kIOPMPSTimeRemainingKey
+ * Time remaining measured in minutes
+ *
+ * Amperage
+ * Type: int
+ * IORegistry Key: kIOPMPSAmperageKey
+ * Current is measured in mA
+ *
+ * Voltage
+ * Type: unsigned int
+ * IORegistry Key: kIOPMPSVoltageKey
+ * Voltage measured in mV
+ *
+ * CycleCount
+ * Type: unsigned int
+ * IORegistry Key: kIOPMPSCycleCountKey
+ * Number of charge/discharge cycles
+ *
+ * AdapterInfo
+ * Type: int
+ * IORegistry Key: kIOPMPSAdapterInfoKey
+ * Power adapter information
+ *
+ * Location
+ * Type: int
+ * IORegistry Key: kIOPMPSLocationKey
+ * Clue about battery's location in machine - Left vs. Right
+ *
+ * ErrorCondition
+ * Type: OSSymbol
+ * IORegistry Key: kIOPMPSErrorConditionKey
+ * String describing error state of battery
+ *
+ * Manufacturer
+ * Type: OSSymbol
+ * IORegistry Key: kIOPMPSManufacturerKey
+ * String describing battery manufacturer
+ *
+ * Model
+ * Type: OSSymbol
+ * IORegistry Key: kIOPMPSModelKey
+ * String describing model number
+ *
+ * Serial
+ * Type: OSSymbol
+ * IORegistry Key: kIOPMPSSerialKey
+ * String describing serial number  or unique info
+ *
+ * LegacyIOBatteryInfo
+ * Type: OSDictionary
+ * IORegistry Key: kIOPMPSLegacyBatteryInfoKey
+ * Dictionary conforming to the OS X 10.0-10.4 
+ */
 
-class IOPMPowerSource : public OSObject
+class IOPMPowerSource : public IOService
 {
     OSDeclareDefaultStructors(IOPMPowerSource)
 
+    friend class IOPMPowerSourceList;
+
  protected:
+    // Tracking for IOPMPowerSourceList
+    IOPMPowerSource         *nextInList;
+    
+    OSDictionary            *properties;
 
-    UInt32             bFlags;
-    UInt32             bTimeRemaining;
-    UInt16             bCurCapacity;
-    UInt16             bMaxCapacity;
-    SInt16             bCurrent;
-    UInt16             bVoltage;
-    UInt16             bBatteryIndex;
+    const OSSymbol *externalConnectedKey;
+    const OSSymbol *externalChargeCapableKey;
+    const OSSymbol *batteryInstalledKey;
+    const OSSymbol *chargingKey;
+    const OSSymbol *warnLevelKey;
+    const OSSymbol *criticalLevelKey;
+    const OSSymbol *currentCapacityKey;
+    const OSSymbol *maxCapacityKey;
+    const OSSymbol *timeRemainingKey;
+    const OSSymbol *amperageKey;
+    const OSSymbol *voltageKey;
+    const OSSymbol *cycleCountKey;
+    const OSSymbol *adapterInfoKey;
+    const OSSymbol *locationKey;
+    const OSSymbol *errorConditionKey;
+    const OSSymbol *manufacturerKey;
+    const OSSymbol *modelKey;
+    const OSSymbol *serialKey;
+    const OSSymbol *batteryInfoKey;
 
  public:
 
-    IOPMPowerSource * nextInList;
-
-    bool               init (unsigned short whichBatteryIndex);
-    unsigned long      capacityPercentRemaining (void);
-    bool               atWarnLevel (void);
-    bool               depleted (void);
+/*! @function powerSource
+    @abstract Creates a new IOPMPowerSource nub. Must be attached to IORegistry,
+        and registered by provider.
+*/
+    static IOPMPowerSource *powerSource(void);
 
-    // accessors
+    virtual bool init(void);
+    
+    virtual void free(void);
 
-    bool               isInstalled (void);
-    bool               isCharging (void);
-    bool               acConnected (void);
-    unsigned long      timeRemaining (void);
-    unsigned long      maxCapacity (void);
-    unsigned long      curCapacity (void);
-    long               currentDrawn (void);
-    unsigned long      voltage (void);
+/*! @function updateStatus
+    @abstract Must be called by physical battery controller when battery state
+                has changed significantly.
+    @discussion The system will not poll this object for battery updates. Rather \
+    the battery's controller must call updateStatus() every time state changes \
+    and the settings will be relayed to higher levels of power management. \
+    The subclassing driver should override this only if the driver needs to add \
+    new settings to the base class.
+*/
+    virtual void updateStatus(void);
 
-    // calculations
+/* Public accessors for battery state
+ */
+    bool externalConnected(void);
+    bool externalChargeCapable(void);
+    bool batteryInstalled(void);
+    bool isCharging(void);
+    bool atWarnLevel(void);
+    bool atCriticalLevel(void);
 
-    // function updateStatus is called whenever the system needs
-    // to obtain the latest power source state...must be overridden
-    // by subclasses.
-    virtual void updateStatus (void);
-};
+    unsigned int currentCapacity(void);
+    unsigned int maxCapacity(void);
+    unsigned int capacityPercentRemaining(void);
+    int timeRemaining(void);
+    int amperage(void);
+    unsigned int voltage(void);
+    unsigned int cycleCount(void);
+    int adapterInfo(void);
+    int location(void);
+    
+    OSSymbol *errorCondition(void);
+    OSSymbol *manufacturer(void);
+    OSSymbol *model(void);
+    OSSymbol *serial(void);
+    OSDictionary *legacyIOBatteryInfo(void);
+    
+protected:
+/* Protected "setter" methods for subclasses
+ * Subclasses should use these setters to modify all battery properties.
+ * 
+ * Subclasses must follow all property changes with a call to updateStatus() 
+ * to flush settings changes to upper level battery API clients.
+ *
+ */
+    void setExternalConnected(bool);
+    void setExternalChargeCapable(bool);
+    void setBatteryInstalled(bool);
+    void setIsCharging(bool);
+    void setAtWarnLevel(bool);
+    void setAtCriticalLevel(bool);
 
+    void setCurrentCapacity(unsigned int);
+    void setMaxCapacity(unsigned int);    
+    void setTimeRemaining(int);
+    void setAmperage(int);    
+    void setVoltage(unsigned int);
+    void setCycleCount(unsigned int);
+    void setAdapterInfo(int);
+    void setLocation(int);
 
+    void setErrorCondition(OSSymbol *);
+    void setManufacturer(OSSymbol *);
+    void setModel(OSSymbol *);
+    void setSerial(OSSymbol *);
+    void setLegacyIOBatteryInfo(OSDictionary *);
+    
+};
 
+#endif
diff --git a/iokit/IOKit/pwr_mgt/IOPMPowerSourceList.h b/iokit/IOKit/pwr_mgt/IOPMPowerSourceList.h
index 69666035a..cb1c8ea37 100644
--- a/iokit/IOKit/pwr_mgt/IOPMPowerSourceList.h
+++ b/iokit/IOKit/pwr_mgt/IOPMPowerSourceList.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -32,27 +32,23 @@ class IOPMPowerSource;
 
 class IOPMPowerSourceList : public OSObject
 {
-OSDeclareDefaultStructors(IOPMPowerSourceList)
-
-private:
-
-IOPMPowerSource * firstItem;		// pointer to first power source in list
-unsigned long	length;			// how many power sources are in the list
-
-
-public:
-void initialize ( void );
-
-IOReturn addToList ( IOPMPowerSource *   newPowerSource );
-
-IOPMPowerSource * firstInList ( void );
-
-IOPMPowerSource * nextInList ( IOPMPowerSource * currentItem );
-
-unsigned long numberOfItems ( void );
-
-IOReturn removeFromList ( IOPMPowerSource * theItem );
-
-void free ( void );
+    OSDeclareDefaultStructors(IOPMPowerSourceList)
+ private:
+    // pointer to first power source in list
+    IOPMPowerSource         *firstItem;
+
+    // how many power sources are in the list
+    unsigned long           length;
+
+  public:
+    void initialize(void);
+    void free(void);
+
+    unsigned long numberOfItems(void);
+    IOReturn addToList(IOPMPowerSource *newPowerSource);
+    IOReturn removeFromList(IOPMPowerSource *theItem);
+    
+    IOPMPowerSource *firstInList(void);
+    IOPMPowerSource *nextInList(IOPMPowerSource *currentItem);
 };
 
diff --git a/iokit/IOKit/pwr_mgt/IOPMinformeeList.h b/iokit/IOKit/pwr_mgt/IOPMinformeeList.h
index 3ce993dcd..273c08996 100644
--- a/iokit/IOKit/pwr_mgt/IOPMinformeeList.h
+++ b/iokit/IOKit/pwr_mgt/IOPMinformeeList.h
@@ -26,6 +26,7 @@
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <libkern/c++/OSObject.h>
+#include <IOKit/IOLocks.h>
 #include <IOKit/IOReturn.h>
 
 class IOPMinformee;
@@ -36,25 +37,24 @@ class IOPMinformeeList : public OSObject
 OSDeclareDefaultStructors(IOPMinformeeList)
 
 private:
-IOPMinformee *	 firstItem;		// pointer to first informee in the list
-unsigned long	length;			// how many informees are in the list
-
+    IOPMinformee       *firstItem;      // pointer to first informee in the list
+    unsigned long       length;         // how many informees are in the list
 
 public:
-void initialize ( void );
-
-IOReturn addToList ( IOPMinformee *   newInformee );
-
-IOPMinformee * firstInList ( void );
-
-IOPMinformee * nextInList ( IOPMinformee * currentItem );
-
-unsigned long numberOfItems ( void );
+    void initialize ( void );
+    void free ( void );
 
-IOPMinformee * findItem ( IOService * driverOrChild );
+    unsigned long numberOfItems ( void );
 
-IOReturn removeFromList ( IOService * theItem );
+    IOReturn addToList ( IOPMinformee *   newInformee );
+    IOReturn removeFromList ( IOService * theItem );
+    
+    IOPMinformee * firstInList ( void );
+    IOPMinformee * nextInList ( IOPMinformee * currentItem );
+    
+    IOPMinformee * findItem ( IOService * driverOrChild );        
 
-void free ( void );
+    // This lock must be held while modifying list or length
+    static IORecursiveLock * getSharedRecursiveLock( void );
 };
 
diff --git a/iokit/IOKit/pwr_mgt/Makefile b/iokit/IOKit/pwr_mgt/Makefile
index 34e8b138c..5b6f80277 100644
--- a/iokit/IOKit/pwr_mgt/Makefile
+++ b/iokit/IOKit/pwr_mgt/Makefile
@@ -30,7 +30,7 @@ EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
 
 ALL_HEADERS = $(shell (cd $(SOURCE); echo *.h))
 
-INSTALL_MI_LIST	= IOPMLibDefs.h IOPM.h
+INSTALL_MI_LIST	= IOPMLibDefs.h IOPM.h IOPMDeprecated.h
 INSTALL_MI_LCL_LIST = ""
 INSTALL_MI_DIR = $(MI_DIR)
 
diff --git a/iokit/IOKit/pwr_mgt/RootDomain.h b/iokit/IOKit/pwr_mgt/RootDomain.h
index d54821e10..e2dd94dfe 100644
--- a/iokit/IOKit/pwr_mgt/RootDomain.h
+++ b/iokit/IOKit/pwr_mgt/RootDomain.h
@@ -31,11 +31,10 @@
 #include <IOKit/IOService.h>
 #include <IOKit/pwr_mgt/IOPM.h>
 
+class IOPMWorkArbiter;
 class IOPMPowerStateQueue;
 class RootDomainUserClient;
 
-#define kRootDomainSupportedFeatures "Supported Features"
-
 enum {
     kRootDomainSleepNotSupported	= 0x00000000,
     kRootDomainSleepSupported 		= 0x00000001,
@@ -43,31 +42,35 @@ enum {
     kPCICantSleep			= 0x00000004
 };
 
-// Constants for use as arguments to the settings callback PMU/SMU defines
-// with registerPMSettingsController
+#define kRootDomainSupportedFeatures "Supported Features"
+
+// Supported Feature bitfields for IOPMrootDomain::publishFeature()
 enum {
-    kIOPMAutoWakeSetting = 1,
-    kIOPMAutoPowerOnSetting,
-    kIOPMWakeOnRingSetting,
-    kIOPMAutoRestartOnPowerLossSetting,
-    kIOPMWakeOnLidSetting,
-    kIOPMWakeOnACChangeSetting,
-    kIOPMTimeZoneSetting
+    kIOPMSupportedOnAC = 1<<0,
+    kIOPMSupportedOnBatt = 1<<1,
+    kIOPMSupportedOnUPS = 1<<2
 };
-typedef int IOPMSystemSettingType;
-
 
-
-typedef IOReturn (*IOPMSettingControllerCallback)(IOPMSystemSettingType arg_type, int arg_val, void *info);
+typedef IOReturn (*IOPMSettingControllerCallback) \
+                    (OSObject *target, const OSSymbol *type, \
+                     OSObject *val, uintptr_t refcon);
 
 extern "C"
 {
-	IONotifier * registerSleepWakeInterest(IOServiceInterestHandler, void *, void * = 0);
-        IONotifier * registerPrioritySleepWakeInterest(IOServiceInterestHandler handler, void * self, void * ref = 0);
-        IOReturn acknowledgeSleepWakeNotification(void * );
-        IOReturn vetoSleepWakeNotification(void * PMrefcon);
-        IOReturn rootDomainRestart ( void );
-        IOReturn rootDomainShutdown ( void );
+    IONotifier * registerSleepWakeInterest(
+               IOServiceInterestHandler, void *, void * = 0);
+               
+    IONotifier * registerPrioritySleepWakeInterest(
+                IOServiceInterestHandler handler, 
+                void * self, void * ref = 0);
+
+    IOReturn acknowledgeSleepWakeNotification(void * );
+
+    IOReturn vetoSleepWakeNotification(void * PMrefcon);
+
+    IOReturn rootDomainRestart ( void );
+
+    IOReturn rootDomainShutdown ( void );
 }
 
 #define IOPM_ROOTDOMAIN_REV		2
@@ -79,6 +82,8 @@ OSDeclareDefaultStructors(IOPMrootDomain)
 public:
 
     class IOService * wrangler;			// we tickle the wrangler on button presses, etc
+    
+    IOPMWorkArbiter * getPMArbiter(void);
 
     static IOPMrootDomain * construct( void );
     virtual bool start( IOService * provider );
@@ -96,24 +101,95 @@ public:
     void stopIgnoringClamshellEventsDuringWakeup ( void );
     void wakeFromDoze( void );
     void broadcast_it (unsigned long, unsigned long );
+
+    // KEXT driver announces support of power management feature
     void publishFeature( const char *feature );
+    
+    // KEXT driver announces support of power management feature
+    // And specifies power sources with kIOPMSupportedOn{AC/Batt/UPS} bitfield.
+    // Returns a unique uint32_t identifier for later removing support for this
+    // feature. 
+    // NULL is acceptable for uniqueFeatureID for kexts without plans to unload.
+    void publishFeature( const char *feature, 
+                            uint32_t supportedWhere,
+                            uint32_t *uniqueFeatureID);
+
+    // KEXT driver announces removal of a previously published power management 
+    // feature. Pass 'uniqueFeatureID' returned from publishFeature()
+    IOReturn removePublishedFeature( uint32_t removeFeatureID );
+
     void unIdleDevice( IOService *, unsigned long );
     void announcePowerSourceChange( void );
-        
+
     // Override of these methods for logging purposes.
     virtual IOReturn changePowerStateTo ( unsigned long ordinal );
     virtual IOReturn changePowerStateToPriv ( unsigned long ordinal );
 
-    IOReturn registerPMSettingController(IOPMSettingControllerCallback, void *);
+/*! @function copyPMSetting
+    @abstract Copy the current value for a PM setting. Returns OSNumber or
+        OSData depending on the setting.
+    @param whichSetting Name of the desired setting. 
+    @result OSObject *value if valid, NULL otherwise. */
+    OSObject *copyPMSetting(OSSymbol *whichSetting);
+    
+/*! @function registerPMSettingController
+    @abstract Register for callbacks on changes to certain PM settings.
+    @param settings NULL terminated array of C strings, each string for a PM 
+        setting that the caller is interested in and wants to get callbacks for. 
+    @param callout C function ptr or member function cast as such.
+    @param target The target of the callback, usually 'this'
+    @param refcon Will be passed to caller in callback; for caller's use.
+    @param handle Caller should keep the OSObject * returned here. If non-NULL,
+        handle will have a retain count of 1 on return. To deregister, pass to
+        unregisterPMSettingController()
+    @result kIOReturnSuccess on success. */
+    IOReturn registerPMSettingController(
+                                 const OSSymbol *settings[],
+                                 IOPMSettingControllerCallback callout,
+                                 OSObject   *target,
+                                 uintptr_t  refcon,
+                                 OSObject   **handle);    // out param
+
+/*! @function registerPMSettingController
+    @abstract Register for callbacks on changes to certain PM settings.
+    @param settings NULL terminated array of C strings, each string for a PM 
+        setting that the caller is interested in and wants to get callbacks for. 
+    @param supportedPowerSources bitfield indicating which power sources these
+        settings are supported for (kIOPMSupportedOnAC, etc.)
+    @param callout C function ptr or member function cast as such.
+    @param target The target of the callback, usually 'this'
+    @param refcon Will be passed to caller in callback; for caller's use.
+    @param handle Caller should keep the OSObject * returned here. If non-NULL,
+        handle will have a retain count of 1 on return. To deregister, pass to
+        unregisterPMSettingController()
+    @result kIOReturnSuccess on success. */
+    IOReturn registerPMSettingController(
+                                 const OSSymbol *settings[],
+                                 uint32_t   supportedPowerSources,
+                                 IOPMSettingControllerCallback callout,
+                                 OSObject   *target,
+                                 uintptr_t  refcon,
+                                 OSObject   **handle);    // out param
 
 private:
 
-    class IORootParent * patriarch;			// points to our parent
-    long		sleepSlider;			// pref: idle time before idle sleep
-    long		longestNonSleepSlider;		// pref: longest of other idle times
-    long		extraSleepDelay;		// sleepSlider - longestNonSleepSlider
-    thread_call_t	extraSleepTimer;		// used to wait between say display idle and system idle
-    thread_call_t   clamshellWakeupIgnore;   // Used to ignore clamshell close events while we're waking from sleep
+    // Points to our parent
+    class IORootParent * patriarch;
+
+    // Pref: idle time before idle sleep
+    long		sleepSlider;			
+
+    // Pref: longest of other idle times (disk and display)
+    long		longestNonSleepSlider;		
+
+    // Difference between sleepSlider and longestNonSleepSlider
+    long		extraSleepDelay;		
+
+    // Used to wait between say display idle and system idle
+    thread_call_t	extraSleepTimer;		
+
+    // Used to ignore clamshell close events while we're waking from sleep
+    thread_call_t   clamshellWakeupIgnore;   
     
     virtual void powerChangeDone ( unsigned long );
     virtual void command_received ( void *, void * , void * , void *);
@@ -133,18 +209,26 @@ private:
     static bool displayWranglerPublished( void * target, void * refCon,
                                     IOService * newService);
 
-    static bool batteryLocationPublished( void * target, void * refCon,
+    static bool batteryPublished( void * target, void * refCon,
                                     IOService * resourceService );
 
+    void adjustPowerState ( void );
     void setQuickSpinDownTimeout ( void );
-    void adjustPowerState( void );
     void restoreUserSpinDownTimeout ( void );
-
+    
+    bool shouldSleepOnClamshellClosed (void );
+    void sendClientClamshellNotification ( void );
+    
     IOLock                  *featuresDictLock;  // guards supportedFeatures
     IOPMPowerStateQueue     *pmPowerStateQueue;
+    
+    IOWorkLoop              *arbiterWorkLoop;
+    IOPMWorkArbiter         *pmArbiter;
+    
     unsigned int user_spindown;       // User's selected disk spindown value
 
     unsigned int systemBooting:1;
+    unsigned int systemShutdown:1;
     unsigned int ignoringClamshell:1;
     unsigned int allowSleep:1;
     unsigned int sleepIsSupported:1;
@@ -155,25 +239,29 @@ private:
 
     unsigned int acAdaptorConnect:1;
     unsigned int ignoringClamshellDuringWakeup:1;
-    unsigned int reservedA:6;
+    unsigned int clamshellIsClosed:1;
+    unsigned int clamshellExists:1;
+    unsigned int reservedA:4;
     unsigned char reservedB[3];
     
-    struct PMSettingCtrl {
-        IOPMSettingControllerCallback       func;
-        void                                *refcon;
-    };
+    OSArray         *allowedPMSettings;
+    
+    // Settings controller info
+    IORecursiveLock        *settingsCtrlLock;  
+    OSDictionary           *settingsCallbacks;
+    OSDictionary           *fPMSettingsDict;
+    IOReturn setPMSetting(const OSSymbol *, OSObject *);
+
+    thread_call_t           diskSyncCalloutEntry;
+    IONotifier              *_batteryPublishNotifier;
+    IONotifier              *_displayWranglerNotifier;
 
-    // Private helper to call PM setting controller
-    IOReturn setPMSetting(int type, OSNumber *);
- 
     struct ExpansionData {    
-        PMSettingCtrl           *_settingController;
-        thread_call_t           diskSyncCalloutEntry;
-        IONotifier              *_batteryLocationNotifier;
-        IONotifier              *_displayWranglerNotifier;
     };
     ExpansionData   *_reserved;
     IOOptionBits platformSleepSupport;
+    
+    friend class PMSettingObject;
 };
 
 class IORootParent: public IOService
@@ -185,6 +273,8 @@ private:
     
 public:
 
+    virtual IOReturn changePowerStateToPriv ( unsigned long ordinal );
+
     bool start ( IOService * nub );
     void shutDownSystem ( void );
     void restartSystem ( void );
diff --git a/iokit/Kernel/IOBufferMemoryDescriptor.cpp b/iokit/Kernel/IOBufferMemoryDescriptor.cpp
index 8b9dd38aa..4821b81dc 100644
--- a/iokit/Kernel/IOBufferMemoryDescriptor.cpp
+++ b/iokit/Kernel/IOBufferMemoryDescriptor.cpp
@@ -29,9 +29,11 @@
 #include <IOKit/system.h>
 
 #include <IOKit/IOLib.h>
+#include <IOKit/IOMapper.h>
 #include <IOKit/IOBufferMemoryDescriptor.h>
 
 #include "IOKitKernelInternal.h"
+#include "IOCopyMapper.h"
 
 __BEGIN_DECLS
 void ipc_port_release_send(ipc_port_t port);
@@ -40,6 +42,12 @@ void ipc_port_release_send(ipc_port_t port);
 vm_map_t IOPageableMapForAddress( vm_address_t address );
 __END_DECLS
 
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+volatile ppnum_t gIOHighestAllocatedPage;
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
 #define super IOGeneralMemoryDescriptor
 OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor,
 				IOGeneralMemoryDescriptor);
@@ -93,8 +101,20 @@ bool IOBufferMemoryDescriptor::initWithOptions(
                                vm_size_t    capacity,
                                vm_offset_t  alignment,
 			       task_t	    inTask)
+{
+    mach_vm_address_t physicalMask = 0;
+    return (initWithPhysicalMask(inTask, options, capacity, alignment, physicalMask));
+}
+
+bool IOBufferMemoryDescriptor::initWithPhysicalMask(
+				task_t		  inTask,
+				IOOptionBits      options,
+				mach_vm_size_t    capacity,
+				mach_vm_address_t alignment,
+				mach_vm_address_t physicalMask)
 {
     kern_return_t 	kr;
+    addr64_t            lastIOAddr;
     vm_map_t 		vmmap = 0;
     IOOptionBits	iomdOptions = kIOMemoryAsReference | kIOMemoryTypeVirtual;
 
@@ -116,6 +136,12 @@ bool IOBufferMemoryDescriptor::initWithOptions(
     if ((inTask != kernel_task) && !(options & kIOMemoryPageable))
         return false;
 
+    if (physicalMask && (alignment <= 1))
+	alignment = ((physicalMask ^ PAGE_MASK) & PAGE_MASK) + 1;
+
+    if ((options & kIOMemoryPhysicallyContiguous) && !physicalMask)
+	physicalMask = 0xFFFFFFFF;
+
     _alignment = alignment;
     if (options & kIOMemoryPageable)
     {
@@ -174,7 +200,7 @@ bool IOBufferMemoryDescriptor::initWithOptions(
 #if IOALLOCDEBUG
        debug_iomallocpageable_size += size;
 #endif
-	if ((NULL == inTask) && (options & kIOMemoryPageable))
+	if (NULL == inTask)
 	    inTask = kernel_task;
 	else if (inTask == kernel_task)
 	{
@@ -182,7 +208,6 @@ bool IOBufferMemoryDescriptor::initWithOptions(
 	}
 	else
 	{
-
 	    if( !reserved) {
 		reserved = IONew( ExpansionData, 1 );
 		if( !reserved)
@@ -193,23 +218,58 @@ bool IOBufferMemoryDescriptor::initWithOptions(
 	    reserved->map = vmmap;
 	}
     }
-    else 
+    else
     {
-        // @@@ gvdl: Need to remove this
-        // Buffer should never auto prepare they should be prepared explicitly
-        // But it never was enforced so what are you going to do?
-        iomdOptions |= kIOMemoryAutoPrepare;
-
-	/* Allocate a wired-down buffer inside kernel space. */
-	if (options & kIOMemoryPhysicallyContiguous)
-	    _buffer = IOMallocContiguous(capacity, alignment, 0);
-	else if (alignment > 1)
-	    _buffer = IOMallocAligned(capacity, alignment);
+	if (IOMapper::gSystem)
+	    // assuming mapped space is 2G
+	    lastIOAddr = (1UL << 31) - PAGE_SIZE;
 	else
-	    _buffer = IOMalloc(capacity);
+	    lastIOAddr = ptoa_64(gIOHighestAllocatedPage);
 
-	if (!_buffer)
-	    return false;
+	if (physicalMask && (lastIOAddr != (lastIOAddr & physicalMask)))
+	{
+	    mach_vm_address_t address;
+	    iomdOptions &= ~kIOMemoryTypeVirtual;
+	    iomdOptions |= kIOMemoryTypePhysical;
+
+	    address = IOMallocPhysical(capacity, physicalMask);
+	    _buffer = (void *) address;
+	    if (!_buffer)
+		return false;
+
+	    if (inTask == kernel_task)
+	    {
+		vmmap = kernel_map;
+	    }
+	    else if (NULL != inTask)
+	    {
+		if( !reserved) {
+		    reserved = IONew( ExpansionData, 1 );
+		    if( !reserved)
+			return( false );
+		}
+		vmmap = get_task_map(inTask);
+		vm_map_reference(vmmap);
+		reserved->map = vmmap;
+	    }
+	    inTask = 0;
+	}
+	else
+	{
+	    // Buffer shouldn't auto prepare they should be prepared explicitly
+	    // But it never was enforced so what are you going to do?
+	    iomdOptions |= kIOMemoryAutoPrepare;
+
+	    /* Allocate a wired-down buffer inside kernel space. */
+	    if (options & kIOMemoryPhysicallyContiguous)
+		_buffer = (void *) IOKernelAllocateContiguous(capacity, alignment);
+	    else if (alignment > 1)
+		_buffer = IOMallocAligned(capacity, alignment);
+	    else
+		_buffer = IOMalloc(capacity);
+	    if (!_buffer)
+		return false;
+	}
     }
 
     _singleRange.v.address = (vm_address_t) _buffer;
@@ -219,22 +279,55 @@ bool IOBufferMemoryDescriptor::initWithOptions(
                                inTask, iomdOptions, /* System mapper */ 0))
 	return false;
 
-    if (options & kIOMemoryPageable)
+    if (physicalMask && !IOMapper::gSystem)
     {
-        kern_return_t kr;
+	IOMDDMACharacteristics mdSummary;
 
-	if (vmmap)
+	bzero(&mdSummary, sizeof(mdSummary));
+	IOReturn rtn = dmaCommandOperation(
+		kIOMDGetCharacteristics,
+		&mdSummary, sizeof(mdSummary));
+	if (rtn)
+	    return false;
+
+	if (mdSummary.fHighestPage)
 	{
-	    kr = doMap(vmmap, (IOVirtualAddress *) &_buffer, kIOMapAnywhere, 0, round_page_32(capacity));
-	    if (KERN_SUCCESS != kr)
+	    ppnum_t highest;
+	    while (mdSummary.fHighestPage > (highest = gIOHighestAllocatedPage))
 	    {
-		_buffer = 0;
-		return( false );
+		if (OSCompareAndSwap(highest, mdSummary.fHighestPage, 
+					(UInt32 *) &gIOHighestAllocatedPage))
+		    break;
 	    }
-	    _singleRange.v.address = (vm_address_t) _buffer;
+	    lastIOAddr = ptoa_64(mdSummary.fHighestPage);
+	}
+	else
+	    lastIOAddr = ptoa_64(gIOLastPage);
+
+	if (lastIOAddr != (lastIOAddr & physicalMask))
+	{
+	    if (kIOMemoryTypePhysical != (_flags & kIOMemoryTypeMask))
+	    {
+		// flag a retry
+		_physSegCount = 1;
+	    }
+	    return false;
 	}
     }
 
+    if (vmmap)
+    {
+	kr = doMap(vmmap, (IOVirtualAddress *) &_buffer, kIOMapAnywhere, 0, capacity);
+	if (KERN_SUCCESS != kr)
+	{
+	    _buffer = 0;
+	    return( false );
+	}
+
+	if (kIOMemoryTypeVirtual & iomdOptions)
+	    _singleRange.v.address = (vm_address_t) _buffer;
+    }
+
     setLength(capacity);
     
     return true;
@@ -249,8 +342,44 @@ IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithOptions(
     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
     
     if (me && !me->initWithOptions(options, capacity, alignment, inTask)) {
+	bool retry = me->_physSegCount;
 	me->release();
 	me = 0;
+	if (retry)
+	{
+	    me = new IOBufferMemoryDescriptor;
+	    if (me && !me->initWithOptions(options, capacity, alignment, inTask))
+	    {
+		me->release();
+		me = 0;
+	    }
+	}
+    }
+    return me;
+}
+
+IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
+					    task_t	      inTask,
+                                            IOOptionBits      options,
+                                            mach_vm_size_t    capacity,
+                                            mach_vm_address_t physicalMask)
+{
+    IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
+    
+    if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask))
+    {
+	bool retry = me->_physSegCount;
+	me->release();
+	me = 0;
+	if (retry)
+	{
+	    me = new IOBufferMemoryDescriptor;
+	    if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask))
+	    {
+		me->release();
+		me = 0;
+	    }
+	}
     }
     return me;
 }
@@ -268,13 +397,7 @@ IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::withOptions(
                                             vm_size_t    capacity,
                                             vm_offset_t  alignment)
 {
-    IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
-    
-    if (me && !me->initWithOptions(options, capacity, alignment, kernel_task)) {
-	me->release();
-	me = 0;
-    }
-    return me;
+    return(IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task, options, capacity, alignment));
 }
 
 
@@ -335,9 +458,21 @@ IOBufferMemoryDescriptor::withBytes(const void * inBytes,
 {
     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 
-    if (me && !me->initWithBytes(inBytes, inLength, inDirection, inContiguous)){
-        me->release();
-        me = 0;
+    if (me && !me->initWithBytes(inBytes, inLength, inDirection, inContiguous))
+    {
+	bool retry = me->_physSegCount;
+	me->release();
+	me = 0;
+	if (retry)
+	{
+	    me = new IOBufferMemoryDescriptor;
+	    if (me && !me->initWithBytes(inBytes, inLength, inDirection, inContiguous))
+	    {
+		me->release();
+		me = 0;
+	    }
+	}
+
     }
     return me;
 }
@@ -351,11 +486,13 @@ void IOBufferMemoryDescriptor::free()
 {
     // Cache all of the relevant information on the stack for use
     // after we call super::free()!
-    IOOptionBits options   = _options;
-    vm_size_t    size	   = _capacity;
-    void *       buffer	   = _buffer;
-    vm_map_t	 vmmap	   = 0;
-    vm_offset_t  alignment = _alignment;
+    IOOptionBits     flags     = _flags;
+    IOOptionBits     options   = _options;
+    vm_size_t        size      = _capacity;
+    void *           buffer    = _buffer;
+    IOVirtualAddress source    = _singleRange.v.address;
+    vm_map_t	     vmmap     = 0;
+    vm_offset_t      alignment = _alignment;
 
     if (reserved)
     {
@@ -382,8 +519,14 @@ void IOBufferMemoryDescriptor::free()
     }
     else if (buffer)
     {
-        if (options & kIOMemoryPhysicallyContiguous)
-            IOFreeContiguous(buffer, size);
+	if (kIOMemoryTypePhysical == (flags & kIOMemoryTypeMask))
+	{
+            if (vmmap)
+                vm_deallocate(vmmap, (vm_address_t) buffer, round_page_32(size));
+	    IOFreePhysical((mach_vm_address_t) source, size);
+	}
+        else if (options & kIOMemoryPhysicallyContiguous)
+            IOKernelFreeContiguous((mach_vm_address_t) buffer, size);
         else if (alignment > 1)
             IOFreeAligned(buffer, size);
         else
@@ -443,14 +586,21 @@ void IOBufferMemoryDescriptor::setDirection(IODirection direction)
 bool
 IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength)
 {
-    vm_size_t actualBytesToCopy = min(withLength, _capacity - _length);
+    vm_size_t   actualBytesToCopy = min(withLength, _capacity - _length);
+    IOByteCount offset;
 
     assert(_length <= _capacity);
-    bcopy(/* from */ bytes, (void *)(_singleRange.v.address + _length),
-          actualBytesToCopy);
+
+    offset = _length;
     _length += actualBytesToCopy;
     _singleRange.v.length += actualBytesToCopy;
 
+    if (_task == kernel_task)
+	bcopy(/* from */ bytes, (void *)(_singleRange.v.address + offset),
+	      actualBytesToCopy);
+    else
+	writeBytes(offset, bytes, actualBytesToCopy);
+
     return true;
 }
 
@@ -461,9 +611,13 @@ IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength)
  */
 void * IOBufferMemoryDescriptor::getBytesNoCopy()
 {
-    return (void *)_singleRange.v.address;
+    if (kIOMemoryTypePhysical == (_flags & kIOMemoryTypeMask))
+	return _buffer;
+    else
+	return (void *)_singleRange.v.address;
 }
 
+
 /*
  * getBytesNoCopy:
  *
@@ -472,13 +626,30 @@ void * IOBufferMemoryDescriptor::getBytesNoCopy()
 void *
 IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start, vm_size_t withLength)
 {
-    if (start < _length && (start + withLength) <= _length)
-        return (void *)(_singleRange.v.address + start);
+    IOVirtualAddress address;
+    if (kIOMemoryTypePhysical == (_flags & kIOMemoryTypeMask))
+	address = (IOVirtualAddress) _buffer;
+    else
+	address = _singleRange.v.address;
+
+   if (start < _length && (start + withLength) <= _length)
+        return (void *)(address + start);
     return 0;
 }
 
+/* DEPRECATED */ void * IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset,
+/* DEPRECATED */ 							IOByteCount * lengthOfSegment)
+{
+    void * bytes = getBytesNoCopy(offset, 0);
+    
+    if (bytes && lengthOfSegment)
+	*lengthOfSegment = _length - offset;
+
+    return bytes;
+}
+
 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 0);
-OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 1);
+OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 1);
 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 2);
 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 3);
 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 4);
diff --git a/iokit/Kernel/IOCPU.cpp b/iokit/Kernel/IOCPU.cpp
index fd87cc975..928079036 100644
--- a/iokit/Kernel/IOCPU.cpp
+++ b/iokit/Kernel/IOCPU.cpp
@@ -227,9 +227,12 @@ bool IOCPU::setProperty(const OSSymbol *aKey, OSObject *anObject)
 
 bool IOCPU::serializeProperties(OSSerialize *serialize) const
 {
-  super::setProperty(gIOCPUStateKey, gIOCPUStateNames[_cpuState]);
-  
-  return super::serializeProperties(serialize);
+	bool result;
+	OSDictionary *dict = dictionaryWithProperties();
+	dict->setObject(gIOCPUStateKey, gIOCPUStateNames[_cpuState]);
+	result = dict->serialize(serialize);
+	dict->release();  
+	return result;
 }
 
 IOReturn IOCPU::setProperties(OSObject *properties)
@@ -392,8 +395,10 @@ void IOCPUInterruptController::setCPUInterruptProperties(IOService *service)
 
 void IOCPUInterruptController::enableCPUInterrupt(IOCPU *cpu)
 {
-  ml_install_interrupt_handler(cpu, cpu->getCPUNumber(), this,
-                               (IOInterruptHandler)&IOCPUInterruptController::handleInterrupt, 0);
+	IOInterruptHandler handler = OSMemberFunctionCast(
+		IOInterruptHandler, this, &IOCPUInterruptController::handleInterrupt);
+
+	ml_install_interrupt_handler(cpu, cpu->getCPUNumber(), this, handler, 0);
   
   enabledCPUs++;
   
diff --git a/iokit/Kernel/IOCatalogue.cpp b/iokit/Kernel/IOCatalogue.cpp
index 73d4bb5ab..f041eaafe 100644
--- a/iokit/Kernel/IOCatalogue.cpp
+++ b/iokit/Kernel/IOCatalogue.cpp
@@ -1168,7 +1168,6 @@ bool IOCatalogue::addExtensionsFromArchive(OSData * mkext)
     return result;
 }
 
-
 /*********************************************************************
 * This function clears out all references to the in-kernel linker,
 * frees the list of startup extensions in extensionDict, and
@@ -1178,10 +1177,13 @@ bool IOCatalogue::addExtensionsFromArchive(OSData * mkext)
 *********************************************************************/
 kern_return_t IOCatalogue::removeKernelLinker(void) {
     kern_return_t result = KERN_SUCCESS;
-    struct segment_command * segment;
+    struct segment_command * segmentLE, *segmentKLD;
+    boolean_t	keepsyms = FALSE;
+#if __ppc__
     char * dt_segment_name;
     void * segment_paddress;
     int    segment_size;
+#endif
 
    /* This must be the very first thing done by this function.
     */
@@ -1196,6 +1198,8 @@ kern_return_t IOCatalogue::removeKernelLinker(void) {
         goto finish;
     }
 
+    PE_parse_boot_arg("keepsyms", &keepsyms);
+ 
     IOLog("Jettisoning kernel linker.\n");
 
     kernelLinkerPresent = 0;
@@ -1215,25 +1219,24 @@ kern_return_t IOCatalogue::removeKernelLinker(void) {
     * memory so that any cross-dependencies (not that there
     * should be any) are handled.
     */
-    segment = getsegbyname("__KLD");
-    if (!segment) {
+    segmentKLD = getsegbyname("__KLD");
+    if (!segmentKLD) {
         IOLog("error removing kernel linker: can't find %s segment\n",
             "__KLD");
         result = KERN_FAILURE;
         goto finish;
     }
-    OSRuntimeUnloadCPPForSegment(segment);
+    OSRuntimeUnloadCPPForSegment(segmentKLD);
 
-    segment = getsegbyname("__LINKEDIT");
-    if (!segment) {
+    segmentLE = getsegbyname("__LINKEDIT");
+    if (!segmentLE) {
         IOLog("error removing kernel linker: can't find %s segment\n",
             "__LINKEDIT");
         result = KERN_FAILURE;
         goto finish;
     }
-    OSRuntimeUnloadCPPForSegment(segment);
-
-
+    OSRuntimeUnloadCPPForSegment(segmentLE);
+#if __ppc__
    /* Free the memory that was set up by bootx.
     */
     dt_segment_name = "Kernel-__KLD";
@@ -1241,12 +1244,29 @@ kern_return_t IOCatalogue::removeKernelLinker(void) {
         IODTFreeLoaderInfo(dt_segment_name, (void *)segment_paddress,
             (int)segment_size);
     }
-
+#elif __i386__
+    /* On x86, use the mapping data from the segment load command to
+     * unload KLD and LINKEDIT directly, unless the keepsyms boot-arg
+     * was enabled. This may invalidate any assumptions about 
+     * "avail_start" defining the lower bound for valid physical addresses.
+     */
+    if (!keepsyms && segmentKLD->vmaddr && segmentKLD->vmsize)
+	    ml_static_mfree(segmentKLD->vmaddr, segmentKLD->vmsize);
+#else
+#error arch
+#endif
+#if __ppc__
     dt_segment_name = "Kernel-__LINKEDIT";
     if (0 == IODTGetLoaderInfo(dt_segment_name, &segment_paddress, &segment_size)) {
         IODTFreeLoaderInfo(dt_segment_name, (void *)segment_paddress,
             (int)segment_size);
     }
+#elif __i386__
+    if (!keepsyms && segmentLE->vmaddr && segmentLE->vmsize)
+	    ml_static_mfree(segmentLE->vmaddr, segmentLE->vmsize);
+#else
+#error arch
+#endif
 
     struct section * sect;
     sect = getsectbyname("__PRELINK", "__symtab");
diff --git a/iokit/Kernel/IOCommandGate.cpp b/iokit/Kernel/IOCommandGate.cpp
index 3c7c22854..f276ebc22 100644
--- a/iokit/Kernel/IOCommandGate.cpp
+++ b/iokit/Kernel/IOCommandGate.cpp
@@ -25,6 +25,8 @@
  * 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
+#include <libkern/OSDebug.h>
+
 #include <IOKit/IOCommandGate.h>
 #include <IOKit/IOWorkLoop.h>
 #include <IOKit/IOReturn.h>
@@ -62,71 +64,98 @@ IOCommandGate::commandGate(OSObject *inOwner, Action inAction)
     return me;
 }
 
-IOReturn IOCommandGate::runCommand(void *arg0, void *arg1,
-                                   void *arg2, void *arg3)
+/* virtual */ void IOCommandGate::disable()
 {
-    IOReturn res;
+    if (workLoop && !workLoop->inGate())
+	OSReportWithBacktrace("IOCommandGate::disable() called when not gated");
 
-    if (!enabled)
-        return kIOReturnNotPermitted;
+    super::disable();
+}
 
-    if (!action)
-        return kIOReturnNoResources;
+/* virtual */ void IOCommandGate::enable()
+{
+    if (workLoop) {
+	closeGate();
+	super::enable();
+	wakeupGate(&enabled, /* oneThread */ false); // Unblock sleeping threads
+	openGate();
+    }
+}
 
-    // closeGate is recursive so don't worry if we already hold the lock.
-    IOTimeStampConstant(IODBG_CMDQ(IOCMDQ_ACTION),
-			(unsigned int) action, (unsigned int) owner);
+/* virtual */ void IOCommandGate::free()
+{
+    setWorkLoop(0);
+    super::free();
+}
 
-    closeGate();
-    res = (*(Action) action)(owner, arg0, arg1, arg2, arg3);
-    openGate();
+/* virtual */ void IOCommandGate::setWorkLoop(IOWorkLoop *inWorkLoop)
+{
+    uintptr_t *sleepersP = (uintptr_t *) &reserved;
+    if (!inWorkLoop && workLoop) {		// tearing down
+	closeGate();
+	*sleepersP |= 1;
+	while (*sleepersP >> 1) {
+	    thread_wakeup_with_result(&enabled, THREAD_INTERRUPTED);
+	    sleepGate(sleepersP, THREAD_UNINT);
+	}
+	*sleepersP = 0;
+	openGate();
+    }
+    else
 
-    return res;
+    super::setWorkLoop(inWorkLoop);
+}
+
+IOReturn IOCommandGate::runCommand(void *arg0, void *arg1,
+                                   void *arg2, void *arg3)
+{
+    return runAction((Action) action, arg0, arg1, arg2, arg3);
+}
+
+IOReturn IOCommandGate::attemptCommand(void *arg0, void *arg1,
+                                       void *arg2, void *arg3)
+{
+    return attemptAction((Action) action, arg0, arg1, arg2, arg3);
 }
 
 IOReturn IOCommandGate::runAction(Action inAction,
                                   void *arg0, void *arg1,
                                   void *arg2, void *arg3)
 {
-    IOReturn res;
-
-    if (!enabled)
-        return kIOReturnNotPermitted;
-
     if (!inAction)
         return kIOReturnBadArgument;
 
     IOTimeStampConstant(IODBG_CMDQ(IOCMDQ_ACTION),
 			(unsigned int) inAction, (unsigned int) owner);
 
-    // closeGate is recursive so don't worry if we already hold the lock.
+    // closeGate is recursive needn't worry if we already hold the lock.
     closeGate();
-    res = (*inAction)(owner, arg0, arg1, arg2, arg3);
-    openGate();
 
-    return res;
-}
-
-IOReturn IOCommandGate::attemptCommand(void *arg0, void *arg1,
-                                       void *arg2, void *arg3)
-{
+    // If the command gate is disabled and we aren't on the workloop thread
+    // itself then sleep until we get enabled.
     IOReturn res;
+    if (!workLoop->onThread()) {
+	while (!enabled) {
+	    uintptr_t *sleepersP = (uintptr_t *) &reserved;
 
-    if (!enabled)
-        return kIOReturnNotPermitted;
+	    *sleepersP += 2;
+	    IOReturn res = sleepGate(&enabled, THREAD_ABORTSAFE);
+	    *sleepersP -= 2;
 
-    if (!action)
-        return kIOReturnNoResources;
+	    bool wakeupTearDown = (*sleepersP & 1);
+	    if (res || wakeupTearDown) {
+		openGate();
 
-    // Try to hold the lock if can't get return immediately.
-    if (!tryCloseGate())
-        return kIOReturnCannotLock;
+		 if (wakeupTearDown)
+		     commandWakeup(sleepersP);	// No further resources used
 
-    // closeGate is recursive so don't worry if we already hold the lock.
-    IOTimeStampConstant(IODBG_CMDQ(IOCMDQ_ACTION),
-			(unsigned int) action, (unsigned int) owner);
+		return kIOReturnAborted;
+	    }
+	}
+    }
 
-    res = (*(Action) action)(owner, arg0, arg1, arg2, arg3);
+    // Must be gated and on the work loop or enabled
+    res = (*inAction)(owner, arg0, arg1, arg2, arg3);
     openGate();
 
     return res;
@@ -138,9 +167,6 @@ IOReturn IOCommandGate::attemptAction(Action inAction,
 {
     IOReturn res;
 
-    if (!enabled)
-        return kIOReturnNotPermitted;
-
     if (!inAction)
         return kIOReturnBadArgument;
 
@@ -148,10 +174,16 @@ IOReturn IOCommandGate::attemptAction(Action inAction,
     if (!tryCloseGate())
         return kIOReturnCannotLock;
 
-    IOTimeStampConstant(IODBG_CMDQ(IOCMDQ_ACTION),
-			(unsigned int) inAction, (unsigned int) owner);
+    // If the command gate is disabled then sleep until we get a wakeup
+    if (!workLoop->onThread() && !enabled)
+        res = kIOReturnNotPermitted;
+    else {
+	IOTimeStampConstant(IODBG_CMDQ(IOCMDQ_ACTION),
+			    (unsigned int) inAction, (unsigned int) owner);
+
+	res = (*inAction)(owner, arg0, arg1, arg2, arg3);
+    }
 
-    res = (*inAction)(owner, arg0, arg1, arg2, arg3);
     openGate();
 
     return res;
diff --git a/iokit/Kernel/IOCommandPool.cpp b/iokit/Kernel/IOCommandPool.cpp
index 3bddd5b8a..2a3b6e71b 100644
--- a/iokit/Kernel/IOCommandPool.cpp
+++ b/iokit/Kernel/IOCommandPool.cpp
@@ -147,9 +147,10 @@ IOCommandPool::getCommand(bool blockForCommand)
     IOReturn	 result  = kIOReturnSuccess;
     IOCommand *command = 0;
 
-    result = fSerializer->runAction((IOCommandGate::Action)
-        &IOCommandPool::gatedGetCommand, 
-            (void *) &command, (void *) blockForCommand);
+    IOCommandGate::Action func = OSMemberFunctionCast(
+	    IOCommandGate::Action, this, &IOCommandPool::gatedGetCommand);
+    result = fSerializer->
+	runAction(func, (void *) &command, (void *) blockForCommand);
     if (kIOReturnSuccess == result)
         return command;
     else
@@ -186,8 +187,9 @@ gatedGetCommand(IOCommand **command, bool blockForCommand)
 void IOCommandPool::
 returnCommand(IOCommand *command)
 {
-    (void) fSerializer->runAction((IOCommandGate::Action)
-        &IOCommandPool::gatedReturnCommand, (void *) command);
+    IOCommandGate::Action func = OSMemberFunctionCast(
+	    IOCommandGate::Action, this, &IOCommandPool::gatedReturnCommand);
+    (void) fSerializer->runAction(func, (void *) command);
 }
 
 
@@ -199,7 +201,7 @@ returnCommand(IOCommand *command)
 IOReturn IOCommandPool::
 gatedReturnCommand(IOCommand *command)
 {
-    queue_enter(&fQueueHead, command, IOCommand *, fCommandChain);
+    queue_enter_first(&fQueueHead, command, IOCommand *, fCommandChain);
     if (fSleepers) {
         fSerializer->commandWakeup(&fSleepers, /* oneThread */ true);
         fSleepers--;
diff --git a/iokit/Kernel/IOCopyMapper.cpp b/iokit/Kernel/IOCopyMapper.cpp
new file mode 100644
index 000000000..eef092d65
--- /dev/null
+++ b/iokit/Kernel/IOCopyMapper.cpp
@@ -0,0 +1,426 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+// 45678901234567890123456789012345678901234567890123456789012345678901234567890
+
+#include "IOCopyMapper.h"
+#include <sys/sysctl.h>
+
+#if 0
+#define DEBG(fmt, args...)  	{ kprintf(fmt, ## args); }
+#else
+#define DEBG(fmt, args...)  	{}
+#endif
+
+extern "C" {
+extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
+extern void ml_get_bouncepool_info(
+			       vm_offset_t *phys_addr,
+			       vm_size_t   *size);
+extern unsigned int vm_lopage_max_count;
+extern unsigned int vm_himemory_mode;
+}
+
+#define super IOMapper
+
+OSDefineMetaClassAndStructors(IOCopyMapper, IOMapper);
+
+// Remember no value can be bigger than 31 bits as the sign bit indicates
+// that this entry is valid to the hardware and that would be bad if it wasn't
+typedef struct FreeDARTEntry {
+#if __BIG_ENDIAN__
+    unsigned int
+    /* bool */	    fValid : 1,
+    /* bool */	    fInUse : 1,	// Allocated but not inserted yet
+    /* bool */		   : 5,	// Align size on nibble boundary for debugging
+    /* uint */	    fSize  : 5,
+    /* uint */	           : 2,
+    /* uint */	    fNext  :18;	// offset of FreeDARTEntry's
+
+#elif __LITTLE_ENDIAN__
+    unsigned int
+    /* uint */	    fNext  :18,	// offset of FreeDARTEntry's
+    /* uint */	           : 2,
+    /* uint */	    fSize  : 5,
+    /* bool */		   : 5,	// Align size on nibble boundary for debugging
+    /* bool */	    fInUse : 1,	// Allocated but not inserted yet
+    /* bool */	    fValid : 1;
+#endif
+#if __BIG_ENDIAN__
+    unsigned int
+    /* uint */	           :14,
+    /* uint */	    fPrev  :18;	// offset of FreeDARTEntry's
+
+#elif __LITTLE_ENDIAN__
+    unsigned int
+    /* uint */	    fPrev  :18,	// offset of FreeDARTEntry's
+    /* uint */	           :14;
+#endif
+} FreeDARTEntry;
+
+typedef struct ActiveDARTEntry {
+#if __BIG_ENDIAN__
+    unsigned int
+    /* bool */	    fValid : 1,	// Must be set to one if valid
+    /* uint */	    fPPNum :31;	// ppnum_t page of translation
+#define ACTIVEDARTENTRY(page)	{ true, page }
+
+#elif __LITTLE_ENDIAN__
+    unsigned int
+    /* uint */	    fPPNum :31,	// ppnum_t page of translation
+    /* bool */	    fValid : 1;	// Must be set to one if valid
+#define ACTIVEDARTENTRY(page)	{ page, true }
+
+#endif
+};
+
+#define kActivePerFree (sizeof(freeDART[0]) / sizeof(ActiveDARTEntry))
+
+static SYSCTL_UINT(_kern, OID_AUTO, copyregionmax, 
+				CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
+				 NULL, 0, "");
+
+static SYSCTL_UINT(_kern, OID_AUTO, lowpagemax, 
+				CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
+				&vm_lopage_max_count, 0, "");
+
+static SYSCTL_UINT(_kern, OID_AUTO, himemorymode, 
+				CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
+				&vm_himemory_mode, 0, "");
+
+bool IOCopyMapper::initHardware(IOService * provider)
+{
+    UInt32 dartSizePages = 0;
+
+    vm_offset_t phys_addr;
+    vm_size_t   size;
+    ml_get_bouncepool_info(&phys_addr, &size);
+
+    if (!size)
+	return (false);
+
+    fBufferPage = atop_32(phys_addr);
+    dartSizePages = (atop_32(size) + kTransPerPage - 1) / kTransPerPage;
+
+    fTableLock = IOLockAlloc();
+
+    if (!fTableLock)
+	return false;
+
+    if (!allocTable(dartSizePages * kMapperPage))
+	return false;
+
+    UInt32 canMapPages = dartSizePages * kTransPerPage;
+    fMapperRegionSize = canMapPages;
+    for (fNumZones = 0; canMapPages; fNumZones++)
+	canMapPages >>= 1;
+    fNumZones -= 3; // correct for overshoot and minumum 16K pages allocation
+
+    invalidateDART(0, fMapperRegionSize);
+
+    breakUp(0, fNumZones, 0);
+    ((FreeDARTEntry *) fTable)->fInUse = true;
+    
+    fMapperRegionUsed    = kMinZoneSize;
+    fMapperRegionMaxUsed = fMapperRegionUsed;
+
+    sysctl__kern_copyregionmax.oid_arg1 = &fMapperRegionMaxUsed;
+
+    sysctl_register_oid(&sysctl__kern_copyregionmax);
+    sysctl_register_oid(&sysctl__kern_lowpagemax);
+    sysctl_register_oid(&sysctl__kern_himemorymode);
+
+    fDummyPage = IOMallocAligned(0x1000, 0x1000);
+    fDummyPageNumber =
+	pmap_find_phys(kernel_pmap, (addr64_t) (uintptr_t) fDummyPage);
+
+    return true;
+}
+
+void IOCopyMapper::free()
+{
+    if (fDummyPage) {
+	IOFreeAligned(fDummyPage, 0x1000);
+	fDummyPage = 0;
+	fDummyPageNumber = 0;
+    }
+
+    if (fTableLock) {
+	IOLockFree(fTableLock);
+	fTableLock = 0;
+    }
+
+    super::free();
+}
+
+// Must be called while locked
+void IOCopyMapper::breakUp(unsigned startIndex, unsigned endIndex, unsigned freeInd)
+{
+    unsigned int zoneSize;
+    FreeDARTEntry *freeDART = (FreeDARTEntry *) fTable;
+
+    do {
+	// Need to break up bigger blocks of memory till we get one in our 
+	// desired zone.
+	endIndex--;
+	zoneSize = (kMinZoneSize/2 << endIndex);
+	ppnum_t tail = freeInd + zoneSize;
+
+	DEBG("breakup z %d start %x tail %x\n", endIndex, freeInd, tail);
+
+	// By definition free lists must be empty
+	fFreeLists[endIndex] = tail;
+	freeDART[tail].fSize = endIndex;
+	freeDART[tail].fNext = freeDART[tail].fPrev = 0;
+    } while (endIndex != startIndex);
+    freeDART[freeInd].fSize = endIndex;
+}
+
+// Zero is never a valid page to return
+ppnum_t IOCopyMapper::iovmAlloc(IOItemCount pages)
+{
+    unsigned int zone, zoneSize, z, cnt;
+    ppnum_t next, ret = 0;
+    FreeDARTEntry *freeDART = (FreeDARTEntry *) fTable;
+
+    // Can't alloc anything of less than minumum
+    if (pages < kMinZoneSize)
+	pages = kMinZoneSize;
+
+    // Can't alloc anything bigger than 1/2 table
+    if (pages >= fMapperRegionSize/2)
+    {
+	panic("iovmAlloc 0x%x", pages);
+	return 0;
+    }
+
+    // Find the appropriate zone for this allocation
+    for (zone = 0, zoneSize = kMinZoneSize; pages > zoneSize; zone++)
+	zoneSize <<= 1;
+
+    {
+	IOLockLock(fTableLock);
+
+	for (;;) {
+	    for (z = zone; z < fNumZones; z++) {
+		if ( (ret = fFreeLists[z]) )
+		    break;
+	    }
+	    if (ret)
+		break;
+
+	    fFreeSleepers++;
+	    IOLockSleep(fTableLock, fFreeLists, THREAD_UNINT);
+	    fFreeSleepers--;
+	}
+
+	// If we didn't find a entry in our size then break up the free block
+	// that we did find.
+	if (zone != z)
+	{
+	    DEBG("breakup %d, %d, 0x%x\n", zone, z, ret);
+	    breakUp(zone, z, ret);
+	}
+
+	freeDART[ret].fInUse = true;	// Mark entry as In Use
+	next = freeDART[ret].fNext;
+	DEBG("va:  0x%x, %d, ret %x next %x\n", (ret * kActivePerFree) + fBufferPage, pages, ret, next);
+
+	fFreeLists[z] = next;
+	if (next)
+	    freeDART[next].fPrev = 0;
+
+	// ret is free list offset not page offset;
+	ret *= kActivePerFree;
+
+	ActiveDARTEntry pageEntry = ACTIVEDARTENTRY(fDummyPageNumber);
+	for (cnt = 0; cnt < pages; cnt++) {
+	    ActiveDARTEntry *activeDART = &fMappings[ret + cnt];
+	    *activeDART = pageEntry;
+	}
+
+	fMapperRegionUsed += pages;
+	if (fMapperRegionUsed > fMapperRegionMaxUsed)
+	    fMapperRegionMaxUsed = fMapperRegionUsed;
+
+	IOLockUnlock(fTableLock);
+    }
+
+    if (ret)
+	ret += fBufferPage;
+
+    return ret;
+}
+
+
+void IOCopyMapper::invalidateDART(ppnum_t pnum, IOItemCount size)
+{
+    bzero((void *) &fMappings[pnum], size * sizeof(fMappings[0]));
+}
+
+void IOCopyMapper::iovmFree(ppnum_t addr, IOItemCount pages)
+{
+    unsigned int zone, zoneSize, z;
+    FreeDARTEntry *freeDART = (FreeDARTEntry *) fTable;
+
+    if (addr < fBufferPage)
+	IOPanic("addr < fBufferPage");
+    addr -= fBufferPage;
+
+    // Can't free anything of less than minumum
+    if (pages < kMinZoneSize)
+	pages = kMinZoneSize;
+
+    // Can't free anything bigger than 1/2 table
+    if (pages >= fMapperRegionSize/2)
+	return;
+
+    // Find the appropriate zone for this allocation
+    for (zone = 0, zoneSize = kMinZoneSize; pages > zoneSize; zone++)
+	zoneSize <<= 1;
+
+    // Grab lock that protects the dart
+    IOLockLock(fTableLock);
+
+    invalidateDART(addr, pages);
+
+    addr /= kActivePerFree;
+
+    // We are freeing a block, check to see if pairs are available for 
+    // coalescing.  We will walk up the entire chain if we can.
+    for (z = zone; z < fNumZones; z++) {
+	ppnum_t pair = addr ^ (kMinZoneSize/2 << z);	// Find pair address
+	if (freeDART[pair].fValid || freeDART[pair].fInUse || (freeDART[pair].fSize != z))
+	    break;
+
+	// The paired alloc entry is free if we are here
+	ppnum_t next = freeDART[pair].fNext;
+	ppnum_t prev = freeDART[pair].fPrev;
+
+	// Remove the pair from its freeList
+	if (prev)
+	    freeDART[prev].fNext = next;
+	else
+	    fFreeLists[z] = next;
+
+	if (next)
+	    freeDART[next].fPrev = prev;
+
+	// Sort the addr and the pair
+	if (addr > pair)
+	    addr = pair;
+    }
+
+    DEBG("vf:  0x%x, %d, z %d, head %x, new %x\n", addr * kActivePerFree + fBufferPage, pages, z, fFreeLists[z], addr);
+
+    // Add the allocation entry into it's free list and re-init it
+    freeDART[addr].fSize = z;
+    freeDART[addr].fNext = fFreeLists[z];
+    if (fFreeLists[z])
+	freeDART[fFreeLists[z]].fPrev = addr;
+    freeDART[addr].fPrev = 0;
+    fFreeLists[z] = addr;
+
+    fMapperRegionUsed -= pages;
+
+    if (fFreeSleepers)
+	IOLockWakeup(fTableLock, fFreeLists, /* oneThread */ false);
+
+    IOLockUnlock(fTableLock);
+}
+
+addr64_t IOCopyMapper::mapAddr(IOPhysicalAddress addr)
+{
+    if (addr < ptoa_32(fBufferPage))
+    {
+	return (addr64_t) addr;	// Not mapped by us anyway
+    }
+
+    addr -= ptoa_32(fBufferPage);
+    if (addr >= ptoa_32(fMapperRegionSize))
+    {
+	return (addr64_t) addr;	// Not mapped by us anyway
+    }
+    else
+    {
+	ActiveDARTEntry *activeDART = (ActiveDARTEntry *) fTable;
+	UInt offset = addr & PAGE_MASK;
+
+	ActiveDARTEntry mappedPage = activeDART[atop_32(addr)];
+	if (mappedPage.fValid)
+    {
+	    return (ptoa_64(mappedPage.fPPNum) | offset);
+	}
+
+	panic("%s::mapAddr(0x%08lx) not mapped for I/O\n", getName(), addr);
+	return 0;
+    }
+}
+
+void IOCopyMapper::iovmInsert(ppnum_t addr, IOItemCount offset, ppnum_t page)
+{
+    addr -= fBufferPage;
+    addr += offset;	// Add the offset page to the base address
+
+    ActiveDARTEntry *activeDART = &fMappings[addr];
+    ActiveDARTEntry entry = ACTIVEDARTENTRY(page);
+    *activeDART = entry;
+}
+
+void IOCopyMapper::iovmInsert(ppnum_t addr, IOItemCount offset,
+	ppnum_t *pageList, IOItemCount pageCount)
+{
+    addr -= fBufferPage;
+    addr += offset;	// Add the offset page to the base address
+
+    IOItemCount i;
+    ActiveDARTEntry *activeDART = &fMappings[addr];
+
+    for (i = 0; i < pageCount; i++)
+    {
+	ActiveDARTEntry entry = ACTIVEDARTENTRY(pageList[i]);
+	activeDART[i] = entry;
+    }
+}
+
+void IOCopyMapper::iovmInsert(ppnum_t addr, IOItemCount offset,
+	upl_page_info_t *pageList, IOItemCount pageCount)
+{
+    addr -= fBufferPage;
+    addr += offset;	// Add the offset page to the base address
+
+    IOItemCount i;
+    ActiveDARTEntry *activeDART = &fMappings[addr];
+
+    for (i = 0; i < pageCount; i++)
+    {
+	ActiveDARTEntry entry = ACTIVEDARTENTRY(pageList[i].phys_addr);
+	activeDART[i] = entry;
+    }
+}
+
+
diff --git a/iokit/Kernel/IOCopyMapper.h b/iokit/Kernel/IOCopyMapper.h
new file mode 100644
index 000000000..5371afa04
--- /dev/null
+++ b/iokit/Kernel/IOCopyMapper.h
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+// 45678901234567890123456789012345678901234567890123456789012345678901234567890
+
+#include <libkern/OSAtomic.h>
+
+#include <IOKit/IOLocks.h>
+#include <IOKit/IOPlatformExpert.h>
+#include <IOKit/IODeviceTreeSupport.h>
+#include <IOKit/IOMapper.h>
+
+// General constants about all VART/DART style Address Re-Mapping Tables
+#define kMapperPage	    (4 * 1024)
+#define kTransPerPage	    (kMapperPage / sizeof(ppnum_t))
+
+#define kMinZoneSize	    4		// Minimum Zone size in pages
+#define kMaxNumZones	    (31 - 14)	// 31 bit mapped in 16K super pages
+
+class IOCopyMapper : public IOMapper
+{
+    OSDeclareDefaultStructors(IOCopyMapper);
+
+// alias the fTable variable into our mappings table
+#define fMappings	((ActiveDARTEntry *) super::fTable)
+
+private:
+
+    UInt32		fFreeLists[kMaxNumZones];
+
+    IOLock		*fTableLock;
+
+    void		*fDummyPage;
+
+    UInt32		 fNumZones;
+    UInt32		 fMapperRegionSize;
+    UInt32		 fMapperRegionUsed;
+    UInt32		 fMapperRegionMaxUsed;
+    UInt32		 fFreeSleepers;
+    ppnum_t		 fDummyPageNumber;
+    ppnum_t		 fBufferPage;
+
+    // Internal functions
+
+    void breakUp(unsigned start, unsigned end, unsigned freeInd);
+    void invalidateDART(ppnum_t pnum, IOItemCount size);
+    void tlbInvalidate(ppnum_t pnum, IOItemCount size);
+
+    virtual void free();
+
+    virtual bool initHardware(IOService * provider);
+public:
+    virtual ppnum_t iovmAlloc(IOItemCount pages);
+    virtual void iovmFree(ppnum_t addr, IOItemCount pages);
+
+    virtual void iovmInsert(ppnum_t addr, IOItemCount offset, ppnum_t page);
+    virtual void iovmInsert(ppnum_t addr, IOItemCount offset,
+                            ppnum_t *pageList, IOItemCount pageCount);
+    virtual void iovmInsert(ppnum_t addr, IOItemCount offset,
+                            upl_page_info_t *pageList, IOItemCount pageCount);
+
+    virtual addr64_t mapAddr(IOPhysicalAddress addr);
+};
+
+extern IOCopyMapper * gIOCopyMapper;
diff --git a/iokit/Kernel/IODMACommand.cpp b/iokit/Kernel/IODMACommand.cpp
new file mode 100644
index 000000000..0c2956ce7
--- /dev/null
+++ b/iokit/Kernel/IODMACommand.cpp
@@ -0,0 +1,970 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <IOKit/assert.h>
+
+#include <libkern/OSTypes.h>
+#include <libkern/OSByteOrder.h>
+
+#include <IOKit/IOReturn.h>
+#include <IOKit/IOLib.h>
+#include <IOKit/IODMACommand.h>
+#include <IOKit/IOMapper.h>
+#include <IOKit/IOMemoryDescriptor.h>
+#include <IOKit/IOBufferMemoryDescriptor.h>
+
+#include "IOKitKernelInternal.h"
+#include "IOCopyMapper.h"
+
+#define MAPTYPE(type)		((UInt) (type) & kTypeMask)
+#define IS_MAPPED(type)		(MAPTYPE(type) == kMapped)
+#define IS_BYPASSED(type)	(MAPTYPE(type) == kBypassed)
+#define IS_NONCOHERENT(type)	(MAPTYPE(type) == kNonCoherent)
+
+
+static bool gIOEnableCopyMapper  = true;
+
+enum 
+{
+    kWalkSyncIn       = 0x01,	// bounce -> md 
+    kWalkSyncOut      = 0x02,	// bounce <- md
+    kWalkSyncAlways   = 0x04,
+    kWalkPreflight    = 0x08,
+    kWalkDoubleBuffer = 0x10,
+    kWalkPrepare      = 0x20,
+    kWalkComplete     = 0x40,
+    kWalkClient       = 0x80
+};
+
+struct ExpansionData
+{
+    IOMDDMAWalkSegmentState fState;
+    IOMDDMACharacteristics  fMDSummary;
+
+    UInt64 fPreparedOffset;
+    UInt64 fPreparedLength;
+
+    UInt8  fCursor;
+    UInt8  fCheckAddressing;
+    UInt8  fIterateOnly;
+    UInt8  fMisaligned;
+    UInt8  fCopyContig;
+    UInt8  fPrepared;
+    UInt8  fDoubleBuffer;
+    UInt8  __pad[1];
+
+    ppnum_t  fCopyPageAlloc;
+    ppnum_t  fCopyPageCount;
+    addr64_t fCopyNext;
+
+    class IOBufferMemoryDescriptor * fCopyMD;
+};
+typedef ExpansionData IODMACommandInternal;
+
+#define fInternalState reserved
+#define fState         reserved->fState
+#define fMDSummary     reserved->fMDSummary
+
+
+#if 1
+// no direction => OutIn
+#define SHOULD_COPY_DIR(op, direction)					    \
+	((kIODirectionNone == (direction))				    \
+	    || (kWalkSyncAlways & (op))					    \
+	    || (((kWalkSyncIn & (op)) ? kIODirectionIn : kIODirectionOut)   \
+						    & (direction)))
+
+#else
+#define SHOULD_COPY_DIR(state, direction) (true)
+#endif
+
+#if 0
+#define DEBG(fmt, args...)  	{ kprintf(fmt, ## args); }
+#else
+#define DEBG(fmt, args...)  	{}
+#endif
+
+
+/**************************** class IODMACommand ***************************/
+
+#undef super
+#define super OSObject
+OSDefineMetaClassAndStructors(IODMACommand, IOCommand);
+
+OSMetaClassDefineReservedUnused(IODMACommand,  0);
+OSMetaClassDefineReservedUnused(IODMACommand,  1);
+OSMetaClassDefineReservedUnused(IODMACommand,  2);
+OSMetaClassDefineReservedUnused(IODMACommand,  3);
+OSMetaClassDefineReservedUnused(IODMACommand,  4);
+OSMetaClassDefineReservedUnused(IODMACommand,  5);
+OSMetaClassDefineReservedUnused(IODMACommand,  6);
+OSMetaClassDefineReservedUnused(IODMACommand,  7);
+OSMetaClassDefineReservedUnused(IODMACommand,  8);
+OSMetaClassDefineReservedUnused(IODMACommand,  9);
+OSMetaClassDefineReservedUnused(IODMACommand, 10);
+OSMetaClassDefineReservedUnused(IODMACommand, 11);
+OSMetaClassDefineReservedUnused(IODMACommand, 12);
+OSMetaClassDefineReservedUnused(IODMACommand, 13);
+OSMetaClassDefineReservedUnused(IODMACommand, 14);
+OSMetaClassDefineReservedUnused(IODMACommand, 15);
+
+IODMACommand *
+IODMACommand::withSpecification(SegmentFunction outSegFunc,
+				UInt8           numAddressBits,
+				UInt64          maxSegmentSize,
+				MappingOptions  mappingOptions,
+				UInt64          maxTransferSize,
+				UInt32          alignment,
+				IOMapper       *mapper,
+				void           *refCon)
+{
+    IODMACommand * me = new IODMACommand;
+
+    if (me && !me->initWithSpecification(outSegFunc,
+					 numAddressBits, maxSegmentSize,
+					 mappingOptions, maxTransferSize,
+					 alignment,      mapper, refCon))
+    {
+        me->release();
+        return 0;
+    };
+
+    return me;
+}
+
+IODMACommand *
+IODMACommand::cloneCommand(void *refCon)
+{
+    return withSpecification(fOutSeg, fNumAddressBits, fMaxSegmentSize,
+	    fMappingOptions, fMaxTransferSize, fAlignMask + 1, fMapper, refCon);
+}
+
+#define kLastOutputFunction ((SegmentFunction) kLastOutputFunction)
+
+bool
+IODMACommand::initWithSpecification(SegmentFunction outSegFunc,
+				    UInt8           numAddressBits,
+				    UInt64          maxSegmentSize,
+				    MappingOptions  mappingOptions,
+				    UInt64          maxTransferSize,
+				    UInt32          alignment,
+				    IOMapper       *mapper,
+				    void           *refCon)
+{
+    if (!super::init() || !outSegFunc || !numAddressBits)
+        return false;
+
+    bool is32Bit = (OutputHost32   == outSegFunc || OutputBig32 == outSegFunc
+                 || OutputLittle32 == outSegFunc);
+    if (is32Bit)
+    {
+	if (!numAddressBits)
+	    numAddressBits = 32;
+	else if (numAddressBits > 32)
+	    return false;		// Wrong output function for bits
+    }
+
+    if (numAddressBits && (numAddressBits < PAGE_SHIFT))
+	return false;
+
+    if (!maxSegmentSize)
+	maxSegmentSize--;	// Set Max segment to -1
+    if (!maxTransferSize)
+	maxTransferSize--;	// Set Max transfer to -1
+
+    if (!mapper)
+    {
+        IOMapper::checkForSystemMapper();
+	mapper = IOMapper::gSystem;
+    }
+
+    fNumSegments     = 0;
+    fBypassMask      = 0;
+    fOutSeg	     = outSegFunc;
+    fNumAddressBits  = numAddressBits;
+    fMaxSegmentSize  = maxSegmentSize;
+    fMappingOptions  = mappingOptions;
+    fMaxTransferSize = maxTransferSize;
+    if (!alignment)
+	alignment = 1;
+    fAlignMask	     = alignment - 1;
+    fMapper          = mapper;
+    fRefCon          = refCon;
+
+    switch (MAPTYPE(mappingOptions))
+    {
+    case kMapped:                   break;
+    case kNonCoherent: fMapper = 0; break;
+    case kBypassed:
+	if (mapper && !mapper->getBypassMask(&fBypassMask))
+	    return false;
+	break;
+    default:
+	return false;
+    };
+
+    reserved = IONew(ExpansionData, 1);
+    if (!reserved)
+	return false;
+    bzero(reserved, sizeof(ExpansionData));
+
+    fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions));
+    
+    return true;
+}
+
+void
+IODMACommand::free()
+{
+    if (reserved)
+	IODelete(reserved, ExpansionData, 1);
+
+    super::free();
+}
+
+IOReturn
+IODMACommand::setMemoryDescriptor(const IOMemoryDescriptor *mem, bool autoPrepare)
+{
+    if (mem == fMemory)
+    {
+	if (!autoPrepare)
+	{
+	    while (fActive)
+		complete();
+	}
+	return kIOReturnSuccess;
+    }
+
+    if (fMemory) {
+	// As we are almost certainly being called from a work loop thread
+	// if fActive is true it is probably not a good time to potentially
+	// block.  Just test for it and return an error
+	if (fActive)
+	    return kIOReturnBusy;
+	clearMemoryDescriptor();
+    };
+
+    if (mem) {
+	bzero(&fMDSummary, sizeof(fMDSummary));
+	IOReturn rtn = mem->dmaCommandOperation(
+		kIOMDGetCharacteristics,
+		&fMDSummary, sizeof(fMDSummary));
+	if (rtn)
+	    return rtn;
+
+	ppnum_t highPage = fMDSummary.fHighestPage ? fMDSummary.fHighestPage : gIOLastPage;
+
+	if ((kMapped == MAPTYPE(fMappingOptions))
+	    && fMapper 
+	    && (!fNumAddressBits || (fNumAddressBits >= 31)))
+	    // assuming mapped space is 2G
+	    fInternalState->fCheckAddressing = false;
+	else
+	    fInternalState->fCheckAddressing = (fNumAddressBits && (highPage >= (1UL << (fNumAddressBits - PAGE_SHIFT))));
+
+	mem->retain();
+	fMemory = mem;
+
+	if (autoPrepare)
+	    return prepare();
+    };
+
+    return kIOReturnSuccess;
+}
+
+IOReturn
+IODMACommand::clearMemoryDescriptor(bool autoComplete)
+{
+    if (fActive && !autoComplete)
+	return (kIOReturnNotReady);
+
+    if (fMemory) {
+	while (fActive)
+	    complete();
+	fMemory->release();
+	fMemory = 0;
+    }
+
+    return (kIOReturnSuccess);
+}
+
+const IOMemoryDescriptor *
+IODMACommand::getMemoryDescriptor() const
+{
+    return fMemory;
+}
+
+
+IOReturn
+IODMACommand::segmentOp(
+			void         *reference,
+			IODMACommand *target,
+			Segment64     segment,
+			void         *segments,
+			UInt32        segmentIndex)
+{
+    IOOptionBits op = (IOOptionBits) reference;
+    addr64_t     maxPhys, address;
+    addr64_t     remapAddr = 0;
+    uint64_t     length;
+    uint32_t     numPages;
+
+    IODMACommandInternal * state = target->reserved;
+
+    if (target->fNumAddressBits && (target->fNumAddressBits < 64))
+	maxPhys = (1ULL << target->fNumAddressBits);
+    else
+	maxPhys = 0;
+    maxPhys--;
+
+    address = segment.fIOVMAddr;
+    length = segment.fLength;
+
+    assert(address);
+    assert(length);
+
+    if (!state->fMisaligned)
+    {
+	state->fMisaligned |= (0 != (target->fAlignMask & address));
+	if (state->fMisaligned) DEBG("misaligned %qx:%qx, %lx\n", address, length, target->fAlignMask);
+    }
+
+    if (state->fMisaligned && (kWalkPreflight & op))
+	return (kIOReturnNotAligned);
+
+    if (!state->fDoubleBuffer)
+    {
+	if ((address + length - 1) <= maxPhys)
+	{
+	    length = 0;
+	}
+	else if (address <= maxPhys)
+	{
+	    DEBG("tail %qx, %qx", address, length);
+	    length = (address + length - maxPhys - 1);
+	    address = maxPhys + 1;
+	    DEBG("-> %qx, %qx\n", address, length);
+	}
+    }
+
+    if (!length)
+	return (kIOReturnSuccess);
+
+    numPages = atop_64(round_page_64(length));
+    remapAddr = state->fCopyNext;
+
+    if (kWalkPreflight & op)
+    {
+	state->fCopyPageCount += numPages;
+    }
+    else
+    {
+	if (kWalkPrepare & op)
+	{
+	    for (IOItemCount idx = 0; idx < numPages; idx++)
+		gIOCopyMapper->iovmInsert(atop_64(remapAddr), idx, atop_64(address) + idx);
+	}
+	if (state->fDoubleBuffer)
+	    state->fCopyNext += length;
+	else
+	{
+	    state->fCopyNext += round_page(length);
+	    remapAddr += (address & PAGE_MASK);
+	}
+
+	if (SHOULD_COPY_DIR(op, target->fMDSummary.fDirection))
+	{
+	    DEBG("cpv: 0x%qx %s 0x%qx, 0x%qx, 0x%02lx\n", remapAddr, 
+			(kWalkSyncIn & op) ? "->" : "<-", 
+			address, length, op);
+	    if (kWalkSyncIn & op)
+	    { // cppvNoModSnk
+		copypv(remapAddr, address, length,
+				cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
+	    }
+	    else
+	    {
+		copypv(address, remapAddr, length,
+				cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
+	    }
+	}
+    }
+
+    return kIOReturnSuccess;
+}
+
+IOReturn
+IODMACommand::walkAll(UInt8 op)
+{
+    IODMACommandInternal * state = fInternalState;
+
+    IOReturn     ret = kIOReturnSuccess;
+    UInt32       numSegments;
+    UInt64       offset;
+
+    if (gIOEnableCopyMapper && (kWalkPreflight & op))
+    {
+	state->fCopyContig     = false;
+	state->fMisaligned     = false;
+	state->fDoubleBuffer   = false;
+	state->fPrepared       = false;
+	state->fCopyNext       = 0;
+	state->fCopyPageAlloc  = 0;
+	state->fCopyPageCount  = 0;
+	state->fCopyMD         = 0;
+
+	if (!(kWalkDoubleBuffer & op))
+	{
+	    offset = 0;
+	    numSegments = 0-1;
+	    ret = genIOVMSegments(segmentOp, (void *) op, &offset, state, &numSegments);
+	}
+
+	op &= ~kWalkPreflight;
+
+	state->fDoubleBuffer = (state->fMisaligned || (kWalkDoubleBuffer & op));
+	if (state->fDoubleBuffer)
+	    state->fCopyPageCount = atop_64(round_page(state->fPreparedLength));
+
+	if (state->fCopyPageCount)
+	{
+	    IOMapper * mapper;
+	    ppnum_t    mapBase = 0;
+
+	    DEBG("preflight fCopyPageCount %d\n", state->fCopyPageCount);
+
+	    mapper = gIOCopyMapper;
+	    if (mapper)
+		mapBase = mapper->iovmAlloc(state->fCopyPageCount);
+	    if (mapBase)
+	    {
+		state->fCopyPageAlloc = mapBase;
+		if (state->fCopyPageAlloc && state->fDoubleBuffer)
+		{
+		    DEBG("contig copy map\n");
+		    state->fCopyContig = true;
+		}
+
+		state->fCopyNext = ptoa_64(state->fCopyPageAlloc);
+		offset = 0;
+		numSegments = 0-1;
+		ret = genIOVMSegments(segmentOp, (void *) op, &offset, state, &numSegments);
+		state->fPrepared = true;
+		op &= ~(kWalkSyncIn | kWalkSyncOut);
+	    }
+	    else
+	    {
+		DEBG("alloc IOBMD\n");
+		state->fCopyMD = IOBufferMemoryDescriptor::withOptions(
+				    fMDSummary.fDirection, state->fPreparedLength, page_size);
+
+		if (state->fCopyMD)
+		{
+		    ret = kIOReturnSuccess;
+		    state->fPrepared = true;
+		}
+		else
+		{
+		    DEBG("IODMACommand !iovmAlloc");
+		    return (kIOReturnNoResources);
+		}
+	    }
+	}
+    }
+
+    if (gIOEnableCopyMapper && state->fPrepared && ((kWalkSyncIn | kWalkSyncOut) & op))
+    {
+	if (state->fCopyPageCount)
+	{
+	    DEBG("sync fCopyPageCount %d\n", state->fCopyPageCount);
+
+	    if (state->fCopyPageAlloc)
+	    {
+		state->fCopyNext = ptoa_64(state->fCopyPageAlloc);
+		offset = 0;
+		numSegments = 0-1;
+		ret = genIOVMSegments(segmentOp, (void *) op, &offset, state, &numSegments);
+	    }
+	    else if (state->fCopyMD)
+	    {
+		DEBG("sync IOBMD\n");
+
+		if (SHOULD_COPY_DIR(op, fMDSummary.fDirection))
+		{
+		    IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory);
+
+		    IOByteCount bytes;
+		    
+		    if (kWalkSyncIn & op)
+			bytes = poMD->writeBytes(state->fPreparedOffset, 
+						    state->fCopyMD->getBytesNoCopy(),
+						    state->fPreparedLength);
+		    else
+			bytes = poMD->readBytes(state->fPreparedOffset, 
+						    state->fCopyMD->getBytesNoCopy(),
+						    state->fPreparedLength);
+		    DEBG("fCopyMD %s %lx bytes\n", (kWalkSyncIn & op) ? "wrote" : "read", bytes);
+		    ret = (bytes == state->fPreparedLength) ? kIOReturnSuccess : kIOReturnUnderrun;
+		}
+		else
+		    ret = kIOReturnSuccess;
+	    }
+	}
+    }
+
+    if (kWalkComplete & op)
+    {
+	if (state->fCopyPageAlloc)
+	{
+	    gIOCopyMapper->iovmFree(state->fCopyPageAlloc, state->fCopyPageCount);
+	    state->fCopyPageAlloc = 0;
+	    state->fCopyPageCount = 0;
+	}
+	if (state->fCopyMD)
+	{
+	    state->fCopyMD->release();
+	    state->fCopyMD = 0;
+	}
+
+	state->fPrepared = false;
+    }
+    return (ret);
+}
+
+IOReturn 
+IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchronize)
+{
+    IODMACommandInternal * state = fInternalState;
+    IOReturn               ret   = kIOReturnSuccess;
+
+    if (!length)
+	length = fMDSummary.fLength;
+
+    if (length > fMaxTransferSize)
+	return kIOReturnNoSpace;
+
+#if 0
+    if (IS_NONCOHERENT(mappingOptions) && flushCache) {
+	IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory);
+
+	poMD->performOperation(kIOMemoryIncoherentIOStore, 0, fMDSummary.fLength);
+    }
+#endif
+    if (fActive++)
+    {
+	if ((state->fPreparedOffset != offset)
+	  || (state->fPreparedLength != length))
+	ret = kIOReturnNotReady;
+    }
+    else
+    {
+	state->fPreparedOffset = offset;
+	state->fPreparedLength = length;
+
+	state->fCopyContig     = false;
+	state->fMisaligned     = false;
+	state->fDoubleBuffer   = false;
+	state->fPrepared       = false;
+	state->fCopyNext       = 0;
+	state->fCopyPageAlloc  = 0;
+	state->fCopyPageCount  = 0;
+	state->fCopyMD         = 0;
+
+	state->fCursor = state->fIterateOnly
+			|| (!state->fCheckAddressing
+			    && (!fAlignMask
+				|| ((fMDSummary.fPageAlign & (1 << 31)) && (0 == (fMDSummary.fPageAlign & fAlignMask)))));
+	if (!state->fCursor)
+	{
+	    IOOptionBits op = kWalkPrepare | kWalkPreflight;
+	    if (synchronize)
+		op |= kWalkSyncOut;
+	    ret = walkAll(op);
+	}
+	if (kIOReturnSuccess == ret)
+	    state->fPrepared = true;
+    }
+    return ret;
+}
+
+IOReturn 
+IODMACommand::complete(bool invalidateCache, bool synchronize)
+{
+    IODMACommandInternal * state = fInternalState;
+    IOReturn               ret   = kIOReturnSuccess;
+
+    if (fActive < 1)
+	return kIOReturnNotReady;
+
+    if (!--fActive)
+    {
+	if (!state->fCursor)
+	{
+	    IOOptionBits op = kWalkComplete;
+	    if (synchronize)
+		op |= kWalkSyncIn;
+	    ret = walkAll(op);
+	}
+	state->fPrepared = false;
+
+#if 0
+	if (IS_NONCOHERENT(fMappingOptions) && invalidateCache)
+	{ 
+	    // XXX gvdl: need invalidate before Chardonnay ships
+	    IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory);
+
+	    poMD->performOperation(kIOMemoryIncoherentIOInvalidate, 0, fMDSummary.fLength);
+	}
+#endif
+    }
+
+    return ret;
+}
+
+IOReturn
+IODMACommand::synchronize(IOOptionBits options)
+{
+    IODMACommandInternal * state = fInternalState;
+    IOReturn		   ret   = kIOReturnSuccess;
+    IOOptionBits           op;
+
+    if (kIODirectionOutIn == (kIODirectionOutIn & options))
+	return kIOReturnBadArgument;
+
+    if (fActive < 1)
+	return kIOReturnNotReady;
+
+    op = 0;
+    if (kForceDoubleBuffer & options)
+    {
+	if (state->fDoubleBuffer)
+	    return kIOReturnSuccess;
+	if (state->fCursor)
+	    state->fCursor = false;
+	else
+	    ret = walkAll(kWalkComplete);
+
+	op |= kWalkPrepare | kWalkPreflight | kWalkDoubleBuffer;
+    }
+    else if (state->fCursor)
+	return kIOReturnSuccess;
+
+    if (kIODirectionIn & options)
+	op |= kWalkSyncIn | kWalkSyncAlways;
+    else if (kIODirectionOut & options)
+	op |= kWalkSyncOut | kWalkSyncAlways;
+
+    ret = walkAll(op);
+
+    return ret;
+}
+
+IOReturn
+IODMACommand::genIOVMSegments(UInt64 *offsetP,
+			      void   *segmentsP,
+			      UInt32 *numSegmentsP)
+{
+    return (genIOVMSegments(clientOutputSegment, (void *) kWalkClient, offsetP, segmentsP, numSegmentsP));
+}
+
+IOReturn
+IODMACommand::genIOVMSegments(InternalSegmentFunction outSegFunc,
+			      void   *reference,
+			      UInt64 *offsetP,
+			      void   *segmentsP,
+			      UInt32 *numSegmentsP)
+{
+    IOOptionBits           op = (IOOptionBits) reference;
+    IODMACommandInternal * internalState = fInternalState;
+    IOOptionBits           mdOp = kIOMDWalkSegments;
+    IOReturn               ret  = kIOReturnSuccess;
+
+    if (!(kWalkComplete & op) && !fActive)
+	return kIOReturnNotReady;
+
+    if (!offsetP || !segmentsP || !numSegmentsP || !*numSegmentsP)
+	return kIOReturnBadArgument;
+
+    IOMDDMAWalkSegmentArgs *state =
+	(IOMDDMAWalkSegmentArgs *) fState;
+
+    UInt64 offset = *offsetP + internalState->fPreparedOffset;
+    UInt64 memLength = internalState->fPreparedOffset + internalState->fPreparedLength;
+
+    if (offset >= memLength)
+	return kIOReturnOverrun;
+
+    if (!offset || offset != state->fOffset) {
+	state->fOffset   = 0;
+	state->fIOVMAddr = 0;
+	state->fMapped = (IS_MAPPED(fMappingOptions) && fMapper);
+	mdOp = kIOMDFirstSegment;
+    };
+	
+    UInt64    bypassMask = fBypassMask;
+    UInt32    segIndex = 0;
+    UInt32    numSegments = *numSegmentsP;
+    Segment64 curSeg = { 0, 0 };
+    addr64_t  maxPhys;
+
+    if (fNumAddressBits && (fNumAddressBits < 64))
+	maxPhys = (1ULL << fNumAddressBits);
+    else
+	maxPhys = 0;
+    maxPhys--;
+
+    while ((state->fIOVMAddr) || state->fOffset < memLength)
+    {
+        if (!state->fIOVMAddr) {
+
+	    IOReturn rtn;
+
+	    state->fOffset = offset;
+	    state->fLength = memLength - offset;
+
+	    if (internalState->fCopyContig && (kWalkClient & op))
+	    {
+		state->fIOVMAddr = ptoa_64(internalState->fCopyPageAlloc) 
+					    + offset - internalState->fPreparedOffset;
+		rtn = kIOReturnSuccess;
+	    }
+	    else
+	    {
+		const IOMemoryDescriptor * memory =
+		    internalState->fCopyMD ? internalState->fCopyMD : fMemory;
+		rtn = memory->dmaCommandOperation(mdOp, fState, sizeof(fState));
+		mdOp = kIOMDWalkSegments;
+	    }
+
+	    if (rtn == kIOReturnSuccess) {
+		assert(state->fIOVMAddr);
+		assert(state->fLength);
+	    }
+	    else if (rtn == kIOReturnOverrun)
+		state->fIOVMAddr = state->fLength = 0;	// At end
+	    else
+		return rtn;
+        };
+
+        if (!curSeg.fIOVMAddr) {
+	    UInt64 length = state->fLength;
+
+            offset          += length;
+            curSeg.fIOVMAddr = state->fIOVMAddr | bypassMask;
+            curSeg.fLength   = length;
+            state->fIOVMAddr = 0;
+        }
+        else if ((curSeg.fIOVMAddr + curSeg.fLength == state->fIOVMAddr)) {
+	    UInt64 length = state->fLength;
+            offset          += length;
+            curSeg.fLength  += length;
+            state->fIOVMAddr = 0;
+        };
+
+
+        if (!state->fIOVMAddr)
+	{
+	    if (kWalkClient & op)
+	    {
+		if ((curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys)
+		{
+		    if (internalState->fCursor)
+		    {
+			curSeg.fIOVMAddr = 0;
+			ret = kIOReturnMessageTooLarge;
+			break;
+		    }
+		    else if (curSeg.fIOVMAddr <= maxPhys)
+		    {
+			UInt64 remain, newLength;
+
+			newLength = (maxPhys + 1 - curSeg.fIOVMAddr);
+			DEBG("trunc %qx, %qx-> %qx\n", curSeg.fIOVMAddr, curSeg.fLength, newLength);
+			remain = curSeg.fLength - newLength;
+			state->fIOVMAddr = newLength + curSeg.fIOVMAddr;
+			curSeg.fLength   = newLength;
+			state->fLength   = remain;
+			offset          -= remain;
+		    }
+		    else if (gIOCopyMapper)
+		    {
+			DEBG("sparse switch %qx, %qx ", curSeg.fIOVMAddr, curSeg.fLength);
+			// Cache this!
+			for (UInt checkRemapIndex = 0; checkRemapIndex < internalState->fCopyPageCount; checkRemapIndex++)
+			{
+			    if (trunc_page_64(curSeg.fIOVMAddr) == gIOCopyMapper->mapAddr(
+							    ptoa_64(internalState->fCopyPageAlloc + checkRemapIndex)))
+			    {
+				curSeg.fIOVMAddr = ptoa_64(internalState->fCopyPageAlloc + checkRemapIndex) + (curSeg.fIOVMAddr & PAGE_MASK);
+				break;
+			    }
+			}
+			DEBG("-> %qx, %qx\n", curSeg.fIOVMAddr, curSeg.fLength);
+		    }
+		}
+	    }
+
+	    if (curSeg.fLength > fMaxSegmentSize)
+	    {
+		UInt64 remain = curSeg.fLength - fMaxSegmentSize;
+
+		state->fIOVMAddr = fMaxSegmentSize + curSeg.fIOVMAddr;
+		curSeg.fLength   = fMaxSegmentSize;
+
+		state->fLength   = remain;
+		offset          -= remain;
+	    }
+
+	    if (internalState->fCursor
+		&& (0 != (fAlignMask & curSeg.fIOVMAddr)))
+	    {
+		curSeg.fIOVMAddr = 0;
+		ret = kIOReturnNotAligned;
+		break;
+	    }
+
+	    if (offset >= memLength)
+	    {
+		curSeg.fLength   -= (offset - memLength);
+		offset = memLength;
+		state->fIOVMAddr = state->fLength = 0;	// At end
+		break;
+	    }
+	}
+
+        if (state->fIOVMAddr) {
+            if ((segIndex + 1 == numSegments))
+                break;
+
+	    ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++);
+            curSeg.fIOVMAddr = 0;
+	    if (kIOReturnSuccess != ret)
+		break;
+        }
+    }
+
+    if (curSeg.fIOVMAddr) {
+	ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++);
+    }
+
+    if (kIOReturnSuccess == ret)
+    {
+	state->fOffset = offset;
+	*offsetP       = offset - internalState->fPreparedOffset;
+	*numSegmentsP  = segIndex;
+    }
+    return ret;
+}
+
+IOReturn 
+IODMACommand::clientOutputSegment(
+	void *reference, IODMACommand *target,
+	Segment64 segment, void *vSegList, UInt32 outSegIndex)
+{
+    IOReturn ret = kIOReturnSuccess;
+
+    if ((target->fNumAddressBits < 64) 
+	&& ((segment.fIOVMAddr + segment.fLength - 1) >> target->fNumAddressBits))
+    {
+	DEBG("kIOReturnMessageTooLarge(fNumAddressBits) %qx, %qx\n", segment.fIOVMAddr, segment.fLength);
+	ret = kIOReturnMessageTooLarge;
+    }
+
+    if (!(*target->fOutSeg)(target, segment, vSegList, outSegIndex))
+    {
+	DEBG("kIOReturnMessageTooLarge(fOutSeg) %qx, %qx\n", segment.fIOVMAddr, segment.fLength);
+	ret = kIOReturnMessageTooLarge;
+    }
+
+    return (ret);
+}
+
+bool 
+IODMACommand::OutputHost32(IODMACommand *,
+	Segment64 segment, void *vSegList, UInt32 outSegIndex)
+{
+    Segment32 *base = (Segment32 *) vSegList;
+    base[outSegIndex].fIOVMAddr = (UInt32) segment.fIOVMAddr;
+    base[outSegIndex].fLength   = (UInt32) segment.fLength;
+    return true;
+}
+
+bool 
+IODMACommand::OutputBig32(IODMACommand *,
+	Segment64 segment, void *vSegList, UInt32 outSegIndex)
+{
+    const UInt offAddr = outSegIndex * sizeof(Segment32);
+    const UInt offLen  = offAddr + sizeof(UInt32);
+    OSWriteBigInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr);
+    OSWriteBigInt32(vSegList, offLen,  (UInt32) segment.fLength);
+    return true;
+}
+
+bool
+IODMACommand::OutputLittle32(IODMACommand *,
+	Segment64 segment, void *vSegList, UInt32 outSegIndex)
+{
+    const UInt offAddr = outSegIndex * sizeof(Segment32);
+    const UInt offLen  = offAddr + sizeof(UInt32);
+    OSWriteLittleInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr);
+    OSWriteLittleInt32(vSegList, offLen,  (UInt32) segment.fLength);
+    return true;
+}
+
+bool
+IODMACommand::OutputHost64(IODMACommand *,
+	Segment64 segment, void *vSegList, UInt32 outSegIndex)
+{
+    Segment64 *base = (Segment64 *) vSegList;
+    base[outSegIndex] = segment;
+    return true;
+}
+
+bool
+IODMACommand::OutputBig64(IODMACommand *,
+	Segment64 segment, void *vSegList, UInt32 outSegIndex)
+{
+    const UInt offAddr = outSegIndex * sizeof(Segment64);
+    const UInt offLen  = offAddr + sizeof(UInt64);
+    OSWriteBigInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr);
+    OSWriteBigInt64(vSegList, offLen,  (UInt64) segment.fLength);
+    return true;
+}
+
+bool
+IODMACommand::OutputLittle64(IODMACommand *,
+	Segment64 segment, void *vSegList, UInt32 outSegIndex)
+{
+    const UInt offAddr = outSegIndex * sizeof(Segment64);
+    const UInt offLen  = offAddr + sizeof(UInt64);
+    OSWriteLittleInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr);
+    OSWriteLittleInt64(vSegList, offLen,  (UInt64) segment.fLength);
+    return true;
+}
+
+
diff --git a/iokit/Kernel/IODeviceTreeSupport.cpp b/iokit/Kernel/IODeviceTreeSupport.cpp
index ce98e39e8..efaf431e5 100644
--- a/iokit/Kernel/IODeviceTreeSupport.cpp
+++ b/iokit/Kernel/IODeviceTreeSupport.cpp
@@ -308,7 +308,11 @@ static void FreePhysicalMemory( vm_offset_t * range )
 {
     vm_offset_t	virt;
 
+#if defined (__i386__)
+    virt = ml_boot_ptovirt( range[0] );
+#else
     virt = ml_static_ptovirt( range[0] );
+#endif
     if( virt) {
         ml_static_mfree( virt, range[1] );
     }
diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp
index bc5c9c965..d38300ad5 100644
--- a/iokit/Kernel/IOHibernateIO.cpp
+++ b/iokit/Kernel/IOHibernateIO.cpp
@@ -198,6 +198,9 @@ uint32_t			gIOHibernateFreeTime  = 0*1000;	// max time to spend freeing pages (m
 
 static IODTNVRAM *		gIOOptionsEntry;
 static IORegistryEntry *	gIOChosenEntry;
+#ifdef __i386__
+static const OSSymbol *         gIOCreateEFIDevicePathSymbol;
+#endif
 
 static IOPolledFileIOVars	          gFileVars;
 static IOHibernateVars			  gIOHibernateVars;
@@ -341,21 +344,27 @@ hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_
 }
 
 static vm_offset_t
-hibernate_page_list_iterate(hibernate_page_list_t * list, 
-				void ** iterator, vm_offset_t * ppnum)
+hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
 {
-    uint32_t count, idx;
-
-    idx = (uint32_t) *iterator;
+    uint32_t		 page = *pPage;
+    uint32_t		 count;
+    hibernate_bitmap_t * bitmap;
 
-    if (!idx)
-	idx = hibernate_page_list_count(list, TRUE, idx);
+    while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
+    {
+	count = hibernate_page_bitmap_count(bitmap, TRUE, page);
+	if (!count)
+	    break;
+	page += count;
+	if (page <= bitmap->last_page)
+	    break;
+    }
 
-    *ppnum = idx;
-    count  = hibernate_page_list_count(list, FALSE, idx);
-    idx   += count;
-    idx   += hibernate_page_list_count(list, TRUE, idx);
-    *iterator  = (void *) idx;
+    *pPage = page;
+    if (bitmap)
+	count = hibernate_page_bitmap_count(bitmap, FALSE, page);
+    else
+	count = 0;
 
     return (count);
 }
@@ -659,21 +668,42 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
 	*fileExtents = extentsData;
     
 	// make imagePath
-	char str1[256];
-	char str2[24];
-	int len = sizeof(str1);
 
-	if ((extentsData->getLength() >= sizeof(IOPolledFileExtent))
-	    && part->getPath(str1, &len, gIODTPlane))
+	if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
 	{
-	    // (strip the plane name)
-	    char * tail = strchr(str1, ':');
-	    if (!tail)
-		tail = str1 - 1;
-	    data = OSData::withBytes(tail + 1, strlen(tail + 1));
-	    sprintf(str2, ",%qx", vars->extentMap[0]);
-	    data->appendBytes(str2, strlen(str2));
+	    char str2[24];
+
+#if __i386__
+	    if (!gIOCreateEFIDevicePathSymbol)
+		gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
+
+	    sprintf(str2, "%qx", vars->extentMap[0]);
+
+	    err = IOService::getPlatform()->callPlatformFunction(
+						gIOCreateEFIDevicePathSymbol, false,
+						(void *) part, (void *) str2, (void *) true,
+						(void *) &data);
+#else
+	    char str1[256];
+	    int len = sizeof(str1);
+
+	    if (!part->getPath(str1, &len, gIODTPlane))
+		err = kIOReturnNotFound;
+	    else
+	    {
+		sprintf(str2, ",%qx", vars->extentMap[0]);
+		// (strip the plane name)
+		char * tail = strchr(str1, ':');
+		if (!tail)
+		    tail = str1 - 1;
+		data = OSData::withBytes(tail + 1, strlen(tail + 1));
+		data->appendBytes(str2, strlen(str2));
+	    }
+#endif
+	if (kIOReturnSuccess == err)
 	    *imagePath = data;
+	else
+	    HIBLOG("error 0x%x getting path\n", err);
 	}
     }
     while (false);
@@ -993,8 +1023,10 @@ IOHibernateSystemSleep(void)
 
     do
     {
-        vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(0, 4 * page_size, page_size);
-        vars->ioBuffer  = IOBufferMemoryDescriptor::withOptions(0, 2 * kDefaultIOSize, page_size);
+        vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
+				    4 * page_size, page_size);
+        vars->ioBuffer  = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, 
+				    2 * kDefaultIOSize, page_size);
 
         if (!vars->srcBuffer || !vars->ioBuffer)
         {
@@ -1060,8 +1092,6 @@ IOHibernateSystemSleep(void)
 	if (gIOOptionsEntry)
 	{
             const OSSymbol *  sym;
-            size_t	      len;
-            char              valueString[16];
 
             sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
             if (sym)
@@ -1071,6 +1101,10 @@ IOHibernateSystemSleep(void)
             }
             data->release();
 
+#ifdef __ppc__
+            size_t	      len;
+            char              valueString[16];
+
 	    vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
             if (gIOChosenEntry)
             {
@@ -1086,7 +1120,6 @@ IOHibernateSystemSleep(void)
 		    if (str2)
 			str2->release();
 		}
-
                 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
                 if (data)
                 {
@@ -1106,7 +1139,57 @@ IOHibernateSystemSleep(void)
                 if (data)
                     gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
             }
-
+#endif /* __ppc__ */
+#ifdef __i386__
+	    struct AppleRTCHibernateVars
+	    {
+		uint8_t     signature[4];
+		uint32_t    revision;
+		uint8_t	    booterSignature[20];
+		uint8_t	    wiredCryptKey[16];
+	    };
+	    AppleRTCHibernateVars rtcVars;
+
+	    rtcVars.signature[0] = 'A';
+	    rtcVars.signature[1] = 'A';
+	    rtcVars.signature[2] = 'P';
+	    rtcVars.signature[3] = 'L';
+	    rtcVars.revision     = 1;
+	    bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
+	    if (gIOHibernateBootSignature[0])
+	    {
+		char c;
+		uint8_t value = 0;
+		for (uint32_t i = 0;
+		    (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
+		    i++)
+		{
+		    if (c >= 'a')
+			c -= 'a' - 10;
+		    else if (c >= 'A')
+			c -= 'A' - 10;
+		    else if (c >= '0')
+			c -= '0';
+		    else
+			continue;
+		    value = (value << 4) | c;
+		    if (i & 1)
+			rtcVars.booterSignature[i >> 1] = value;
+		}
+	    }
+	    data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
+	    if (data)
+	    {
+		IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data);
+		data->release();
+	    }
+            if (gIOChosenEntry)
+            {
+                data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
+                if (data)
+                    gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
+            }
+#else /* !__i386__ */
             if (kIOHibernateModeEncrypt & gIOHibernateMode)
             {
                 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
@@ -1124,7 +1207,7 @@ IOHibernateSystemSleep(void)
                     if (sym && data)
                     {
                         char c;
-                        uint8_t value;
+                        uint8_t value = 0;
                         for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
                         {
                             if (c >= 'a')
@@ -1147,7 +1230,6 @@ IOHibernateSystemSleep(void)
                         data->release();
                 }
             }
-
             if (!vars->haveFastBoot)
             {
                 // set boot volume to zero
@@ -1161,6 +1243,7 @@ IOHibernateSystemSleep(void)
                                             &newVolume, sizeof(newVolume));
                 }
             }
+#endif /* !__i386__ */
 	}
 	// --
 
@@ -1233,6 +1316,7 @@ IOHibernateSystemWake(void)
 
     // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
 
+#ifdef __ppc__
     OSData * data = OSData::withCapacity(4);
     if (gIOOptionsEntry && data)
     {
@@ -1286,6 +1370,11 @@ IOHibernateSystemWake(void)
 	    // just sync the variables in case a later panic syncs nvram (it won't sync variables)
 	    gIOOptionsEntry->syncOFVariables();
     }
+#endif
+
+#ifdef __i386__
+    IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey);
+#endif
 
     if (vars->srcBuffer)
 	vars->srcBuffer->release();
@@ -1306,6 +1395,13 @@ IOHibernateSystemPostWake(void)
 {
     if (gIOHibernateFileRef)
     {
+	// invalidate the image file
+	gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
+	int err = kern_write_file(gIOHibernateFileRef, 0,
+				    (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
+	if (KERN_SUCCESS != err)
+	    HIBLOG("kern_write_file(%d)\n", err);
+
 	kern_close_file_for_direct_io(gIOHibernateFileRef);
         gIOHibernateFileRef = 0;
     }
@@ -1358,6 +1454,8 @@ hibernate_setup_for_wake(void)
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
+#define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
+
 extern "C" boolean_t
 hibernate_write_image(void)
 {
@@ -1365,6 +1463,8 @@ hibernate_write_image(void)
     IOHibernateVars *        vars  = &gIOHibernateVars;
     IOPolledFileExtent *     fileExtents;
 
+    C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
+
     uint32_t	 pageCount, pagesDone;
     IOReturn     err;
     vm_offset_t  ppnum;
@@ -1597,21 +1697,15 @@ hibernate_write_image(void)
 
         // mark more areas for no save, but these are not available 
         // for trashing during restore
+
+	hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
     
-#if !__i386__
         page = atop_32(sectHIBB);
         count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
-#else
-        // XXX
-        page = atop_32(sectHIBB & 0x3FFFFFFF);
-        count = atop_32(round_page((sectHIBB + sectSizeHIB) & 0x3FFFFFFF)) - page;
-#endif
         hibernate_set_page_state(vars->page_list, vars->page_list_wired,
                                         page, count,
                                         kIOHibernatePageStateFree);
         pageCount -= count;
-    
-
 
         if (vars->previewBuffer) for (count = 0;
                                         (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
@@ -1625,15 +1719,15 @@ hibernate_write_image(void)
 
         src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
     
-        void * iter = 0;
-        pagesDone   = 0;
+        ppnum     = 0;
+        pagesDone = 0;
     
         HIBLOG("writing %d pages\n", pageCount);
 
         do
         {
             count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
-                                                    &iter, &ppnum);
+                                                    &ppnum);
 //          kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
     
             iterDone = !count;
@@ -1731,7 +1825,7 @@ hibernate_write_image(void)
 
                 iterDone = false;
                 pageType = 1;
-                iter = 0;
+                ppnum = 0;
                 image1Size = vars->fileVars->position;
                 if (cryptvars)
                 {
@@ -1902,10 +1996,6 @@ hibernate_machine_init(void)
     if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
 	return;
 
-    if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
-        hibernate_page_list_discard(vars->page_list);
-
-
     sum = gIOHibernateCurrentHeader->actualImage1Sum;
     pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
 
@@ -1923,11 +2013,18 @@ hibernate_machine_init(void)
 	    gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1], 
 	    gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]); 
 
-    HIBPRINT("video %lx %ld %ld %ld\n",
+    HIBPRINT("video %x %d %d %d\n",
 	    gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth, 
 	    gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height); 
 
-    if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
+    if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
+        hibernate_page_list_discard(vars->page_list);
+
+    boot_args *args = (boot_args *) PE_state.bootArgs;
+
+    if (vars->videoMapping 
+	&& gIOHibernateGraphicsInfo->physicalAddress
+	&& (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress))
     {
         vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height 
                                         * gIOHibernateGraphicsInfo->rowBytes);
@@ -1936,7 +2033,19 @@ hibernate_machine_init(void)
                     vars->videoMapSize, kIOMapInhibitCache );
     }
 
-    uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();;
+    uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+
+    if (gIOHibernateWakeMapSize)
+    {
+	err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(gIOHibernateWakeMap), 
+						    gIOHibernateWakeMapSize);
+	if (kIOReturnSuccess == err)
+	    hibernate_newruntime_map(src, gIOHibernateWakeMapSize, 
+				     gIOHibernateCurrentHeader->systemTableOffset);
+	gIOHibernateWakeMap = 0;
+	gIOHibernateWakeMapSize = 0;
+    }
+
     uint32_t decoOffset;
 
     clock_get_uptime(&allTime);
diff --git a/iokit/Kernel/IOHibernateInternal.h b/iokit/Kernel/IOHibernateInternal.h
index f4dabcf44..fdaf45d06 100644
--- a/iokit/Kernel/IOHibernateInternal.h
+++ b/iokit/Kernel/IOHibernateInternal.h
@@ -97,3 +97,6 @@ extern int         sectSizeHIB;
 extern vm_offset_t sectDATAB;
 extern int         sectSizeDATA;
 
+extern vm_offset_t gIOHibernateWakeMap;	    // ppnum
+extern vm_size_t   gIOHibernateWakeMapSize;
+
diff --git a/iokit/Kernel/IOHibernateRestoreKernel.c b/iokit/Kernel/IOHibernateRestoreKernel.c
index 16d60cb05..b1b99fbac 100644
--- a/iokit/Kernel/IOHibernateRestoreKernel.c
+++ b/iokit/Kernel/IOHibernateRestoreKernel.c
@@ -30,6 +30,7 @@
 #include <mach/mach_types.h>
 #include <mach/vm_param.h>
 #include <IOKit/IOHibernatePrivate.h>
+#include <IOKit/IOLib.h>
 #include <pexpert/boot.h>
 #include <crypto/aes.h>
 
@@ -54,11 +55,13 @@ hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
 static hibernate_cryptwakevars_t _cryptWakeVars;
 hibernate_cryptwakevars_t * gIOHibernateCryptWakeVars = &_cryptWakeVars;
 
+vm_offset_t gIOHibernateWakeMap;    	    // ppnum
+vm_size_t   gIOHibernateWakeMapSize;
+
 #if __i386__
 extern void   acpi_wake_prot_entry(void);
 #endif
 
-
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #define BASE 65521L /* largest prime smaller than 65536 */
@@ -98,29 +101,8 @@ hibernate_sum(uint8_t *buf, int32_t len)
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-#if __ppc__
-static __inline__ unsigned int cntlzw(unsigned int num)
-{
-  unsigned int result;
-  __asm__ volatile("cntlzw %0, %1" : "=r" (result) : "r" (num));
-  return result;
-}
-#elif __i386__
-static __inline__ unsigned int cntlzw(unsigned int num)
-{
-    unsigned int result;
-    __asm__ volatile(	"bsrl   %1, %0\n\t"
-                     	"cmovel %2, %0"
-                     : "=r" (result)
-                     : "rm" (num), "r" (63));
-    return 31 ^ result;
-}
-#else
-#error arch
-#endif
-
-void 
-hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
+static hibernate_bitmap_t *
+hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
 {
     uint32_t             bank;
     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
@@ -128,117 +110,129 @@ hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page
     for (bank = 0; bank < list->bank_count; bank++)
     {
 	if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
-	{
-	    page -= bitmap->first_page;
-	    if (set)
-		bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
-		//setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
-	    else
-		bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
-		//clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
 	    break;
-	}
 	bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
     }
+    if (bank == list->bank_count)
+	bitmap = 0;
+	
+    return (bitmap);
 }
 
-boolean_t 
-hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
+hibernate_bitmap_t *
+hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)
 {
-    boolean_t		 result = TRUE;
-    uint32_t             bank;
+    uint32_t             bank, page = *pPage;
     hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
 
     for (bank = 0; bank < list->bank_count; bank++)
     {
-	if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
+	if (page <= bitmap->first_page)
 	{
-	    page -= bitmap->first_page;
-            result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
+	    *pPage = bitmap->first_page;
 	    break;
 	}
+	if (page <= bitmap->last_page)
+	    break;
 	bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
     }
+    if (bank == list->bank_count)
+	bitmap = 0;
+	
+    return (bitmap);
+}
+
+void 
+hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
+{
+    hibernate_bitmap_t * bitmap;
+
+    bitmap = hibernate_page_bitmap(list, page);
+    if (bitmap)
+    {
+	page -= bitmap->first_page;
+	if (set)
+	    bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
+	    //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
+	else
+	    bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
+	    //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
+    }
+}
+
+boolean_t 
+hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
+{
+    boolean_t		 result = TRUE;
+    hibernate_bitmap_t * bitmap;
+
+    bitmap = hibernate_page_bitmap(list, page);
+    if (bitmap)
+    {
+	page -= bitmap->first_page;
+	result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
+    }
     return (result);
 }
 
-// count bits clear or set (set == TRUE) starting at index page.
+// count bits clear or set (set == TRUE) starting at page.
 uint32_t
-hibernate_page_list_count(hibernate_page_list_t * list, uint32_t set, uint32_t page)
+hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page)
 {
-    uint32_t			bank, count;
-    hibernate_bitmap_t *	bitmap;
+    uint32_t index, bit, bits;
+    uint32_t count;
 
-    bitmap = &list->bank_bitmap[0];
-    count  = 0;
+    count = 0;
 
-    for (bank = 0; bank < list->bank_count; bank++)
+    index = (page - bitmap->first_page) >> 5;
+    bit = (page - bitmap->first_page) & 31;
+
+    bits = bitmap->bitmap[index];
+    if (set)
+	bits = ~bits;
+    bits = (bits << bit);
+    if (bits)
+	count += __builtin_clz(bits);
+    else
     {
-	// bits between banks are "set"
-	if (set && (page < bitmap->first_page))
+	count += 32 - bit;
+	while (++index < bitmap->bitmapwords)
 	{
-	    count += bitmap->first_page - page;
-	    page  = bitmap->first_page;
-	}
-	if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
-	{
-	    uint32_t index, bit, bits;
-	
-	    index = (page - bitmap->first_page) >> 5;
-	    bit = (page - bitmap->first_page) & 31;
-	
-	    while (TRUE)
+	    bits = bitmap->bitmap[index];
+	    if (set)
+		bits = ~bits;
+	    if (bits)
 	    {
-		bits = bitmap->bitmap[index];
-		if (set)
-		    bits = ~bits;
-		bits = (bits << bit);
-		count += cntlzw(bits);
-		if (bits)
-		    break;
-		count -= bit;
-	    
-		while (++index < bitmap->bitmapwords)
-		{
-		    bits = bitmap->bitmap[index];
-		    if (set)
-			bits = ~bits;
-		    count += cntlzw(bits);
-		    if (bits)
-			break;
-		}
-		if (bits)
-		    break;
-		if (!set)
-		    break;
-		// bits between banks are "set"
-		bank++;
-		if (bank >= list->bank_count)
-		    break;
-		count -= (bitmap->last_page + 1);
-		bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
-		count += bitmap->first_page;
-		index = 0;
-		bit = 0;			    
+		count += __builtin_clz(bits);
+		break;
 	    }
-	    break;
+	    count += 32;
 	}
-	bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
     }
 
     return (count);
 }
 
-
-static uint32_t
-hibernate_page_list_grab(hibernate_page_list_t * map, uint32_t * _nextFree)
+static vm_offset_t
+hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)
 {
-    uint32_t nextFree = *_nextFree;
+    uint32_t		 nextFree = *pNextFree;
+    uint32_t		 nextFreeInBank;
+    hibernate_bitmap_t * bitmap;
 
-    if (!nextFree)
-	nextFree = hibernate_page_list_count(map, 0, 0);
+    nextFreeInBank = nextFree + 1;
+    while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank)))
+    {
+	nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank);
+	if (nextFreeInBank <= bitmap->last_page)
+	{
+	    *pNextFree = nextFreeInBank;
+	    break;
+	}
+    }
 
-    *_nextFree = nextFree + 1 + hibernate_page_list_count(map, 0, nextFree + 1);
+    if (!bitmap)
+	IOPanic(__FUNCTION__);
 
     return (nextFree);
 }
@@ -251,14 +245,8 @@ store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
     uint32_t sum;
 
     dst = ptoa_64(ppnum);
-#if __ppc__
     if (ppnum < 0x00100000)
 	buffer = (uint32_t *) (uint32_t) dst;
-#elif __i386__
-    if (ppnum < atop_32(0xC0000000)) {
-        buffer = (uint32_t *) (uint32_t) dst;
-    }
-#endif
 
     if (compressedSize != PAGE_SIZE)
     {
@@ -290,9 +278,11 @@ bcopy_internal(const void *src, void *dst, uint32_t len)
     }
 }
 
+#define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
+
 long 
 hibernate_kernel_entrypoint(IOHibernateImageHeader * header, 
-                            void * p2, void * p3, __unused void * p4)
+                            void * p2, void * p3, void * p4)
 {
     typedef void (*ResetProc)(void);
     uint32_t idx;
@@ -315,11 +305,18 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
     uint32_t lastMapPage;
     uint32_t lastPageIndexPage;
 
+    C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
 
     bcopy_internal(header, 
                 gIOHibernateCurrentHeader, 
                 sizeof(IOHibernateImageHeader));
 
+    if (!p2)
+    {
+	count = header->graphicsInfoOffset;
+	if (count)
+	    p2 = (void *)(((uintptr_t) header) - count);
+    }
     if (p2) 
         bcopy_internal(p2, 
                 gIOHibernateGraphicsInfo, 
@@ -327,6 +324,12 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
     else
         gIOHibernateGraphicsInfo->physicalAddress = gIOHibernateGraphicsInfo->depth = 0;
 
+    if (!p3)
+    {
+	count = header->cryptVarsOffset;
+	if (count)
+	    p3 = (void *)(((uintptr_t) header) - count);
+    }
     if (p3)
         bcopy_internal(p3, 
                 gIOHibernateCryptWakeVars, 
@@ -363,8 +366,19 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
     }
 
     nextFree = 0;
+    hibernate_page_list_grab(map, &nextFree);
     buffer = (uint32_t *) ptoa_32(hibernate_page_list_grab(map, &nextFree));
 
+    if (header->memoryMapSize && (count = header->memoryMapOffset))
+    {
+	p4 = (void *)(((uintptr_t) header) - count);
+	gIOHibernateWakeMap     = hibernate_page_list_grab(map, &nextFree);
+	gIOHibernateWakeMapSize = header->memoryMapSize;
+	bcopy_internal(p4, (void  *) ptoa_32(gIOHibernateWakeMap), gIOHibernateWakeMapSize);
+    }
+    else
+	gIOHibernateWakeMapSize = 0;
+
     sum = gIOHibernateCurrentHeader->actualRestore1Sum;
     gIOHibernateCurrentHeader->diag[0] = (uint32_t) header;
     gIOHibernateCurrentHeader->diag[1] = sum;
@@ -413,6 +427,8 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
                 compressedSize = kIOHibernateTagLength & tag;
             }
 
+//    SINT(ppnum);
+
 	    conflicts = (((ppnum >= atop_32(map)) && (ppnum <= lastMapPage))
 		      || ((ppnum >= atop_32(src)) && (ppnum <= lastImagePage)));
 
@@ -498,7 +514,8 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
 #elif __i386__
     ResetProc proc;
     proc = (ResetProc) acpi_wake_prot_entry;
-
+    // flush caches
+    __asm__("wbinvd");
     proc();
 #endif
   
diff --git a/iokit/Kernel/IOInterruptController.cpp b/iokit/Kernel/IOInterruptController.cpp
index 81ef62db2..f65dbe3b6 100644
--- a/iokit/Kernel/IOInterruptController.cpp
+++ b/iokit/Kernel/IOInterruptController.cpp
@@ -433,6 +433,8 @@ IOReturn IOSharedInterruptController::initInterruptController(IOInterruptControl
   _numInterruptSources = 1;
   
   // Set up the IOInterruptSource to point at this.
+  parentController->retain();
+  parentSource->retain();
   _interruptSources[0].interruptController = parentController;
   _interruptSources[0].vectorData = parentSource;
   
@@ -672,7 +674,8 @@ IOReturn IOSharedInterruptController::disableInterrupt(IOService *nub,
 
 IOInterruptAction IOSharedInterruptController::getInterruptHandlerAddress(void)
 {
-    return (IOInterruptAction)&IOSharedInterruptController::handleInterrupt;
+    return OSMemberFunctionCast(IOInterruptAction,
+			this, &IOSharedInterruptController::handleInterrupt);
 }
 
 IOReturn IOSharedInterruptController::handleInterrupt(void * /*refCon*/,
diff --git a/iokit/Kernel/IOKitKernelInternal.h b/iokit/Kernel/IOKitKernelInternal.h
index a5646f3d5..cdadc3ace 100644
--- a/iokit/Kernel/IOKitKernelInternal.h
+++ b/iokit/Kernel/IOKitKernelInternal.h
@@ -46,17 +46,52 @@ kern_return_t IOIteratePageableMaps(vm_size_t size,
 vm_map_t IOPageableMapForAddress( vm_address_t address );
 SInt32 OSKernelStackRemaining( void );
 
+mach_vm_address_t
+IOKernelAllocateContiguous(mach_vm_size_t size, mach_vm_size_t alignment);
+void
+IOKernelFreeContiguous(mach_vm_address_t address, mach_vm_size_t size);
+
 extern vm_size_t debug_iomallocpageable_size;
 
 // osfmk/device/iokit_rpc.c
 // LP64todo - these need to expand 
-extern kern_return_t IOMapPages( vm_map_t map, vm_offset_t va, vm_offset_t pa,
-                                 vm_size_t length, unsigned int mapFlags);
-extern kern_return_t IOUnmapPages(vm_map_t map, vm_offset_t va, vm_size_t length);
+extern kern_return_t IOMapPages(vm_map_t map, mach_vm_address_t va, mach_vm_address_t pa,
+			mach_vm_size_t length, unsigned int options);
+extern kern_return_t IOUnmapPages(vm_map_t map, mach_vm_address_t va, mach_vm_size_t length);
+
+extern kern_return_t IOProtectCacheMode(vm_map_t map, mach_vm_address_t va,
+					mach_vm_size_t length, unsigned int mapFlags);
+
+extern ppnum_t IOGetLastPageNumber(void);
+
+extern ppnum_t gIOLastPage;
 
 /* Physical to physical copy (ints must be disabled) */
 extern void bcopy_phys(addr64_t from, addr64_t to, int size);
 
 __END_DECLS
 
+// Used for dedicated communications for IODMACommand
+enum  {
+    kIOMDWalkSegments         = 0x00000001,
+    kIOMDFirstSegment	      = 0x00000002 | kIOMDWalkSegments,
+    kIOMDGetCharacteristics   = 0x00000004,
+    kIOMDLastDMACommandOperation
+};
+struct IOMDDMACharacteristics {
+    UInt64 fLength;
+    UInt32 fSGCount;
+    UInt32 fPages;
+    UInt32 fPageAlign;
+    ppnum_t fHighestPage;
+    IODirection fDirection;
+    UInt8 fIsMapped, fIsPrepared;
+};
+struct IOMDDMAWalkSegmentArgs {
+    UInt64 fOffset;			// Input/Output offset
+    UInt64 fIOVMAddr, fLength;		// Output variables
+    UInt8 fMapped;			// Input Variable, Require mapped IOVMA
+};
+typedef UInt8 IOMDDMAWalkSegmentState[128];
+
 #endif /* ! _IOKIT_KERNELINTERNAL_H */
diff --git a/iokit/Kernel/IOLib.c b/iokit/Kernel/IOLib.cpp
similarity index 81%
rename from iokit/Kernel/IOLib.c
rename to iokit/Kernel/IOLib.cpp
index 19f05bd80..5ed1c4708 100644
--- a/iokit/Kernel/IOLib.c
+++ b/iokit/Kernel/IOLib.cpp
@@ -44,14 +44,27 @@
 #include <IOKit/IOLib.h> 
 #include <IOKit/IOLocks.h> 
 #include <IOKit/IOMapper.h>
+#include <IOKit/IOBufferMemoryDescriptor.h>
 #include <IOKit/IOKitDebug.h> 
 
 #include "IOKitKernelInternal.h"
 
+extern "C"
+{
+
+
 mach_timespec_t IOZeroTvalspec = { 0, 0 };
 
 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
 
+extern kern_return_t	kmem_suballoc(
+				vm_map_t	parent,
+				vm_offset_t	*addr,
+				vm_size_t	size,
+				boolean_t	pageable,
+				boolean_t	anywhere,
+				vm_map_t	*new_map);
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 lck_grp_t	*IOLockGroup;
@@ -144,9 +157,9 @@ IOThread IOCreateThread(IOThreadFunc fcn, void *arg)
 }
 
 
-volatile void IOExitThread(void)
+void IOExitThread(void)
 {
-	(void) thread_terminate(current_thread());
+    (void) thread_terminate(current_thread());
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -270,130 +283,168 @@ void IOFreeAligned(void * address, vm_size_t size)
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-struct _IOMallocContiguousEntry
+void
+IOKernelFreeContiguous(mach_vm_address_t address, mach_vm_size_t size)
 {
-    void *		virtual;
-    ppnum_t		ioBase;
-    queue_chain_t	link;
-};
-typedef struct _IOMallocContiguousEntry _IOMallocContiguousEntry;
+    mach_vm_address_t allocationAddress;
+    mach_vm_size_t    adjustedSize;
 
-void * IOMallocContiguous(vm_size_t size, vm_size_t alignment,
-			   IOPhysicalAddress * physicalAddress)
+    if (!address)
+	return;
+
+    assert(size);
+
+    adjustedSize = (2 * size) + sizeof(mach_vm_size_t) + sizeof(mach_vm_address_t);
+    if (adjustedSize >= page_size) {
+
+	kmem_free( kernel_map, (vm_address_t) address, size);
+
+    } else {
+
+	adjustedSize = *((mach_vm_size_t *)
+			(address - sizeof(mach_vm_address_t) - sizeof(mach_vm_size_t)));
+	allocationAddress = *((mach_vm_address_t *)
+			(address - sizeof(mach_vm_address_t) ));
+	kfree((void *)allocationAddress, adjustedSize);
+    }
+
+#if IOALLOCDEBUG
+    debug_iomalloc_size -= size;
+#endif
+}
+
+mach_vm_address_t
+IOKernelAllocateContiguous(mach_vm_size_t size, mach_vm_size_t alignment)
 {
     kern_return_t	kr;
-    vm_address_t	address;
-    vm_address_t	allocationAddress;
-    vm_size_t		adjustedSize;
-    vm_offset_t		alignMask;
-    ppnum_t		pagenum;
+    mach_vm_address_t	address;
+    mach_vm_address_t	allocationAddress;
+    mach_vm_size_t	adjustedSize;
+    mach_vm_address_t	alignMask;
 
     if (size == 0)
-        return 0;
+	return (0);
     if (alignment == 0) 
         alignment = 1;
 
     alignMask = alignment - 1;
-    adjustedSize = (2 * size) + sizeof(vm_size_t) + sizeof(vm_address_t);
+    adjustedSize = (2 * size) + sizeof(mach_vm_size_t) + sizeof(mach_vm_address_t);
 
     if (adjustedSize >= page_size)
     {
+	vm_offset_t virt;
 	adjustedSize = size;
 	if (adjustedSize > page_size)
 	{
-	    kr = kmem_alloc_contig(kernel_map, &address, size,
+	    kr = kmem_alloc_contig(kernel_map, &virt, size,
 				    alignMask, 0);
 	}
 	else
 	{
-	    kr = kernel_memory_allocate(kernel_map, &address,
+	    kr = kernel_memory_allocate(kernel_map, &virt,
 					size, alignMask, 0);
 	}
-	if (KERN_SUCCESS != kr)
+	if (KERN_SUCCESS == kr)
+	    address = virt;
+	else
 	    address = 0;
     }
     else
     {
 	adjustedSize += alignMask;
-        allocationAddress = (vm_address_t) kalloc(adjustedSize);
+        allocationAddress = (mach_vm_address_t) kalloc(adjustedSize);
 
         if (allocationAddress) {
 
             address = (allocationAddress + alignMask
-                    + (sizeof(vm_size_t) + sizeof(vm_address_t)))
+                    + (sizeof(mach_vm_size_t) + sizeof(mach_vm_address_t)))
                     & (~alignMask);
 
             if (atop_32(address) != atop_32(address + size - 1))
                 address = round_page_32(address);
 
-            *((vm_size_t *)(address - sizeof(vm_size_t)
-                            - sizeof(vm_address_t))) = adjustedSize;
-            *((vm_address_t *)(address - sizeof(vm_address_t)))
+            *((mach_vm_size_t *)(address - sizeof(mach_vm_size_t)
+                            - sizeof(mach_vm_address_t))) = adjustedSize;
+            *((mach_vm_address_t *)(address - sizeof(mach_vm_address_t)))
                             = allocationAddress;
 	} else
 	    address = 0;
     }
 
+#if IOALLOCDEBUG
+    if (address)
+	debug_iomalloc_size += size;
+#endif
+
+    return (address);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+struct _IOMallocContiguousEntry
+{
+    mach_vm_address_t	       virtualAddr;
+    IOBufferMemoryDescriptor * md;
+    queue_chain_t	       link;
+};
+typedef struct _IOMallocContiguousEntry _IOMallocContiguousEntry;
+
+void * IOMallocContiguous(vm_size_t size, vm_size_t alignment,
+			   IOPhysicalAddress * physicalAddress)
+{
+    mach_vm_address_t	address = 0;
+
+    if (size == 0)
+	return 0;
+    if (alignment == 0) 
+	alignment = 1;
+
     /* Do we want a physical address? */
-    if (address && physicalAddress)
+    if (!physicalAddress)
     {
-	do
+	address = IOKernelAllocateContiguous(size, alignment);
+    }
+    else do
+    {
+	IOBufferMemoryDescriptor * bmd;
+	mach_vm_address_t          physicalMask;
+        vm_offset_t		   alignMask;
+
+	alignMask = alignment - 1;
+	physicalMask = 0xFFFFFFFF ^ (alignMask & PAGE_MASK);
+	bmd = IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
+		kernel_task, kIOMemoryPhysicallyContiguous, size, physicalMask);
+	if (!bmd)
+	    break;
+	
+	_IOMallocContiguousEntry *
+	entry = IONew(_IOMallocContiguousEntry, 1);
+	if (!entry)
 	{
-	    /* Get the physical page */
-	    pagenum = pmap_find_phys(kernel_pmap, (addr64_t) address);
-	    if(pagenum)
-	    {
-		IOByteCount offset;
-		ppnum_t base;
-    
-		base = IOMapperIOVMAlloc((size + PAGE_MASK) >> PAGE_SHIFT);
-		if (base)
-		{
-		    _IOMallocContiguousEntry *
-		    entry = IONew(_IOMallocContiguousEntry, 1);
-		    if (!entry)
-		    {
-			IOFreeContiguous((void *) address, size);
-			address = 0;
-			break;
-		    }
-		    entry->virtual = (void *) address;
-		    entry->ioBase  = base;
-		    lck_mtx_lock(gIOMallocContiguousEntriesLock);
-		    queue_enter( &gIOMallocContiguousEntries, entry, 
-				_IOMallocContiguousEntry *, link );
-		    lck_mtx_unlock(gIOMallocContiguousEntriesLock);
-    
-		    *physicalAddress = (IOPhysicalAddress)((base << PAGE_SHIFT) | (address & PAGE_MASK));
-		    for (offset = 0; offset < ((size + PAGE_MASK) >> PAGE_SHIFT); offset++, pagenum++)
-			IOMapperInsertPage( base, offset, pagenum );
-		}
-		else
-		    *physicalAddress = (IOPhysicalAddress)((pagenum << PAGE_SHIFT) | (address & PAGE_MASK));
-	    } 
-	    else
-		/* Did not find, return 0 */
-		*physicalAddress = (IOPhysicalAddress) 0;
+	    bmd->release();
+	    break;
 	}
-	while (false);
+	entry->virtualAddr = (mach_vm_address_t) bmd->getBytesNoCopy();
+	entry->md          = bmd;
+	lck_mtx_lock(gIOMallocContiguousEntriesLock);
+	queue_enter( &gIOMallocContiguousEntries, entry, 
+		    _IOMallocContiguousEntry *, link );
+	lck_mtx_unlock(gIOMallocContiguousEntriesLock);
+
+	address          = (mach_vm_address_t) entry->virtualAddr;
+	*physicalAddress = bmd->getPhysicalAddress();
     }
-
-    assert(0 == (address & alignMask));
-
-#if IOALLOCDEBUG
-    if( address)
-	debug_iomalloc_size += size;
-#endif
+    while (false);
 
     return (void *) address;
 }
 
-void IOFreeContiguous(void * address, vm_size_t size)
+void IOFreeContiguous(void * _address, vm_size_t size)
 {
-    vm_address_t	       allocationAddress;
-    vm_size_t		       adjustedSize;
     _IOMallocContiguousEntry * entry;
-    ppnum_t		       base = 0;
+    IOMemoryDescriptor *       md = NULL;
+
+    mach_vm_address_t address = (mach_vm_address_t) _address;
 
     if( !address)
 	return;
@@ -404,8 +455,8 @@ void IOFreeContiguous(void * address, vm_size_t size)
     queue_iterate( &gIOMallocContiguousEntries, entry,
 		    _IOMallocContiguousEntry *, link )
     {
-	if( entry->virtual == address ) {
-	    base = entry->ioBase;
+	if( entry->virtualAddr == address ) {
+	    md   = entry->md;
 	    queue_remove( &gIOMallocContiguousEntries, entry,
 			    _IOMallocContiguousEntry *, link );
 	    break;
@@ -413,29 +464,15 @@ void IOFreeContiguous(void * address, vm_size_t size)
     }
     lck_mtx_unlock(gIOMallocContiguousEntriesLock);
 
-    if (base)
+    if (md)
     {
-	IOMapperIOVMFree(base, (size + PAGE_MASK) >> PAGE_SHIFT);
+	md->release();
 	IODelete(entry, _IOMallocContiguousEntry, 1);
     }
-
-    adjustedSize = (2 * size) + sizeof(vm_size_t) + sizeof(vm_address_t);
-    if (adjustedSize >= page_size) {
-
-        kmem_free( kernel_map, (vm_address_t) address, size);
-
-    } else {
-        adjustedSize = *((vm_size_t *)( (vm_address_t) address
-                                - sizeof(vm_address_t) - sizeof(vm_size_t)));
-        allocationAddress = *((vm_address_t *)( (vm_address_t) address
-				- sizeof(vm_address_t) ));
-
-        kfree((void *)allocationAddress, adjustedSize);
+    else
+    {
+	IOKernelFreeContiguous((mach_vm_address_t) address, size);
     }
-
-#if IOALLOCDEBUG
-    debug_iomalloc_size -= size;
-#endif
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -601,7 +638,7 @@ IOReturn IOSetProcessorCacheMode( task_t task, IOVirtualAddress address,
 	pagenum = pmap_find_phys(kernel_pmap, (addr64_t)address);
 	if( pagenum) {
             ret = IOUnmapPages( get_task_map(task), address, page_size );
-	    ret = IOMapPages( get_task_map(task), address, pagenum << PAGE_SHIFT, page_size, cacheMode );
+	    ret = IOMapPages( get_task_map(task), address, ptoa_64(pagenum), page_size, cacheMode );
 	} else
 	    ret = kIOReturnVMError;
 
@@ -724,3 +761,5 @@ unsigned int IOAlignmentToSize(IOAlignment align)
     }
     return size;
 }
+
+} /* extern "C" */
diff --git a/iokit/Kernel/IOMapper.cpp b/iokit/Kernel/IOMapper.cpp
index 5efde78f9..56d412bf3 100644
--- a/iokit/Kernel/IOMapper.cpp
+++ b/iokit/Kernel/IOMapper.cpp
@@ -29,10 +29,16 @@
 #include <IOKit/IOMapper.h>
 #include <libkern/c++/OSData.h>
 
+#include "IOCopyMapper.h"
+
+__BEGIN_DECLS
+extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
+__END_DECLS
+
 #define super IOService
 OSDefineMetaClassAndAbstractStructors(IOMapper, IOService);
 
-OSMetaClassDefineReservedUnused(IOMapper, 0);
+OSMetaClassDefineReservedUsed(IOMapper, 0);
 OSMetaClassDefineReservedUnused(IOMapper, 1);
 OSMetaClassDefineReservedUnused(IOMapper, 2);
 OSMetaClassDefineReservedUnused(IOMapper, 3);
@@ -136,98 +142,44 @@ void IOMapper::iovmInsert(ppnum_t addr, IOItemCount offset,
         iovmInsert(addr, offset + i, pageList[i].phys_addr);
 }
 
-struct ARTTableData {
-    void *v;
-    upl_t u[0];
-};
-#define getARTDataP(data) ((ARTTableData *) (data)->getBytesNoCopy())
-
-OSData *
-IOMapper::NewARTTable(IOByteCount size,
-                      void ** virtAddrP, ppnum_t *physAddrP)
+OSData * IOMapper::
+NewARTTable(IOByteCount size, void ** virtAddrP, ppnum_t *physAddrP)
 {
-    OSData *ret;
+    if (!virtAddrP || !physAddrP)
+	return 0;
+
     kern_return_t kr;
-    vm_address_t startUpl;
-    ARTTableData *dataP;
-    unsigned int dataSize;
-    upl_page_info_t *pl = 0;
+    vm_address_t address;
 
-    // Each UPL can deal with about one meg at the moment
     size = round_page_32(size);
-    dataSize = sizeof(ARTTableData) + sizeof(upl_t) * size / (1024 * 1024);
-    ret = OSData::withCapacity(dataSize);
-    if (!ret)
+    kr = kmem_alloc_contig(kernel_map, &address, size, PAGE_MASK, 0);
+    if (kr)
         return 0;
 
-    // Append 0's to the buffer, in-other-words reset to nulls.
-    ret->appendBytes(NULL, sizeof(ARTTableData));
-    dataP = getARTDataP(ret);
+    ppnum_t pagenum = pmap_find_phys(kernel_pmap, (addr64_t) address);
+    if (pagenum)
+	*physAddrP = pagenum;
+    else {
+	FreeARTTable((OSData *) address, size);
+	address = 0;
+    }
 
-    kr = kmem_alloc_contig(kernel_map, &startUpl, size, PAGE_MASK, 0);
-    if (kr)
-        return 0;
+    *virtAddrP = (void *) address;
 
-    dataP->v = (void *) startUpl;
-
-    do {
-        upl_t iopl;
-        int upl_flags = UPL_SET_INTERNAL | UPL_SET_LITE
-                      | UPL_SET_IO_WIRE | UPL_COPYOUT_FROM;
-        vm_size_t iopl_size = size;
-
-        kr = vm_map_get_upl(kernel_map,
-                            (vm_map_offset_t)startUpl,
-                            &iopl_size,
-                            &iopl,
-                            0,
-                            0,
-                            &upl_flags,
-                            0);
-        if (kr) {
-            panic("IOMapper:vm_map_get_upl returned 0x%x\n");
-            goto bail;
-        }
-
-        if (!ret->appendBytes(&iopl, sizeof(upl_t)))
-            goto bail;
-            
-        startUpl += iopl_size;
-        size -= iopl_size;
-    } while(size);
-
-    // Need to re-establish the dataP as the OSData may have grown.
-    dataP = getARTDataP(ret);
-
-    // Now grab the page entry of the first page and get its phys addr
-    pl = UPL_GET_INTERNAL_PAGE_LIST(dataP->u[0]);
-    *physAddrP = pl->phys_addr;
-    *virtAddrP = dataP->v;
-
-    return ret;
-
-bail:
-    FreeARTTable(ret, size);
-    return 0;
+    return (OSData *) address;
 }
 
 void IOMapper::FreeARTTable(OSData *artHandle, IOByteCount size)
 {
-    assert(artHandle);
-
-    ARTTableData *dataP = getARTDataP(artHandle);
+    vm_address_t address = (vm_address_t) artHandle;
 
-    int numupls = ((artHandle->getLength() - sizeof(*dataP)) / sizeof(upl_t));
-    for (int i = 0; i < numupls; i++) {
-        upl_abort(dataP->u[i], 0);
-        upl_deallocate(dataP->u[i]);
-    }
+    size = round_page_32(size);
+    kmem_free(kernel_map, address, size);	// Just panic if address is 0
+}
 
-    if (dataP->v) {
-        size = round_page_32(size);
-        kmem_free(kernel_map, (vm_address_t) dataP->v, size);
-    }
-    artHandle->release();
+bool IOMapper::getBypassMask(addr64_t *maskP) const
+{
+    return false;
 }
 
 __BEGIN_DECLS
@@ -391,4 +343,24 @@ void IOMappedWrite64(IOPhysicalAddress address, UInt64 value)
         ml_phys_write_double((vm_offset_t) address, value);
 }
 
+mach_vm_address_t IOMallocPhysical(mach_vm_size_t size, mach_vm_address_t mask)
+{
+    mach_vm_address_t address = 0;
+    if (gIOCopyMapper)
+    {
+	address = ptoa_64(gIOCopyMapper->iovmAlloc(atop_64(round_page(size))));
+    }
+
+    return (address);
+}
+
+void IOFreePhysical(mach_vm_address_t address, mach_vm_size_t size)
+{
+    if (gIOCopyMapper)
+    {
+	gIOCopyMapper->iovmFree(atop_64(address), atop_64(round_page(size)));
+    }
+}
+
+
 __END_DECLS
diff --git a/iokit/Kernel/IOMemoryDescriptor.cpp b/iokit/Kernel/IOMemoryDescriptor.cpp
index f8927e9c8..4c6d57e08 100644
--- a/iokit/Kernel/IOMemoryDescriptor.cpp
+++ b/iokit/Kernel/IOMemoryDescriptor.cpp
@@ -44,6 +44,7 @@
 #include <IOKit/IOKitDebug.h>
 
 #include "IOKitKernelInternal.h"
+#include "IOCopyMapper.h"
 
 #include <libkern/c++/OSContainers.h>
 #include <libkern/c++/OSDictionary.h>
@@ -60,11 +61,8 @@ __BEGIN_DECLS
 #include <mach/memory_object_types.h>
 #include <device/device_port.h>
 
-#ifndef i386
 #include <mach/vm_prot.h>
 #include <vm/vm_fault.h>
-struct phys_entry      *pmap_find_physentry(ppnum_t pa);
-#endif
 
 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
 void ipc_port_release_send(ipc_port_t port);
@@ -103,9 +101,14 @@ __END_DECLS
 
 #define kIOMaximumMappedIOByteCount	(512*1024*1024)
 
-static IOMapper * gIOSystemMapper;
+static IOMapper * gIOSystemMapper = NULL;
+
+IOCopyMapper *	  gIOCopyMapper = NULL;
+
 static ppnum_t	  gIOMaximumMappedIOPageCount = atop_32(kIOMaximumMappedIOByteCount);
 
+ppnum_t		  gIOLastPage;
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 OSDefineMetaClassAndAbstractStructors( IOMemoryDescriptor, OSObject )
@@ -124,6 +127,12 @@ static IORecursiveLock * gIOMemoryLock;
 #define WAKEUP	\
     IORecursiveLockWakeup( gIOMemoryLock, (void *)this, /* one-thread */ false)
 
+#if 0
+#define DEBG(fmt, args...)  	{ kprintf(fmt, ## args); }
+#else
+#define DEBG(fmt, args...)  	{}
+#endif
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 class _IOMemoryMap : public IOMemoryMap
@@ -294,15 +303,21 @@ kern_return_t device_close(
 // This means that pointers are not passed and NULLs don't have to be
 // checked for as a NULL reference is illegal.
 static inline void
-getAddrLenForInd(user_addr_t &addr, IOPhysicalLength &len, // Output variables
+getAddrLenForInd(addr64_t &addr, IOPhysicalLength &len, // Output variables
      UInt32 type, IOGeneralMemoryDescriptor::Ranges r, UInt32 ind)
 {
-    assert(kIOMemoryTypePhysical == type || kIOMemoryTypeUIO == type
-	|| kIOMemoryTypeVirtual  == type);
+    assert(kIOMemoryTypeUIO       == type
+	|| kIOMemoryTypeVirtual   == type || kIOMemoryTypeVirtual64 == type
+	|| kIOMemoryTypePhysical  == type || kIOMemoryTypePhysical64 == type);
     if (kIOMemoryTypeUIO == type) {
 	user_size_t us;
 	uio_getiov((uio_t) r.uio, ind, &addr, &us); len = us;
     }
+    else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type)) {
+	IOAddressRange cur = r.v64[ind];
+	addr = cur.address;
+	len  = cur.length;
+    }
     else {
 	IOVirtualRange cur = r.v[ind];
 	addr = cur.address;
@@ -334,6 +349,15 @@ IOMemoryDescriptor::withAddress(vm_address_t address,
                                 IODirection  direction,
                                 task_t       task)
 {
+#if TEST_V64
+    if (task)
+    {
+	IOOptionBits options = (IOOptionBits) direction;
+	if (task == kernel_task)
+	    options |= kIOMemoryAutoPrepare;
+	return (IOMemoryDescriptor::withAddressRange(address, length, options, task));
+    }
+#endif
     IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
     if (that)
     {
@@ -351,6 +375,9 @@ IOMemoryDescriptor::withPhysicalAddress(
 				IOByteCount		length,
 				IODirection      	direction )
 {
+#if TEST_P64
+    return (IOMemoryDescriptor::withAddressRange(address, length, (IOOptionBits) direction, NULL));
+#endif
     IOGeneralMemoryDescriptor *self = new IOGeneralMemoryDescriptor;
     if (self
     && !self->initWithPhysicalAddress(address, length, direction)) {
@@ -379,6 +406,39 @@ IOMemoryDescriptor::withRanges(	IOVirtualRange * ranges,
     return 0;
 }
 
+IOMemoryDescriptor *
+IOMemoryDescriptor::withAddressRange(mach_vm_address_t address,
+                                       mach_vm_size_t length,
+                                       IOOptionBits   options,
+                                       task_t         task)
+{
+    IOAddressRange range = { address, length };
+    return (IOMemoryDescriptor::withAddressRanges(&range, 1, options, task));
+}
+
+IOMemoryDescriptor *
+IOMemoryDescriptor::withAddressRanges(IOAddressRange *   ranges,
+                                       UInt32           rangeCount,
+                                       IOOptionBits     options,
+                                       task_t           task)
+{
+    IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
+    if (that)
+    {
+	if (task)
+	    options |= kIOMemoryTypeVirtual64;
+	else
+	    options |= kIOMemoryTypePhysical64;
+
+       if (that->initWithOptions(ranges, rangeCount, 0, task, options, /* mapper */ 0))
+           return that;
+
+       that->release();
+    }
+
+    return 0;
+}
+
 
 /*
  * withRanges:
@@ -453,8 +513,8 @@ IOMemoryDescriptor::withSubRange(IOMemoryDescriptor *	of,
     return self;
 }
 
-IOMemoryDescriptor * IOMemoryDescriptor::
-    withPersistentMemoryDescriptor(IOMemoryDescriptor *originalMD)
+IOMemoryDescriptor *
+IOMemoryDescriptor::withPersistentMemoryDescriptor(IOMemoryDescriptor *originalMD)
 {
     IOGeneralMemoryDescriptor *origGenMD = 
 	OSDynamicCast(IOGeneralMemoryDescriptor, originalMD);
@@ -466,8 +526,8 @@ IOMemoryDescriptor * IOMemoryDescriptor::
 	return 0;
 }
 
-IOMemoryDescriptor * IOGeneralMemoryDescriptor::
-    withPersistentMemoryDescriptor(IOGeneralMemoryDescriptor *originalMD)
+IOMemoryDescriptor *
+IOGeneralMemoryDescriptor::withPersistentMemoryDescriptor(IOGeneralMemoryDescriptor *originalMD)
 {
     ipc_port_t sharedMem = (ipc_port_t) originalMD->createNamedEntry();
 
@@ -677,6 +737,7 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
     switch (type) {
     case kIOMemoryTypeUIO:
     case kIOMemoryTypeVirtual:
+    case kIOMemoryTypeVirtual64:
         assert(task);
         if (!task)
             return false;
@@ -684,6 +745,7 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
             break;
 
     case kIOMemoryTypePhysical:		// Neither Physical nor UPL should have a task
+    case kIOMemoryTypePhysical64:
 	mapper = kIOMapperNone;
 
     case kIOMemoryTypeUPL:
@@ -710,10 +772,15 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
 
         while (_wireCount)
             complete();
-        if (_kernPtrAligned)
-            unmapFromKernel();
         if (_ranges.v && _rangesIsAllocated)
-            IODelete(_ranges.v, IOVirtualRange, _rangesCount);
+	{
+	    if (kIOMemoryTypeUIO == type)
+		uio_free((uio_t) _ranges.v);
+	    else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type))
+		IODelete(_ranges.v64, IOAddressRange, _rangesCount);
+	    else
+		IODelete(_ranges.v, IOVirtualRange, _rangesCount);
+	}
 	if (_memEntry)
 	    { ipc_port_release_send((ipc_port_t) _memEntry); _memEntry = 0; }
     }
@@ -726,7 +793,7 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
     // Grab the appropriate mapper
     if (mapper == kIOMapperNone)
         mapper = 0;	// No Mapper
-    else if (!mapper) {
+    else if (mapper == kIOMapperSystem) {
         IOMapper::checkForSystemMapper();
         gIOSystemMapper = mapper = IOMapper::gSystem;
     }
@@ -738,10 +805,12 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
 
     // DEPRECATED variable initialisation
     _direction             = (IODirection) (_flags & kIOMemoryDirectionMask);
-    _position              = 0;
-    _kernPtrAligned        = 0;
-    _cachedPhysicalAddress = 0;
-    _cachedVirtualAddress  = 0;
+
+    __iomd_reservedA = 0;
+    __iomd_reservedB = 0;
+    __iomd_reservedC = 0;
+
+    _highestPage = 0;
 
     if (kIOMemoryTypeUPL == type) {
 
@@ -761,7 +830,7 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
         dataP->fMapper = mapper;
         dataP->fPageCnt = 0;
 
-        _wireCount++;	// UPLs start out life wired
+ //       _wireCount++;	// UPLs start out life wired
 
         _length    = count;
         _pages    += atop_32(offset + count + PAGE_MASK) - atop_32(offset);
@@ -773,6 +842,9 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
         // Set the flag kIOPLOnDevice convieniently equal to 1
         iopl.fFlags  = pageList->device | kIOPLExternUPL;
         iopl.fIOMDOffset = 0;
+
+        _highestPage = upl_get_highest_page(iopl.fIOPL);
+
         if (!pageList->device) {
             // Pre-compute the offset into the UPL's page list
             pageList = &pageList[atop_32(offset)];
@@ -792,7 +864,8 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
         _memoryEntries->appendBytes(&iopl, sizeof(iopl));
     }
     else {
-	// kIOMemoryTypeVirtual | kIOMemoryTypeUIO | kIOMemoryTypePhysical
+	// kIOMemoryTypeVirtual  | kIOMemoryTypeVirtual64 | kIOMemoryTypeUIO 
+	// kIOMemoryTypePhysical | kIOMemoryTypePhysical64
 	
 	// Initialize the memory descriptor
 	if (options & kIOMemoryAsReference) {
@@ -805,13 +878,27 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
 	    _ranges.v = (IOVirtualRange *) buffers;
 	}
 	else {
-	    assert(kIOMemoryTypeUIO != type);
-
 	    _rangesIsAllocated = true;
-	    _ranges.v = IONew(IOVirtualRange, count);
-	    if (!_ranges.v)
-                return false;
-	    bcopy(buffers, _ranges.v, count * sizeof(IOVirtualRange));
+	    switch (_flags & kIOMemoryTypeMask)
+	    {
+	      case kIOMemoryTypeUIO:
+		_ranges.v = (IOVirtualRange *) uio_duplicate((uio_t) buffers);
+		break;
+
+	      case kIOMemoryTypeVirtual64:
+	      case kIOMemoryTypePhysical64:
+		_ranges.v64 = IONew(IOAddressRange, count);
+		if (!_ranges.v64)
+		    return false;
+		bcopy(buffers, _ranges.v, count * sizeof(IOAddressRange));
+		break;
+	      case kIOMemoryTypeVirtual:
+		_ranges.v = IONew(IOVirtualRange, count);
+		if (!_ranges.v)
+		    return false;
+		bcopy(buffers, _ranges.v, count * sizeof(IOVirtualRange));
+		break;
+	    }
 	} 
 
 	// Find starting address within the vector of ranges
@@ -826,8 +913,15 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
 	    getAddrLenForInd(addr, len, type, vec, ind);
 	    pages += (atop_64(addr + len + PAGE_MASK) - atop_64(addr));
 	    len += length;
-	    assert(len > length);	// Check for 32 bit wrap around
+	    assert(len >= length);	// Check for 32 bit wrap around
 	    length = len;
+
+	    if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
+	    {
+		ppnum_t highPage = atop_64(addr + len - 1);
+		if (highPage > _highestPage)
+		    _highestPage = highPage;
+	    }
 	} 
 	_length      = length;
 	_pages       = pages;
@@ -835,9 +929,9 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
 
         // Auto-prepare memory at creation time.
         // Implied completion when descriptor is free-ed
-        if (kIOMemoryTypePhysical == type)
+        if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
             _wireCount++;	// Physical MDs are, by definition, wired
-        else { /* kIOMemoryTypeVirtual | kIOMemoryTypeUIO */
+        else { /* kIOMemoryTypeVirtual | kIOMemoryTypeVirtual64 | kIOMemoryTypeUIO */
             ioGMDData *dataP;
             unsigned dataSize = computeDataSize(_pages, /* upls */ count * 2);
 
@@ -883,10 +977,16 @@ void IOGeneralMemoryDescriptor::free()
     if (_memoryEntries)
         _memoryEntries->release();
 
-    if (_kernPtrAligned)
-        unmapFromKernel();
     if (_ranges.v && _rangesIsAllocated)
-        IODelete(_ranges.v, IOVirtualRange, _rangesCount);
+    {
+	IOOptionBits type = _flags & kIOMemoryTypeMask;
+	if (kIOMemoryTypeUIO == type)
+	    uio_free((uio_t) _ranges.v);
+	else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type))
+	    IODelete(_ranges.v64, IOAddressRange, _rangesCount);
+	else
+	    IODelete(_ranges.v, IOVirtualRange, _rangesCount);
+    }
 
     if (reserved && reserved->devicePager)
 	device_pager_deallocate( (memory_object_t) reserved->devicePager );
@@ -940,17 +1040,17 @@ IOOptionBits IOMemoryDescriptor::getTag( void )
 }
 
 // @@@ gvdl: who is using this API?  Seems like a wierd thing to implement.
-IOPhysicalAddress IOMemoryDescriptor::getSourceSegment( IOByteCount   offset,
-                                                        IOByteCount * length )
+IOPhysicalAddress
+IOMemoryDescriptor::getSourceSegment( IOByteCount   offset, IOByteCount * length )
 {
-    IOPhysicalAddress physAddr = 0;
+    addr64_t physAddr = 0;
 
     if( prepare() == kIOReturnSuccess) {
-        physAddr = getPhysicalSegment( offset, length );
+        physAddr = getPhysicalSegment64( offset, length );
         complete();
     }
 
-    return( physAddr );
+    return( (IOPhysicalAddress) physAddr ); // truncated but only page offset is used
 }
 
 IOByteCount IOMemoryDescriptor::readBytes
@@ -1044,137 +1144,247 @@ extern "C" unsigned int IODefaultCacheBits(addr64_t pa);
                     panic("IOGMD::setPosition deprecated");
 /* DEPRECATED */ }
 
-IOPhysicalAddress IOGeneralMemoryDescriptor::getPhysicalSegment
-                        (IOByteCount offset, IOByteCount *lengthOfSegment)
+IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const
 {
-    IOPhysicalAddress address = 0;
-    IOPhysicalLength  length  = 0;
+    if (kIOMDGetCharacteristics == op) {
 
-//  assert(offset <= _length);
-    if (offset < _length) // (within bounds?)
-    {
-        if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) {
-            unsigned int ind;
+	if (dataSize < sizeof(IOMDDMACharacteristics))
+	    return kIOReturnUnderrun;
 
-            // Physical address based memory descriptor
+	IOMDDMACharacteristics *data = (IOMDDMACharacteristics *) vData;
+	data->fLength = _length;
+	data->fSGCount = _rangesCount;
+	data->fPages = _pages;
+	data->fDirection = _direction;
+	if (!_wireCount)
+	    data->fIsPrepared = false;
+	else {
+	    data->fIsPrepared = true;
+	    data->fHighestPage = _highestPage;
+	    if (_memoryEntries) {
+		ioGMDData *gmdData = getDataP(_memoryEntries);
+		ioPLBlock *ioplList = getIOPLList(gmdData);
+		UInt count = getNumIOPL(_memoryEntries, gmdData);
+
+		data->fIsMapped = (gmdData->fMapper && _pages && (count > 0)
+			       && ioplList[0].fMappedBase);
+		if (count == 1)
+		    data->fPageAlign = (ioplList[0].fPageOffset & PAGE_MASK) | ~PAGE_MASK;
+	    }
+	    else
+		data->fIsMapped = false;
+	}
 
-            // Find offset within descriptor and make it relative
-            // to the current _range.
-            for (ind = 0 ; offset >= _ranges.p[ind].length; ind++ )
-                offset -= _ranges.p[ind].length;
-    
-            IOPhysicalRange cur = _ranges.p[ind];
-            address = cur.address + offset;
-            length  = cur.length  - offset;
-
-            // see how far we can coalesce ranges
-            for (++ind; ind < _rangesCount; ind++) {
-                cur =  _ranges.p[ind];
-        
-                if (address + length != cur.address)
-                    break;
-    
-                length += cur.length;
-            }
+	return kIOReturnSuccess;
+    }
+    else if (!(kIOMDWalkSegments & op))
+	return kIOReturnBadArgument;
+
+    // Get the next segment
+    struct InternalState {
+	IOMDDMAWalkSegmentArgs fIO;
+	UInt fOffset2Index;
+	UInt fIndex;
+	UInt fNextOffset;
+    } *isP;
+
+    // Find the next segment
+    if (dataSize < sizeof(*isP))
+	return kIOReturnUnderrun;
+
+    isP = (InternalState *) vData;
+    UInt offset = isP->fIO.fOffset;
+    bool mapped = isP->fIO.fMapped;
+
+    if (offset >= _length)
+	return (offset == _length)? kIOReturnOverrun : kIOReturnInternalError;
+
+    // Validate the previous offset
+    UInt ind, off2Ind = isP->fOffset2Index;
+    if ((kIOMDFirstSegment != op) 
+	&& offset 
+	&& (offset == isP->fNextOffset || off2Ind <= offset))
+	ind = isP->fIndex;
+    else
+	ind = off2Ind = 0;	// Start from beginning
 
-            // @@@ gvdl: should be assert(address);
-            // but can't as NVidia GeForce creates a bogus physical mem
-	    assert(address
-		|| /* nvidia */ (!_ranges.p[0].address && 1 == _rangesCount));
-            assert(length);
-        }
-        else do {
-            // We need wiring & we are wired.
-            assert(_wireCount);
+    UInt length;
+    UInt64 address;
+    if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) {
 
-            if (!_wireCount)
-	    {
-		panic("IOGMD: not wired for getPhysicalSegment()");
-                continue;
-	    }
+	// Physical address based memory descriptor
+	const IOPhysicalRange *physP = (IOPhysicalRange *) &_ranges.p[0];
 
-            assert(_memoryEntries);
+	// Find the range after the one that contains the offset
+	UInt len;
+	for (len = 0; off2Ind <= offset; ind++) {
+	    len = physP[ind].length;
+	    off2Ind += len;
+	}
 
-            ioGMDData * dataP = getDataP(_memoryEntries);
-            const ioPLBlock *ioplList = getIOPLList(dataP);
-            UInt ind, numIOPLs = getNumIOPL(_memoryEntries, dataP);
-            upl_page_info_t *pageList = getPageList(dataP);
+	// Calculate length within range and starting address
+	length   = off2Ind - offset;
+	address  = physP[ind - 1].address + len - length;
 
-            assert(numIOPLs > 0);
+	// see how far we can coalesce ranges
+	while (ind < _rangesCount && address + length == physP[ind].address) {
+	    len = physP[ind].length;
+	    length += len;
+	    off2Ind += len;
+	    ind++;
+	}
 
-            // Scan through iopl info blocks looking for block containing offset
-            for (ind = 1; ind < numIOPLs; ind++) {
-                if (offset < ioplList[ind].fIOMDOffset)
-                    break;
-            }
+	// correct contiguous check overshoot
+	ind--;
+	off2Ind -= len;
+    }
+    else if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64) {
 
-            // Go back to actual range as search goes past it
-            ioPLBlock ioplInfo = ioplList[ind - 1];
+	// Physical address based memory descriptor
+	const IOAddressRange *physP = (IOAddressRange *) &_ranges.v64[0];
 
-            if (ind < numIOPLs)
-                length = ioplList[ind].fIOMDOffset;
-            else
-                length = _length;
-            length -= offset;			// Remainder within iopl
+	// Find the range after the one that contains the offset
+	mach_vm_size_t len;
+	for (len = 0; off2Ind <= offset; ind++) {
+	    len = physP[ind].length;
+	    off2Ind += len;
+	}
 
-            // Subtract offset till this iopl in total list
-            offset -= ioplInfo.fIOMDOffset;
+	// Calculate length within range and starting address
+	length   = off2Ind - offset;
+	address  = physP[ind - 1].address + len - length;
 
-            // This is a mapped IOPL so we just need to compute an offset
-            // relative to the mapped base.
-            if (ioplInfo.fMappedBase) {
-                offset += (ioplInfo.fPageOffset & PAGE_MASK);
-                address = ptoa_32(ioplInfo.fMappedBase) + offset;
-                continue;
-            }
+	// see how far we can coalesce ranges
+	while (ind < _rangesCount && address + length == physP[ind].address) {
+	    len = physP[ind].length;
+	    length += len;
+	    off2Ind += len;
+	    ind++;
+	}
+
+	// correct contiguous check overshoot
+	ind--;
+	off2Ind -= len;
+    }
+    else do {
+	if (!_wireCount)
+	    panic("IOGMD: not wired for the IODMACommand");
 
-            // Currently the offset is rebased into the current iopl.
-            // Now add the iopl 1st page offset.
-            offset += ioplInfo.fPageOffset;
+	assert(_memoryEntries);
 
-            // For external UPLs the fPageInfo field points directly to
-            // the upl's upl_page_info_t array.
-            if (ioplInfo.fFlags & kIOPLExternUPL)
-                pageList = (upl_page_info_t *) ioplInfo.fPageInfo;
-            else
-                pageList = &pageList[ioplInfo.fPageInfo];
+	ioGMDData * dataP = getDataP(_memoryEntries);
+	const ioPLBlock *ioplList = getIOPLList(dataP);
+	UInt numIOPLs = getNumIOPL(_memoryEntries, dataP);
+	upl_page_info_t *pageList = getPageList(dataP);
 
-            // Check for direct device non-paged memory
-            if ( ioplInfo.fFlags & kIOPLOnDevice ) {
-                address = ptoa_32(pageList->phys_addr) + offset;
-                continue;
-            }
+	assert(numIOPLs > 0);
 
-            // Now we need compute the index into the pageList
-            ind = atop_32(offset);
-            offset &= PAGE_MASK;
+	// Scan through iopl info blocks looking for block containing offset
+	while (ind < numIOPLs && offset >= ioplList[ind].fIOMDOffset)
+	    ind++;
 
-            IOPhysicalAddress pageAddr = pageList[ind].phys_addr;
-            address = ptoa_32(pageAddr) + offset;
-
-            // Check for the remaining data in this upl being longer than the
-            // remainder on the current page.  This should be checked for
-            // contiguous pages
-            if (length > PAGE_SIZE - offset) {
-                // See if the next page is contiguous.  Stop looking when we hit
-                // the end of this upl, which is indicated by the
-                // contigLength >= length.
-                IOByteCount contigLength = PAGE_SIZE - offset;
-
-                // Look for contiguous segment
-                while (contigLength < length
-                &&     ++pageAddr == pageList[++ind].phys_addr) {
-                    contigLength += PAGE_SIZE;
-                }
-                if (length > contigLength)
-                    length = contigLength;
-            }
-    
-            assert(address);
-            assert(length);
+	// Go back to actual range as search goes past it
+	ioPLBlock ioplInfo = ioplList[ind - 1];
+	off2Ind = ioplInfo.fIOMDOffset;
+
+	if (ind < numIOPLs)
+	    length = ioplList[ind].fIOMDOffset;
+	else
+	    length = _length;
+	length -= offset;			// Remainder within iopl
+
+	// Subtract offset till this iopl in total list
+	offset -= off2Ind;
+
+	// If a mapped address is requested and this is a pre-mapped IOPL
+	// then just need to compute an offset relative to the mapped base.
+	if (mapped && ioplInfo.fMappedBase) {
+	    offset += (ioplInfo.fPageOffset & PAGE_MASK);
+	    address = ptoa_64(ioplInfo.fMappedBase) + offset;
+	    continue;	// Done leave do/while(false) now
+	}
+
+	// The offset is rebased into the current iopl.
+	// Now add the iopl 1st page offset.
+	offset += ioplInfo.fPageOffset;
+
+	// For external UPLs the fPageInfo field points directly to
+	// the upl's upl_page_info_t array.
+	if (ioplInfo.fFlags & kIOPLExternUPL)
+	    pageList = (upl_page_info_t *) ioplInfo.fPageInfo;
+	else
+	    pageList = &pageList[ioplInfo.fPageInfo];
+
+	// Check for direct device non-paged memory
+	if ( ioplInfo.fFlags & kIOPLOnDevice ) {
+	    address = ptoa_64(pageList->phys_addr) + offset;
+	    continue;	// Done leave do/while(false) now
+	}
 
-        } while (0);
+	// Now we need compute the index into the pageList
+	UInt pageInd = atop_32(offset);
+	offset &= PAGE_MASK;
+
+	// Compute the starting address of this segment
+	IOPhysicalAddress pageAddr = pageList[pageInd].phys_addr;
+	address = ptoa_64(pageAddr) + offset;
+
+	// length is currently set to the length of the remainider of the iopl.
+	// We need to check that the remainder of the iopl is contiguous.
+	// This is indicated by pageList[ind].phys_addr being sequential.
+	IOByteCount contigLength = PAGE_SIZE - offset;
+	while (contigLength < length
+		&& ++pageAddr == pageList[++pageInd].phys_addr)
+	{
+	    contigLength += PAGE_SIZE;
+	}
+
+	if (contigLength < length)
+	    length = contigLength;
+	
+
+	assert(address);
+	assert(length);
+
+    } while (false);
+
+    // Update return values and state
+    isP->fIO.fIOVMAddr = address;
+    isP->fIO.fLength   = length;
+    isP->fIndex        = ind;
+    isP->fOffset2Index = off2Ind;
+    isP->fNextOffset   = isP->fIO.fOffset + length;
+
+    return kIOReturnSuccess;
+}
+
+addr64_t
+IOGeneralMemoryDescriptor::getPhysicalSegment64(IOByteCount offset, IOByteCount *lengthOfSegment)
+{
+    IOReturn    ret;
+    IOByteCount length  = 0;
+    addr64_t    address = 0;
 
+    if (offset < _length) // (within bounds?)
+    {
+	IOMDDMAWalkSegmentState _state;
+	IOMDDMAWalkSegmentArgs * state = (IOMDDMAWalkSegmentArgs *) &_state;
+
+	state->fOffset = offset;
+	state->fLength = _length - offset;
+	state->fMapped = false;
+
+	ret = dmaCommandOperation(kIOMDFirstSegment, _state, sizeof(_state));
+
+	if ((kIOReturnSuccess != ret) && (kIOReturnOverrun != ret))
+		DEBG("getPhysicalSegment64 dmaCommandOperation(%lx), %p, offset %qx, addr %qx, len %qx\n", 
+					ret, this, state->fOffset,
+					state->fIOVMAddr, state->fLength);
+	if (kIOReturnSuccess == ret)
+	{
+	    address = state->fIOVMAddr;
+	    length  = state->fLength;
+	}
         if (!address)
             length = 0;
     }
@@ -1182,29 +1392,80 @@ IOPhysicalAddress IOGeneralMemoryDescriptor::getPhysicalSegment
     if (lengthOfSegment)
         *lengthOfSegment = length;
 
-    return address;
+    return (address);
+}
+
+IOPhysicalAddress
+IOGeneralMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *lengthOfSegment)
+{
+    IOReturn          ret;
+    IOByteCount       length  = 0;
+    addr64_t	      address = 0;
+
+//  assert(offset <= _length);
+
+    if (offset < _length) // (within bounds?)
+    {
+	IOMDDMAWalkSegmentState _state;
+	IOMDDMAWalkSegmentArgs * state = (IOMDDMAWalkSegmentArgs *) &_state;
+
+	state->fOffset = offset;
+	state->fLength = _length - offset;
+	state->fMapped = true;
+
+	ret = dmaCommandOperation(
+		kIOMDFirstSegment, _state, sizeof(_state));
+
+	if ((kIOReturnSuccess != ret) && (kIOReturnOverrun != ret))
+	    DEBG("getPhysicalSegment dmaCommandOperation(%lx), %p, offset %qx, addr %qx, len %qx\n", 
+				    ret, this, state->fOffset,
+				    state->fIOVMAddr, state->fLength);
+	if (kIOReturnSuccess == ret)
+	{
+	    address = state->fIOVMAddr;
+	    length  = state->fLength;
+	}
+
+        if (!address)
+            length = 0;
+    }
+
+    if ((address + length) > 0x100000000ULL)
+    {
+	panic("getPhysicalSegment() out of 32b range 0x%qx, len 0x%x, class %s",
+		    address, length, (getMetaClass())->getClassName());
+    }
+
+    if (lengthOfSegment)
+        *lengthOfSegment = length;
+
+    return ((IOPhysicalAddress) address);
 }
 
-addr64_t IOMemoryDescriptor::getPhysicalSegment64
-                        (IOByteCount offset, IOByteCount *lengthOfSegment)
+addr64_t
+IOMemoryDescriptor::getPhysicalSegment64(IOByteCount offset, IOByteCount *lengthOfSegment)
 {
     IOPhysicalAddress phys32;
     IOByteCount	      length;
     addr64_t 	      phys64;
+    IOMapper *        mapper = 0;
 
     phys32 = getPhysicalSegment(offset, lengthOfSegment);
     if (!phys32)
 	return 0;
 
     if (gIOSystemMapper)
+	mapper = gIOSystemMapper;
+
+    if (mapper)
     {
 	IOByteCount origLen;
 
-	phys64 = gIOSystemMapper->mapAddr(phys32);
+	phys64 = mapper->mapAddr(phys32);
 	origLen = *lengthOfSegment;
 	length = page_size - (phys64 & (page_size - 1));
 	while ((length < origLen)
-	    && ((phys64 + length) == gIOSystemMapper->mapAddr(phys32 + length)))
+	    && ((phys64 + length) == mapper->mapAddr(phys32 + length)))
 	    length += page_size;
 	if (length > origLen)
 	    length = origLen;
@@ -1217,8 +1478,8 @@ addr64_t IOMemoryDescriptor::getPhysicalSegment64
     return phys64;
 }
 
-IOPhysicalAddress IOGeneralMemoryDescriptor::
-getSourceSegment(IOByteCount offset, IOByteCount *lengthOfSegment)
+IOPhysicalAddress
+IOGeneralMemoryDescriptor::getSourceSegment(IOByteCount offset, IOByteCount *lengthOfSegment)
 {
     IOPhysicalAddress address = 0;
     IOPhysicalLength  length  = 0;
@@ -1283,6 +1544,42 @@ getSourceSegment(IOByteCount offset, IOByteCount *lengthOfSegment)
 
 
 
+IOReturn 
+IOMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const
+{
+    if (kIOMDGetCharacteristics == op) {
+	if (dataSize < sizeof(IOMDDMACharacteristics))
+	    return kIOReturnUnderrun;
+
+	IOMDDMACharacteristics *data = (IOMDDMACharacteristics *) vData;
+	data->fLength = getLength();
+	data->fSGCount = 0;
+	data->fDirection = _direction;
+	if (IOMapper::gSystem)
+	    data->fIsMapped = true;
+	data->fIsPrepared = true;	// Assume prepared - fails safe
+    }
+    else if (kIOMDWalkSegments & op) {
+	if (dataSize < sizeof(IOMDDMAWalkSegmentArgs))
+	    return kIOReturnUnderrun;
+
+	IOMDDMAWalkSegmentArgs *data = (IOMDDMAWalkSegmentArgs *) vData;
+	IOByteCount offset  = (IOByteCount) data->fOffset;
+
+	IOPhysicalLength length;
+	IOMemoryDescriptor *ncmd = const_cast<IOMemoryDescriptor *>(this);
+	if (data->fMapped && IOMapper::gSystem)
+	    data->fIOVMAddr = ncmd->getPhysicalSegment(offset, &length);
+	else
+	    data->fIOVMAddr = ncmd->getPhysicalSegment64(offset, &length);
+	data->fLength = length;
+    }
+    else
+	return kIOReturnBadArgument;
+
+    return kIOReturnSuccess;
+}
+
 IOReturn IOMemoryDescriptor::setPurgeable( IOOptionBits newState,
                                            IOOptionBits * oldState )
 {
@@ -1414,10 +1711,12 @@ io_get_kernel_static_upl(
 	vm_size_t		*upl_size,
 	upl_t			*upl,
 	upl_page_info_array_t	page_list,
-	unsigned int		*count)
+	unsigned int		*count,
+	ppnum_t			*highest_page)
 {
     unsigned int pageCount, page;
     ppnum_t phys;
+    ppnum_t highestPage = 0;
 
     pageCount = atop_32(*upl_size);
     if (pageCount > *count)
@@ -1436,8 +1735,12 @@ io_get_kernel_static_upl(
 	page_list[page].dirty	  = 0;
 	page_list[page].precious  = 0;
 	page_list[page].device	  = 0;
+	if (phys > highestPage)
+	    highestPage = page;
     }
 
+    *highest_page = highestPage;
+
     return ((page >= pageCount) ? kIOReturnSuccess : kIOReturnVMError);
 }
 
@@ -1451,7 +1754,7 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
     ipc_port_t sharedMem = (ipc_port_t) _memEntry;
 
     assert(!_wireCount);
-    assert(kIOMemoryTypeVirtual == type || kIOMemoryTypeUIO == type);
+    assert(kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type);
 
     if (_pages >= gIOMaximumMappedIOPageCount)
 	return kIOReturnNoResources;
@@ -1470,7 +1773,7 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
         forDirection = _direction;
 
     int uplFlags;    // This Mem Desc's default flags for upl creation
-    switch (forDirection)
+    switch (kIODirectionOutIn & forDirection)
     {
     case kIODirectionOut:
         // Pages do not need to be marked as dirty on commit
@@ -1485,6 +1788,11 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
     }
     uplFlags |= UPL_SET_IO_WIRE | UPL_SET_LITE;
 
+#ifdef UPL_NEED_32BIT_ADDR
+    if (kIODirectionPrepareToPhys32 & forDirection) 
+	uplFlags |= UPL_NEED_32BIT_ADDR;
+#endif
+
     // Find the appropriate vm_map for the given task
     vm_map_t curMap;
     if (_task == kernel_task && (kIOMemoryBufferPageable & _flags))
@@ -1496,10 +1804,12 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
     Ranges vec = _ranges;
     unsigned int pageIndex = 0;
     IOByteCount mdOffset = 0;
+    ppnum_t highestPage = 0;
     for (UInt range = 0; range < _rangesCount; range++) {
         ioPLBlock iopl;
 	user_addr_t startPage;
         IOByteCount numBytes;
+	ppnum_t highPage = 0;
 
 	// Get the startPage address and length of vec[range]
 	getAddrLenForInd(startPage, numBytes, type, vec, range);
@@ -1539,7 +1849,8 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
 						&ioplSize,
 						&iopl.fIOPL,
 						baseInfo,
-						&numPageInfo);
+						&numPageInfo,
+						&highPage);
 	    }
 	    else if (sharedMem) {
 		error = memory_object_iopl_request(sharedMem, 
@@ -1565,6 +1876,11 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
             if (error != KERN_SUCCESS)
                 goto abortExit;
 
+	    if (iopl.fIOPL)
+		highPage = upl_get_highest_page(iopl.fIOPL);
+	    if (highPage > highestPage)
+		highestPage = highPage;
+
             error = kIOReturnNoMemory;
 
             if (baseInfo->device) {
@@ -1579,7 +1895,7 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
             }
             else {
                 iopl.fFlags = 0;
-                if (mapper)
+		if (mapper)
                     mapper->iovmInsert(mapBase, pageIndex,
                                        baseInfo, numPageInfo);
             }
@@ -1621,6 +1937,8 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
         }
     }
 
+    _highestPage = highestPage;
+
     return kIOReturnSuccess;
 
 abortExit:
@@ -1660,7 +1978,7 @@ IOReturn IOGeneralMemoryDescriptor::prepare(IODirection forDirection)
     IOOptionBits type = _flags & kIOMemoryTypeMask;
 
     if (!_wireCount
-    && (kIOMemoryTypeVirtual == type || kIOMemoryTypeUIO == type) ) {
+    && (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) ) {
         error = wireVirtual(forDirection);
         if (error)
             return error;
@@ -1691,7 +2009,7 @@ IOReturn IOGeneralMemoryDescriptor::complete(IODirection /* forDirection */)
     if (!_wireCount) {
 	IOOptionBits type = _flags & kIOMemoryTypeMask;
 
-        if (kIOMemoryTypePhysical == type) {
+        if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type)) {
             /* kIOMemoryTypePhysical */
             // DO NOTHING
         }
@@ -1704,7 +2022,7 @@ IOReturn IOGeneralMemoryDescriptor::complete(IODirection /* forDirection */)
                 dataP->fMapper->iovmFree(ioplList[0].fMappedBase, _pages);
 
             // Only complete iopls that we created which are for TypeVirtual
-            if (kIOMemoryTypeVirtual == type || kIOMemoryTypeUIO == type) {
+            if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) {
                 for (UInt ind = 0; ind < count; ind++)
 		    if (ioplList[ind].fIOPL) {
 			 upl_commit(ioplList[ind].fIOPL, 0, 0);
@@ -1755,7 +2073,7 @@ IOReturn IOGeneralMemoryDescriptor::doMap(
         vm_size_t size = ptoa_32(_pages);
 
         if( _task) {
-#ifndef i386
+
             memory_object_size_t actualSize = size;
             kr = mach_make_memory_entry_64(get_task_map(_task),
                         &actualSize, range0Addr,
@@ -1772,10 +2090,9 @@ IOReturn IOGeneralMemoryDescriptor::doMap(
             }
 
             if( KERN_SUCCESS != kr)
-#endif /* !i386 */
                 sharedMem = MACH_PORT_NULL;
 
-        } else do {
+        } else do {	// _task == 0, must be physical
 
             memory_object_t 	pager;
 	    unsigned int    	flags = 0;
@@ -1850,11 +2167,9 @@ IOReturn IOGeneralMemoryDescriptor::doMap(
     }
 
 
-#ifndef i386
     if( 0 == sharedMem)
       kr = kIOReturnVMError;
     else
-#endif
       kr = super::doMap( addressMap, atAddress,
                            options, sourceOffset, length );
 
@@ -2263,19 +2578,6 @@ IOReturn IOMemoryDescriptor::handleFault(
 		segLen - pageOffset);
 #endif
 
-
-
-
-
-#ifdef i386  
-	/* i386 doesn't support faulting on device memory yet */
-	if( addressMap && (kIOReturnSuccess == err))
-            err = IOMapPages( addressMap, address, (IOPhysicalAddress) physAddr, segLen, options );
-        assert( KERN_SUCCESS == err );
-	if( err)
-	    break;
-#endif
-
         if( pager) {
             if( reserved && reserved->pagerContig) {
                 IOPhysicalLength	allLen;
@@ -2299,7 +2601,7 @@ IOReturn IOMemoryDescriptor::handleFault(
             if( err)
                 break;
         }
-#ifndef i386
+
 	/*  *** ALERT *** */
 	/*  *** Temporary Workaround *** */
 
@@ -2327,7 +2629,7 @@ IOReturn IOMemoryDescriptor::handleFault(
 
 	/*  *** Temporary Workaround *** */
 	/*  *** ALERT *** */
-#endif
+
 	sourceOffset += segLen - pageOffset;
 	address += segLen;
 	bytes -= segLen;
@@ -2421,28 +2723,46 @@ IOReturn _IOMemoryMap::redirect( task_t safeTask, bool doRedirect )
     } else {
 
         LOCK;
-        if( logical && addressMap
-	&& (!safeTask || (get_task_map(safeTask) != addressMap))
-        && (0 == (options & kIOMapStatic)))
+
+	do
 	{
-	    IOUnmapPages( addressMap, logical, length );
-            if(!doRedirect && safeTask
-	     && ((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical))
-	     {
-                err = vm_deallocate( addressMap, logical, length );
-                err = memory->doMap( addressMap, &logical,
-                                     (options & ~kIOMapAnywhere) /*| kIOMapReserve*/,
-                                     offset, length );
-            } else
-                err = kIOReturnSuccess;
+	    if (!logical)
+		break;
+	    if (!addressMap)
+		break;
+
+	    if ((!safeTask || (get_task_map(safeTask) != addressMap))
+	      && (0 == (options & kIOMapStatic)))
+	    {
+		IOUnmapPages( addressMap, logical, length );
+		if(!doRedirect && safeTask
+		 && (((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) 
+		    || ((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64)))
+		 {
+		    err = vm_deallocate( addressMap, logical, length );
+		    err = memory->doMap( addressMap, &logical,
+					 (options & ~kIOMapAnywhere) /*| kIOMapReserve*/,
+					 offset, length );
+		} else
+		    err = kIOReturnSuccess;
 #ifdef DEBUG
-	    IOLog("IOMemoryMap::redirect(%d, %p) %x:%lx from %p\n", doRedirect, this, logical, length, addressMap);
+		IOLog("IOMemoryMap::redirect(%d, %p) %x:%lx from %p\n", doRedirect, this, logical, length, addressMap);
 #endif
-        }
-        UNLOCK;
+	    }
+	    else if (kIOMapWriteCombineCache == (options & kIOMapCacheMask))
+	    {
+		IOOptionBits newMode;
+		newMode = (options & ~kIOMapCacheMask) | (doRedirect ? kIOMapInhibitCache : kIOMapWriteCombineCache);
+		IOProtectCacheMode(addressMap, logical, length, newMode);
+	    }
+	}
+	while (false);
+
+	UNLOCK;
     }
 
-    if (((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) 
+    if ((((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
+	 || ((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
      && safeTask
      && (doRedirect != (0 != (memory->_flags & kIOMemoryRedirected))))
 	memory->redirect(safeTask, doRedirect);
@@ -2600,8 +2920,8 @@ _IOMemoryMap * _IOMemoryMap::copyCompatible(
     return( mapping );
 }
 
-IOPhysicalAddress _IOMemoryMap::getPhysicalSegment( IOByteCount _offset,
-	       					    IOPhysicalLength * _length)
+IOPhysicalAddress 
+_IOMemoryMap::getPhysicalSegment( IOByteCount _offset, IOPhysicalLength * _length)
 {
     IOPhysicalAddress	address;
 
@@ -2626,6 +2946,20 @@ void IOMemoryDescriptor::initialize( void )
 
     IORegistryEntry::getRegistryRoot()->setProperty(kIOMaximumMappedIOByteCountKey,
 						    ptoa_64(gIOMaximumMappedIOPageCount), 64);
+    if (!gIOCopyMapper)
+    {
+    	IOMapper *
+	mapper = new IOCopyMapper;
+	if (mapper)
+	{
+	    if (mapper->init() && mapper->start(NULL))
+		gIOCopyMapper = (IOCopyMapper *) mapper;
+	    else
+		mapper->release();
+	}
+    }
+
+    gIOLastPage = IOGetLastPageNumber();
 }
 
 void IOMemoryDescriptor::free( void )
@@ -2694,7 +3028,8 @@ IOReturn _IOMemoryMap::redirect(IOMemoryDescriptor * newBackingMemory,
 
     if (logical && addressMap) do 
     {
-	if ((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
+	if (((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
+	    || ((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
 	{
 	    physMem = memory;
 	    physMem->retain();
@@ -2770,7 +3105,8 @@ IOMemoryMap * IOMemoryDescriptor::makeMapping(
 	    if (owner != this)
 		continue;
 
-	    if ((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
+	    if (((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
+		|| ((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
 	    {
 		phys = getPhysicalSegment(offset, &physLen);
 		if (!phys || (physLen < length))
@@ -2953,8 +3289,71 @@ void IOSubMemoryDescriptor::free( void )
 }
 
 
-IOPhysicalAddress IOSubMemoryDescriptor::getPhysicalSegment( IOByteCount offset,
-						      	IOByteCount * length )
+IOReturn
+IOSubMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const
+{
+    IOReturn rtn;
+
+    if (kIOMDGetCharacteristics == op) {
+
+	rtn = _parent->dmaCommandOperation(op, vData, dataSize);
+	if (kIOReturnSuccess == rtn) {
+	    IOMDDMACharacteristics *data = (IOMDDMACharacteristics *) vData;
+	    data->fLength = _length;
+	    data->fSGCount = 0;	// XXX gvdl: need to compute and pages
+	    data->fPages = 0;
+	    data->fPageAlign = 0;
+	}
+
+	return rtn;
+    }
+    else if (kIOMDWalkSegments & op) {
+	if (dataSize < sizeof(IOMDDMAWalkSegmentArgs))
+	    return kIOReturnUnderrun;
+
+	IOMDDMAWalkSegmentArgs *data =
+	    reinterpret_cast<IOMDDMAWalkSegmentArgs *>(vData);
+	UInt offset = data->fOffset;
+	UInt remain = _length - offset;
+	if ((int) remain <= 0)
+	    return (!remain)? kIOReturnOverrun : kIOReturnInternalError;
+
+	data->fOffset = offset + _start;
+	rtn = _parent->dmaCommandOperation(op, vData, dataSize);
+	if (data->fLength > remain)
+	    data->fLength = remain;
+	data->fOffset  = offset;
+
+	return rtn;
+    }
+    else
+	return kIOReturnBadArgument;
+}
+
+addr64_t
+IOSubMemoryDescriptor::getPhysicalSegment64(IOByteCount offset, IOByteCount * length)
+{
+    addr64_t	address;
+    IOByteCount	actualLength;
+
+    assert(offset <= _length);
+
+    if( length)
+        *length = 0;
+
+    if( offset >= _length)
+        return( 0 );
+
+    address = _parent->getPhysicalSegment64( offset + _start, &actualLength );
+
+    if( address && length)
+	*length = min( _length - offset, actualLength );
+
+    return( address );
+}
+
+IOPhysicalAddress
+IOSubMemoryDescriptor::getPhysicalSegment( IOByteCount offset, IOByteCount * length )
 {
     IOPhysicalAddress	address;
     IOByteCount		actualLength;
@@ -2988,8 +3387,8 @@ IOReturn IOSubMemoryDescriptor::doMap(
     return (_parent->doMap(addressMap, atAddress, options, sourceOffset + _start, length));
 }
 
-IOPhysicalAddress IOSubMemoryDescriptor::getSourceSegment( IOByteCount offset,
-						      	   IOByteCount * length )
+IOPhysicalAddress 
+IOSubMemoryDescriptor::getSourceSegment( IOByteCount offset, IOByteCount * length )
 {
     IOPhysicalAddress	address;
     IOByteCount		actualLength;
@@ -3336,7 +3735,7 @@ OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 1);
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 2);
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 3);
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 4);
-OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 5);
+OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 5);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 6);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 7);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 8);
@@ -3349,5 +3748,9 @@ OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 14);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 15);
 
 /* ex-inline function implementation */
-IOPhysicalAddress IOMemoryDescriptor::getPhysicalAddress()
+IOPhysicalAddress 
+IOMemoryDescriptor::getPhysicalAddress()
         { return( getPhysicalSegment( 0, 0 )); }
+
+
+
diff --git a/iokit/Kernel/IOMultiMemoryDescriptor.cpp b/iokit/Kernel/IOMultiMemoryDescriptor.cpp
index 224a6bdba..fb1a38f22 100644
--- a/iokit/Kernel/IOMultiMemoryDescriptor.cpp
+++ b/iokit/Kernel/IOMultiMemoryDescriptor.cpp
@@ -276,9 +276,35 @@ IOReturn IOMultiMemoryDescriptor::complete(IODirection forDirection)
 
 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 
-IOPhysicalAddress IOMultiMemoryDescriptor::getPhysicalSegment(
-                                                       IOByteCount   offset,
-                                                       IOByteCount * length )
+addr64_t IOMultiMemoryDescriptor::getPhysicalSegment64( 
+				    IOByteCount   offset, IOByteCount * length )
+{
+    //
+    // This method returns the physical address of the byte at the given offset
+    // into the memory,  and optionally the length of the physically contiguous
+    // segment from that offset.
+    //
+
+    assert(offset <= _length);
+
+    for ( unsigned index = 0; index < _descriptorsCount; index++ ) 
+    {
+        if ( offset < _descriptors[index]->getLength() )
+        {
+            return _descriptors[index]->getPhysicalSegment64(offset, length);
+        }
+        offset -= _descriptors[index]->getLength();
+    }
+
+    if ( length )  *length = 0;
+
+    return 0;
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+IOPhysicalAddress IOMultiMemoryDescriptor::getPhysicalSegment( 
+				    IOByteCount   offset, IOByteCount * length )
 {
     //
     // This method returns the physical address of the byte at the given offset
diff --git a/iokit/Kernel/IOPMPowerSource.cpp b/iokit/Kernel/IOPMPowerSource.cpp
index 71e7c13bc..281377d3f 100644
--- a/iokit/Kernel/IOPMPowerSource.cpp
+++ b/iokit/Kernel/IOPMPowerSource.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -27,146 +27,373 @@
  */
 
 #include <IOKit/pwr_mgt/IOPMPowerSource.h>
+#include <IOKit/pwr_mgt/IOPM.h>
+#include <IOKit/IOMessage.h>
+#include <IOKit/IOLib.h>
 
-#define super OSObject
+#define super IOService
 
-OSDefineMetaClassAndStructors(IOPMPowerSource, OSObject)
+OSDefineMetaClassAndStructors(IOPMPowerSource, IOService)
 
-// **********************************************************************************
-// init
+// *****************************************************************************
+// powerSource
 //
-// **********************************************************************************
-bool IOPMPowerSource::init (unsigned short whichBatteryIndex)
-{
-  if (!super::init ())
-    return false;
+// Static initializer for IOPMPowerSource. Returns a new instance of the class
+// which the caller must attach to the power plane.
+// *****************************************************************************
 
-  bBatteryIndex = whichBatteryIndex;
-  nextInList    = 0;
+IOPMPowerSource *IOPMPowerSource::powerSource(void)
+{
+    IOPMPowerSource *ps = new IOPMPowerSource;    
 
-  return true;
+    if(ps) {
+        ps->init();
+        return ps;
+    }    
+    return NULL;
 }
 
-// **********************************************************************************
-// capacityPercentRemaining
+// *****************************************************************************
+// init
 //
-// **********************************************************************************
-unsigned long IOPMPowerSource::capacityPercentRemaining (void)
+// *****************************************************************************
+bool IOPMPowerSource::init (void)
 {
-   unsigned long percentage = 0;
+    if (!super::init()) {
+        return false;
+    }
 
-   if (bMaxCapacity > 0)
-     percentage = (bCurCapacity * 100) / bMaxCapacity;
- 
-   // always return a non-zero value unless the real capacity IS zero.
-   if (percentage == 0 && bCurCapacity > 0)
-     percentage = 1;
+    nextInList = NULL;
+
+    properties = OSDictionary::withCapacity(10);
+    if(!properties) return false;
+    properties->setCapacityIncrement(1);
 
-   return percentage;
+    externalConnectedKey = OSSymbol::withCString(kIOPMPSExternalConnectedKey);
+    externalChargeCapableKey = OSSymbol::withCString(kIOPMPSExternalChargeCapableKey);
+    batteryInstalledKey = OSSymbol::withCString(kIOPMPSBatteryInstalledKey);
+    chargingKey = OSSymbol::withCString(kIOPMPSIsChargingKey);
+    warnLevelKey = OSSymbol::withCString(kIOPMPSAtWarnLevelKey);
+    criticalLevelKey = OSSymbol::withCString(kIOPMPSAtCriticalLevelKey);
+    currentCapacityKey = OSSymbol::withCString(kIOPMPSCurrentCapacityKey);
+    maxCapacityKey = OSSymbol::withCString(kIOPMPSMaxCapacityKey);
+    timeRemainingKey = OSSymbol::withCString(kIOPMPSTimeRemainingKey);
+    amperageKey = OSSymbol::withCString(kIOPMPSAmperageKey);
+    voltageKey = OSSymbol::withCString(kIOPMPSVoltageKey);
+    cycleCountKey = OSSymbol::withCString(kIOPMPSCycleCountKey);
+    adapterInfoKey = OSSymbol::withCString(kIOPMPSAdapterInfoKey);
+    locationKey = OSSymbol::withCString(kIOPMPSLocationKey);
+    errorConditionKey = OSSymbol::withCString(kIOPMPSErrorConditionKey);
+    manufacturerKey = OSSymbol::withCString(kIOPMPSManufacturerKey);
+    modelKey = OSSymbol::withCString(kIOPMPSModelKey);
+    serialKey = OSSymbol::withCString(kIOPMPSSerialKey);
+    batteryInfoKey = OSSymbol::withCString(kIOPMPSLegacyBatteryInfoKey);
+
+    return true;
 }
 
-// **********************************************************************************
-// atWarnLevel
+// *****************************************************************************
+// free
 //
-// **********************************************************************************
-bool IOPMPowerSource::atWarnLevel (void)
+// *****************************************************************************
+void IOPMPowerSource::free(void)
 {
-  return bFlags & kBatteryAtWarn;
+    if(properties) properties->release();
+    if(externalConnectedKey) externalConnectedKey->release();
+    if(externalChargeCapableKey) externalChargeCapableKey->release();
+    if(batteryInstalledKey) batteryInstalledKey->release();
+    if(chargingKey) chargingKey->release();
+    if(warnLevelKey) warnLevelKey->release();
+    if(criticalLevelKey) criticalLevelKey->release();
+    if(currentCapacityKey) currentCapacityKey->release();
+    if(maxCapacityKey) maxCapacityKey->release();
+    if(timeRemainingKey) timeRemainingKey->release();
+    if(amperageKey) amperageKey->release();
+    if(voltageKey) voltageKey->release();
+    if(cycleCountKey) cycleCountKey->release();
+    if(adapterInfoKey) adapterInfoKey->release();
+    if(errorConditionKey) errorConditionKey->release();
+    if(manufacturerKey) manufacturerKey->release();
+    if(modelKey) modelKey->release();
+    if(serialKey) serialKey->release();
+    if(locationKey) locationKey->release();
+    if(batteryInfoKey) batteryInfoKey->release();
 }
 
-// **********************************************************************************
-// acConnected
+// *****************************************************************************
+// updateStatus
 //
-// **********************************************************************************
-bool IOPMPowerSource::acConnected (void)
+// Update power source state in IORegistry and message interested clients
+// notifying them of our change.
+// *****************************************************************************
+void IOPMPowerSource::updateStatus (void)
 {
-  return bFlags & kACInstalled;
+    OSCollectionIterator            *iterator;
+    OSObject                        *iteratorKey;
+    OSObject                        *obj;
+
+    iterator = OSCollectionIterator::withCollection(properties);
+    if(!iterator) return;
+
+    while ((iteratorKey = iterator->getNextObject())) {
+        OSSymbol *key;
+    
+        key = OSDynamicCast(OSSymbol, iteratorKey);
+        if (!key) continue;
+        obj = properties->getObject(key);
+        if(!obj) continue;
+        setProperty(key, obj);
+    }
+    iterator->release();
+
+    // And up goes the flare
+    messageClients(kIOPMMessageBatteryStatusHasChanged);
 }
 
-// **********************************************************************************
-// depleted
-//
-// **********************************************************************************
-bool IOPMPowerSource::depleted (void)
-{
-  return bFlags & kBatteryDepleted;
+
+/*******************************************************************************
+ *
+ * PROTECTED Accessors. All the setters! Yay!
+ *
+ ******************************************************************************/
+ 
+void IOPMPowerSource::setExternalConnected(bool b) {
+    properties->setObject(
+                externalConnectedKey,
+                b ? kOSBooleanTrue:kOSBooleanFalse);    
 }
 
-// **********************************************************************************
-// isInstalled
-//
-// **********************************************************************************
-bool IOPMPowerSource::isInstalled (void)
-{
-  return bFlags & kBatteryInstalled;
+void IOPMPowerSource::setExternalChargeCapable(bool b) {
+    properties->setObject(
+                externalChargeCapableKey,
+                b ? kOSBooleanTrue:kOSBooleanFalse);    
 }
 
-// **********************************************************************************
-// isCharging
-//
-// **********************************************************************************
-bool IOPMPowerSource::isCharging (void)
-{
-  return bFlags & kBatteryCharging;
+void IOPMPowerSource::setBatteryInstalled(bool b) {
+    properties->setObject(
+                batteryInstalledKey,
+                b ? kOSBooleanTrue:kOSBooleanFalse);    
 }
 
-// **********************************************************************************
-// timeRemaining
-//
-// **********************************************************************************
-unsigned long IOPMPowerSource::timeRemaining (void)
-{
-  return bTimeRemaining;
+void IOPMPowerSource::setIsCharging(bool b) {
+    properties->setObject(
+                chargingKey,
+                b ? kOSBooleanTrue:kOSBooleanFalse);    
 }
 
-// **********************************************************************************
-// maxCapacity
-//
-// **********************************************************************************
-unsigned long IOPMPowerSource::maxCapacity (void)
-{
-  return bMaxCapacity;
+void IOPMPowerSource::setAtWarnLevel(bool b) {
+    properties->setObject(
+                warnLevelKey,
+                b ? kOSBooleanTrue:kOSBooleanFalse);    
 }
 
-// **********************************************************************************
-// curCapacity
-//
-// **********************************************************************************
-unsigned long IOPMPowerSource::curCapacity (void)
-{
-  return bCurCapacity;
+void IOPMPowerSource::setAtCriticalLevel(bool b) {
+    properties->setObject(
+                criticalLevelKey,
+                b ? kOSBooleanTrue:kOSBooleanFalse);    
 }
 
-// **********************************************************************************
-// currentDrawn
-//
-// **********************************************************************************
-long IOPMPowerSource::currentDrawn (void)
-{
-  return bCurrent;
+
+void IOPMPowerSource::setCurrentCapacity(unsigned int val) {
+    OSNumber *n = OSNumber::withNumber(val, 32);
+    properties->setObject(
+                currentCapacityKey,
+                n);
+    n->release();
 }
 
-// **********************************************************************************
-// voltage
-//
-// **********************************************************************************
+void IOPMPowerSource::setMaxCapacity(unsigned int val) {
+    OSNumber *n = OSNumber::withNumber(val, 32);
+    properties->setObject(
+                maxCapacityKey,
+                n);
+    n->release();
+}
 
-unsigned long IOPMPowerSource::voltage (void)
-{
-  return bVoltage;
+void IOPMPowerSource::setTimeRemaining(int val) {
+    OSNumber *n = OSNumber::withNumber(val, 32);
+    properties->setObject(
+                timeRemainingKey,
+                n);
+    n->release();
 }
 
-// **********************************************************************************
-// updateStatus
-//
-// **********************************************************************************
+void IOPMPowerSource::setAmperage(int val) {
+    OSNumber *n = OSNumber::withNumber(val, 32);
+    properties->setObject(
+                amperageKey,
+                n);
+    n->release();
+}
 
-void IOPMPowerSource::updateStatus (void)
+void IOPMPowerSource::setVoltage(unsigned int val) {
+    OSNumber *n = OSNumber::withNumber(val, 32);
+    properties->setObject(
+                voltageKey,
+                n);
+    n->release();
+}
+
+void IOPMPowerSource::setCycleCount(unsigned int val) {
+    OSNumber *n = OSNumber::withNumber(val, 32);
+    properties->setObject(
+                cycleCountKey,
+                n);
+    n->release();
+}
+
+void IOPMPowerSource::setAdapterInfo(int val) {
+    OSNumber *n = OSNumber::withNumber(val, 32);
+    properties->setObject(
+                adapterInfoKey,
+                n);
+    n->release();
+}
+
+void IOPMPowerSource::setLocation(int val) {
+    OSNumber *n = OSNumber::withNumber(val, 32);
+    properties->setObject(
+                locationKey,
+                n);
+    n->release();
+}
+
+void IOPMPowerSource::setErrorCondition(OSSymbol *s) {
+    properties->setObject(errorConditionKey, s);
+}
+
+void IOPMPowerSource::setManufacturer(OSSymbol *s) {
+    properties->setObject(manufacturerKey, s);
+}
+
+void IOPMPowerSource::setModel(OSSymbol *s) {
+    properties->setObject(modelKey, s);
+}
+
+void IOPMPowerSource::setSerial(OSSymbol *s) {
+    properties->setObject(serialKey, s);
+}
+
+void IOPMPowerSource::setLegacyIOBatteryInfo(OSDictionary *d) {
+    properties->setObject(batteryInfoKey, d);
+}
+
+
+
+
+/*******************************************************************************
+ *
+ * PUBLIC Accessors. All the getters! Boo!
+ *
+ ******************************************************************************/
+
+bool IOPMPowerSource::externalConnected(void) {
+    return (kOSBooleanTrue == properties->getObject(externalConnectedKey));
+}
+
+bool IOPMPowerSource::externalChargeCapable(void) {
+    return (kOSBooleanTrue == properties->getObject(externalChargeCapableKey));
+}
+
+bool IOPMPowerSource::batteryInstalled(void) {
+    return (kOSBooleanTrue == properties->getObject(batteryInstalledKey));
+}
+
+bool IOPMPowerSource::isCharging(void) {
+    return (kOSBooleanTrue == properties->getObject(chargingKey));
+}
+
+bool IOPMPowerSource::atWarnLevel(void) {
+    return (kOSBooleanTrue == properties->getObject(warnLevelKey));
+}
+
+bool IOPMPowerSource::atCriticalLevel(void) {
+    return (kOSBooleanTrue == properties->getObject(criticalLevelKey));
+}
+
+unsigned int IOPMPowerSource::currentCapacity(void) {
+    OSNumber        *n;
+    n = OSDynamicCast(OSNumber, properties->getObject(currentCapacityKey));
+    if(!n) return 0;
+    else return (unsigned int)n->unsigned32BitValue();
+}
+
+unsigned int IOPMPowerSource::maxCapacity(void) {
+    OSNumber        *n;
+    n = OSDynamicCast(OSNumber, properties->getObject(maxCapacityKey));
+    if(!n) return 0;
+    else return (unsigned int)n->unsigned32BitValue();
+}
+
+unsigned int IOPMPowerSource::capacityPercentRemaining(void) 
 {
+    unsigned int _currentCapacity = currentCapacity();
+    unsigned int _maxCapacity = maxCapacity();
+    if(0 == _maxCapacity) {
+        return 0;
+    } else {
+        return ((100*_currentCapacity) / _maxCapacity);
+    }
+}
 
+int IOPMPowerSource::timeRemaining(void) {
+    OSNumber        *n;
+    n = OSDynamicCast(OSNumber, properties->getObject(timeRemainingKey));
+    if(!n) return 0;
+    else return (int)n->unsigned32BitValue();
 }
 
+int IOPMPowerSource::amperage(void) {
+    OSNumber        *n;
+    n = OSDynamicCast(OSNumber, properties->getObject(amperageKey));
+    if(!n) return 0;
+    else return (int)n->unsigned32BitValue();
+}
 
+unsigned int IOPMPowerSource::voltage(void) {
+    OSNumber        *n;
+    n = OSDynamicCast(OSNumber, properties->getObject(voltageKey));
+    if(!n) return 0;
+    else return (unsigned int)n->unsigned32BitValue();
+}
 
+unsigned int IOPMPowerSource::cycleCount(void) {
+    OSNumber        *n;
+    n = OSDynamicCast(OSNumber, properties->getObject(cycleCountKey));
+    if(!n) return 0;
+    else return (unsigned int)n->unsigned32BitValue();
+}
 
+int IOPMPowerSource::adapterInfo(void) {
+    OSNumber        *n;
+    n = OSDynamicCast(OSNumber, properties->getObject(adapterInfoKey));
+    if(!n) return 0;
+    else return (int)n->unsigned32BitValue();
+}
 
+int IOPMPowerSource::location(void) {
+    OSNumber        *n;
+    n = OSDynamicCast(OSNumber, properties->getObject(locationKey));
+    if(!n) return 0;
+    else return (unsigned int)n->unsigned32BitValue();
+}
+
+OSSymbol *IOPMPowerSource::errorCondition(void) {
+    return OSDynamicCast(OSSymbol, properties->getObject(errorConditionKey));
+}
+
+OSSymbol *IOPMPowerSource::manufacturer(void) {
+    return OSDynamicCast(OSSymbol, properties->getObject(manufacturerKey));
+}
+
+OSSymbol *IOPMPowerSource::model(void) {
+    return OSDynamicCast(OSSymbol, properties->getObject(modelKey));
+}
+
+OSSymbol *IOPMPowerSource::serial(void) {
+    return OSDynamicCast(OSSymbol, properties->getObject(serialKey));
+}
+
+OSDictionary *IOPMPowerSource::legacyIOBatteryInfo(void) {
+    return OSDynamicCast(OSDictionary, properties->getObject(batteryInfoKey));
+}
diff --git a/iokit/Kernel/IOPMPowerSourceList.cpp b/iokit/Kernel/IOPMPowerSourceList.cpp
index f8ad3dc5d..a933ebf76 100644
--- a/iokit/Kernel/IOPMPowerSourceList.cpp
+++ b/iokit/Kernel/IOPMPowerSourceList.cpp
@@ -32,54 +32,61 @@
 #define super OSObject
 OSDefineMetaClassAndStructors(IOPMPowerSourceList,OSObject)
 
-//*********************************************************************************
+//******************************************************************************
 // init
 //
-//*********************************************************************************
+//******************************************************************************
 void IOPMPowerSourceList::initialize ( void )
 {
     firstItem = NULL;
     length = 0;
 }
 
-//*********************************************************************************
+//******************************************************************************
 // addToList
 //
-//*********************************************************************************
+//******************************************************************************
 
-IOReturn IOPMPowerSourceList::addToList ( IOPMPowerSource * newPowerSource )
+IOReturn IOPMPowerSourceList::addToList(IOPMPowerSource *newPowerSource)
 {
     IOPMPowerSource * nextPowerSource;
-    nextPowerSource = firstItem;				// Is new object already in the list?
-    while (  nextPowerSource != NULL ) {
-        if ( nextPowerSource == newPowerSource ) {
-            return IOPMNoErr;				// yes, just return
+
+    // Is new object already in the list?
+    nextPowerSource = firstItem;
+    while (  nextPowerSource != NULL ) 
+    {
+        if ( nextPowerSource == newPowerSource ) 
+        {
+            // yes, just return
+            return IOPMNoErr;				
         }
         nextPowerSource = nextInList(nextPowerSource);
     }
-    newPowerSource->nextInList = firstItem;		// add it to list
+
+    // add it to list
+    newPowerSource->nextInList = firstItem;
     firstItem = newPowerSource;
-    length += 1;
+    length++;
     return IOPMNoErr;
 }
 
 
-//*********************************************************************************
+//******************************************************************************
 // firstInList
 //
-//*********************************************************************************
+//******************************************************************************
 
 IOPMPowerSource * IOPMPowerSourceList::firstInList ( void )
 {
     return firstItem;
 }
 
-//*********************************************************************************
+//******************************************************************************
 // nextInList
 //
-//*********************************************************************************
+//******************************************************************************
 
-IOPMPowerSource * IOPMPowerSourceList::nextInList ( IOPMPowerSource * currentItem )
+IOPMPowerSource * IOPMPowerSourceList::nextInList(IOPMPowerSource *currentItem)
 {
     if ( currentItem != NULL ) {
        return (currentItem->nextInList);
@@ -87,54 +94,56 @@ IOPMPowerSource * IOPMPowerSourceList::nextInList ( IOPMPowerSource * currentIte
     return NULL;
 }
 
-//*********************************************************************************
+//******************************************************************************
 // numberOfItems
 //
-//*********************************************************************************
+//******************************************************************************
 
 unsigned long IOPMPowerSourceList::numberOfItems ( void )
 {
     return length;
 }
 
-//*********************************************************************************
+//******************************************************************************
 // removeFromList
 //
 // Find the item in the list, unlink it, and free it.
-//*********************************************************************************
+//******************************************************************************
 
 IOReturn IOPMPowerSourceList::removeFromList ( IOPMPowerSource * theItem )
 {
     IOPMPowerSource * item = firstItem;
     IOPMPowerSource * temp;
 
-    if ( item != NULL ) {
-        if ( item  == theItem ) {
-            firstItem = item->nextInList;
+    if ( NULL == item) goto exit;
+    
+    if ( item  == theItem ) {
+        firstItem = item->nextInList;
+        length--;
+        item->release();
+        return IOPMNoErr;
+    }
+    while ( item->nextInList != NULL ) {
+        if ( item->nextInList == theItem ) {
+            temp = item->nextInList;
+            item->nextInList = temp->nextInList;
             length--;
-            item->release();
+            temp->release();
             return IOPMNoErr;
         }
-        while ( item->nextInList != NULL ) {
-            if ( item->nextInList == theItem ) {
-                temp = item->nextInList;
-                item->nextInList = temp->nextInList;
-                length--;
-                temp->release();
-                return IOPMNoErr;
-            }
-            item = item->nextInList;
-        }
+        item = item->nextInList;
     }
+
+exit:
     return IOPMNoErr;
 }
 
 
-//*********************************************************************************
+//******************************************************************************
 // free
 //
 // Free all items in the list, and then free the list itself
-//*********************************************************************************
+//******************************************************************************
 
 void IOPMPowerSourceList::free (void )
 {
@@ -146,7 +155,7 @@ void IOPMPowerSourceList::free (void )
         next->release();
         next = firstItem;        
     }
-super::free();
+    super::free();
 }
 
 
diff --git a/iokit/Kernel/IOPMPowerStateQueue.cpp b/iokit/Kernel/IOPMPowerStateQueue.cpp
index 3aca81e51..617f6f91f 100644
--- a/iokit/Kernel/IOPMPowerStateQueue.cpp
+++ b/iokit/Kernel/IOPMPowerStateQueue.cpp
@@ -27,10 +27,46 @@
  */
  
 #include "IOPMPowerStateQueue.h"
+#include "IOKit/IOLocks.h"
 #undef super
 #define super IOEventSource
 OSDefineMetaClassAndStructors(IOPMPowerStateQueue, IOEventSource);
 
+#ifdef __i386__ /* ppc does this right and doesn't need these routines */
+static
+void *	OSDequeueAtomic(void ** inList, SInt32 inOffset)
+{
+	void *	oldListHead;
+	void *	newListHead;
+
+	do {
+		oldListHead = *inList;
+		if (oldListHead == NULL) {
+			break;
+		}
+		
+		newListHead = *(void **) (((char *) oldListHead) + inOffset);
+	} while (! OSCompareAndSwap((UInt32)oldListHead,
+					(UInt32)newListHead, (UInt32 *)inList));
+	return oldListHead;
+}
+
+static
+void	OSEnqueueAtomic(void ** inList, void * inNewLink, SInt32 inOffset)
+{
+	void *	oldListHead;
+	void *	newListHead = inNewLink;
+	void **	newLinkNextPtr = (void **) (((char *) inNewLink) + inOffset);
+
+	do {
+		oldListHead = *inList;
+		*newLinkNextPtr = oldListHead;
+	} while (! OSCompareAndSwap((UInt32)oldListHead, (UInt32)newListHead,
+					(UInt32 *)inList));
+}
+#endif /* __i386__ */
+
+
 IOPMPowerStateQueue *IOPMPowerStateQueue::PMPowerStateQueue(OSObject *inOwner)
 {
     IOPMPowerStateQueue     *me = new IOPMPowerStateQueue;
@@ -50,7 +86,9 @@ bool IOPMPowerStateQueue::init(OSObject *owner, Action action)
 
     // Queue of powerstate changes
     changes = NULL;
-
+#ifdef __i386__
+    if (!(tmpLock = IOLockAlloc()))  panic("IOPMPowerStateQueue::init can't alloc lock");
+#endif
     return true;
 }
 
@@ -67,8 +105,13 @@ bool IOPMPowerStateQueue::unIdleOccurred(IOService *inTarget, unsigned long inSt
     new_one->target = inTarget;
     
     // Change to queue
+#ifdef __i386__
+    IOLockLock(tmpLock);
+#endif
     OSEnqueueAtomic((void **)&changes, (void *)new_one, 0);
-    
+#ifdef __i386__
+    IOLockUnlock(tmpLock);
+#endif
     signalWorkAvailable();
 
     return true;
@@ -83,8 +126,14 @@ bool IOPMPowerStateQueue::checkForWork()
     UInt16                      theAction;
     
     // Dequeue and process the state change request
+#ifdef __i386__
+    IOLockLock(tmpLock);
+#endif
     if((theNode = (PowerChangeEntry *)OSDequeueAtomic((void **)&changes, 0)))
     {
+#ifdef __i386__
+      IOLockUnlock(tmpLock);
+#endif
         theState = theNode->state;
         theTarget = theNode->target;
         theAction = theNode->actionType;
@@ -93,11 +142,15 @@ bool IOPMPowerStateQueue::checkForWork()
         switch (theAction)
         {
             case kUnIdle:
-                theTarget->command_received(theState, 0, 0, 0);
+                theTarget->command_received((void *)theState, 0, 0, 0);
                 break;
         }
     }
-
+#ifdef __i386__
+    else {
+      IOLockUnlock(tmpLock);
+    }
+#endif
     // Return true if there's more work to be done
     if(changes) return true;
     else return false;
diff --git a/iokit/Kernel/IOPMPowerStateQueue.h b/iokit/Kernel/IOPMPowerStateQueue.h
index 80d4a96e8..ef376f269 100644
--- a/iokit/Kernel/IOPMPowerStateQueue.h
+++ b/iokit/Kernel/IOPMPowerStateQueue.h
@@ -55,6 +55,9 @@ private:
     };
 
     void                        *changes;
+#ifdef __i386__
+    IOLock                      *tmpLock;
+#endif
 
 protected:
     virtual bool checkForWork(void);
diff --git a/iokit/Kernel/IOPMWorkArbiter.cpp b/iokit/Kernel/IOPMWorkArbiter.cpp
new file mode 100644
index 000000000..44c9842a2
--- /dev/null
+++ b/iokit/Kernel/IOPMWorkArbiter.cpp
@@ -0,0 +1,159 @@
+/*
+ * Copyright (c) 2001-2002 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+ 
+#include "IOPMWorkArbiter.h"
+#include "IOKit/IOLocks.h"
+#undef super
+#define super IOEventSource
+OSDefineMetaClassAndStructors(IOPMWorkArbiter, IOEventSource);
+
+/*************************************************************************/
+static
+void *    _dequeue(void ** inList, SInt32 inOffset)
+{
+    void *    oldListHead;
+    void *    newListHead;
+
+    do {
+        oldListHead = *inList;
+        if (oldListHead == NULL) {
+            break;
+        }
+        
+        newListHead = *(void **) (((char *) oldListHead) + inOffset);
+    } while (! OSCompareAndSwap((UInt32)oldListHead,
+                    (UInt32)newListHead, (UInt32 *)inList));
+    return oldListHead;
+}
+
+/*************************************************************************/
+static
+void    _enqueue(void ** inList, void * inNewLink, SInt32 inOffset)
+{
+    void *    oldListHead;
+    void *    newListHead = inNewLink;
+    void **    newLinkNextPtr = (void **) (((char *) inNewLink) + inOffset);
+
+    do {
+        oldListHead = *inList;
+        *newLinkNextPtr = oldListHead;
+    } while (! OSCompareAndSwap((UInt32)oldListHead, (UInt32)newListHead,
+                    (UInt32 *)inList));
+}
+
+/*************************************************************************/
+IOPMWorkArbiter *IOPMWorkArbiter::pmWorkArbiter(
+    IOPMrootDomain *inOwner)
+{
+    IOPMWorkArbiter     *me = new IOPMWorkArbiter;
+
+    if(me && !me->init((OSObject *)inOwner, 0) )
+    {
+        me->release();
+        return NULL;
+    }
+
+    return me;
+}
+
+/*************************************************************************/
+bool IOPMWorkArbiter::init(OSObject *owner, Action action)
+{
+    if(!(super::init(owner, (IOEventSource::Action) action))) 
+        return false;
+        
+    events = NULL;
+    fRootDomain = (IOPMrootDomain *)owner;
+    
+    if (!(tmpLock = IOLockAlloc())) {
+        panic("IOPMWorkArbiter::init can't alloc lock");
+    }
+    return true;
+}
+
+
+/*************************************************************************/
+bool IOPMWorkArbiter::clamshellStateChangeOccurred(uint32_t messageValue)
+{
+    PMEventEntry                 *new_one = NULL;
+
+    new_one = (PMEventEntry *)IOMalloc(sizeof(PMEventEntry));
+    if(!new_one) return false;
+    
+    new_one->actionType = IOPMWorkArbiter::kRootDomainClamshellChanged;
+    new_one->target = (IOService *)fRootDomain;
+    new_one->intArgument = messageValue;
+       new_one->target->retain();
+    
+    IOLockLock(tmpLock);
+    _enqueue((void **)&events, (void *)new_one, 0);
+    IOLockUnlock(tmpLock);
+    signalWorkAvailable();
+
+    return true;
+}
+
+
+/*************************************************************************/
+// checkForWork() is called in a gated context
+bool IOPMWorkArbiter::checkForWork()
+{
+    PMEventEntry                *theNode;
+    IOService                   *theTarget;
+    UInt16                      theAction;
+    
+    // Dequeue and process the state change request
+    IOLockLock(tmpLock);
+    if((theNode = (PMEventEntry *)_dequeue((void **)&events, 0)))
+    {
+      IOLockUnlock(tmpLock);
+        theTarget = theNode->target;
+        theAction = theNode->actionType;
+
+        switch (theAction)
+        {
+            case kRootDomainClamshellChanged:
+                theTarget->messageClients(
+                                kIOPMMessageClamshellStateChange, 
+                                (void *)theNode->intArgument);
+                break;
+        }
+
+               if (theTarget)
+                       theTarget->release();
+        IOFree((void *)theNode, sizeof(PMEventEntry));
+    }
+    else {
+      IOLockUnlock(tmpLock);
+    }
+    // Return true if there's more work to be done
+    if(events) return true;
+    else return false;
+}
+
+
diff --git a/iokit/Kernel/IOPMWorkArbiter.h b/iokit/Kernel/IOPMWorkArbiter.h
new file mode 100644
index 000000000..af98cf284
--- /dev/null
+++ b/iokit/Kernel/IOPMWorkArbiter.h
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2001-2002 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+ 
+#ifndef _IOPMWorkArbiter_H_
+#define _IOPMWorkArbiter_H_
+ 
+#include <IOKit/IOWorkLoop.h>
+#include <IOKit/IOEventSource.h>
+#include <IOKit/IOService.h>
+extern "C" {
+    #include <kern/queue.h>
+}
+
+class IOPMWorkArbiter : public IOEventSource
+{
+    OSDeclareDefaultStructors(IOPMWorkArbiter);
+
+private:
+    enum {
+        kAllAcked = 0,
+        kDriverAcked,
+        kRootDomainClamshellChanged
+    };
+
+    // Queue of requested states
+    struct PMEventEntry 
+    {
+        void                    *next;
+        IOService               *target;
+        uint16_t                actionType;
+        uint32_t                intArgument;
+    };
+
+    IOPMrootDomain              *fRootDomain;
+
+    void                        *events;
+    IOLock                      *tmpLock;
+
+protected:
+    virtual bool checkForWork(void);
+
+public:
+    IOLock              *arbiterLock;
+
+    //typedef void (*Action)(IOService *target, unsigned long state);
+
+    virtual bool init(OSObject *owner, Action action = 0);
+
+    // static initialiser
+    static IOPMWorkArbiter *pmWorkArbiter(IOPMrootDomain *owner);
+      
+    /* IOPMrootDomain work */
+    bool clamshellStateChangeOccurred(uint32_t messageValue);
+ };
+ 
+ 
+ #endif /* _IOPMWorkArbiter_H_ */
+ 
diff --git a/iokit/Kernel/IOPMinformeeList.cpp b/iokit/Kernel/IOPMinformeeList.cpp
index 2ba440a14..57ac0bb7f 100644
--- a/iokit/Kernel/IOPMinformeeList.cpp
+++ b/iokit/Kernel/IOPMinformeeList.cpp
@@ -42,6 +42,32 @@ void IOPMinformeeList::initialize ( void )
     length = 0;
 }
 
+//******************************************************************************
+// getSharedRecursiveLock
+//
+//******************************************************************************
+IORecursiveLock *IOPMinformeeList::getSharedRecursiveLock( void )
+{
+    static IORecursiveLock *sharedListLock = NULL;
+    
+    /* A running system could have 50-60+ instances of IOPMInformeeList. 
+     * They'll share this lock, since list insertion and removal is relatively
+     * rare, and generally tied to major events like device discovery.
+     *
+     * getSharedRecursiveLock() is called from IOStartIOKit to initialize
+     * the sharedListLock before any IOPMinformeeLists are instantiated.
+     *
+     * The IOPMinformeeList class will be around for the lifetime of the system,
+     * we don't worry about freeing this lock.
+     */
+
+    if ( NULL == sharedListLock )
+    {
+        sharedListLock = IORecursiveLockAlloc();
+    }
+    return sharedListLock;
+}
+
 //*********************************************************************************
 // addToList
 //
@@ -50,16 +76,78 @@ void IOPMinformeeList::initialize ( void )
 IOReturn IOPMinformeeList::addToList ( IOPMinformee * newInformee )
 {
     IOPMinformee * nextInformee;
-    nextInformee = firstItem;				// Is new object already in the list?
-    while (  nextInformee != NULL ) {
-        if ( nextInformee->whatObject == newInformee->whatObject ) {
-            return IOPMNoErr;				// yes, just return
+    IORecursiveLock    *listLock = getSharedRecursiveLock();
+
+    if(!listLock)
+        return kIOReturnError;
+
+    IORecursiveLockLock(listLock);
+    nextInformee = firstItem;				
+    
+    // Is new object already in the list?
+    while (  nextInformee != NULL ) 
+    {
+        if ( nextInformee->whatObject == newInformee->whatObject ) 
+        {
+            // object is present; just exit
+            goto unlock_and_exit;
         }
         nextInformee = nextInList(nextInformee);
     }
-    newInformee->nextInList = firstItem;		// add it to list
+
+    // add it to the front of the list
+    newInformee->nextInList = firstItem;
     firstItem = newInformee;
-    length += 1;
+    length++;
+
+unlock_and_exit:
+    IORecursiveLockUnlock(listLock);
+    return IOPMNoErr;
+}
+
+
+//*********************************************************************************
+// removeFromList
+//
+// Find the item in the list, unlink it, and free it.
+//*********************************************************************************
+
+IOReturn IOPMinformeeList::removeFromList ( IOService * theItem )
+{
+    IOPMinformee * item = firstItem;
+    IOPMinformee * temp;
+    IORecursiveLock    *listLock = getSharedRecursiveLock();
+
+    if ( NULL == item ) 
+        return IOPMNoErr;
+    if(!listLock) 
+        return kIOReturnError;
+
+    IORecursiveLockLock( listLock );
+    
+    if ( item->whatObject == theItem ) 
+    {
+        firstItem = item->nextInList;
+        length--;
+        item->release();
+        goto unlock_and_exit;
+    }
+    
+    while ( item->nextInList != NULL ) 
+    {
+        if ( item->nextInList->whatObject == theItem ) 
+        {
+            temp = item->nextInList;
+            item->nextInList = temp->nextInList;
+            length--;
+            temp->release();
+            goto unlock_and_exit;
+        }
+        item = item->nextInList;
+    }
+
+unlock_and_exit:
+    IORecursiveLockUnlock(listLock);
     return IOPMNoErr;
 }
 
@@ -119,38 +207,6 @@ IOPMinformee * IOPMinformeeList::findItem ( IOService * driverOrChild )
 }
 
 
-//*********************************************************************************
-// removeFromList
-//
-// Find the item in the list, unlink it, and free it.
-//*********************************************************************************
-
-IOReturn IOPMinformeeList::removeFromList ( IOService * theItem )
-{
-    IOPMinformee * item = firstItem;
-    IOPMinformee * temp;
-
-    if ( item != NULL ) {
-        if ( item->whatObject == theItem ) {
-            firstItem = item->nextInList;
-            length--;
-            item->release();
-            return IOPMNoErr;
-        }
-        while ( item->nextInList != NULL ) {
-            if ( item->nextInList->whatObject == theItem ) {
-                temp = item->nextInList;
-                item->nextInList = temp->nextInList;
-                length--;
-                temp->release();
-                return IOPMNoErr;
-            }
-            item = item->nextInList;
-        }
-    }
-    return IOPMNoErr;
-}
-
 
 //*********************************************************************************
 // free
diff --git a/iokit/Kernel/IOPMrootDomain.cpp b/iokit/Kernel/IOPMrootDomain.cpp
index f34eaf86c..f448027bf 100644
--- a/iokit/Kernel/IOPMrootDomain.cpp
+++ b/iokit/Kernel/IOPMrootDomain.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -35,17 +35,21 @@
 #include <IOKit/pwr_mgt/IOPMPrivate.h>
 #include <IOKit/IODeviceTreeSupport.h>
 #include <IOKit/IOMessage.h>
+#include <IOKit/IOReturn.h>
 #include "RootDomainUserClient.h"
 #include "IOKit/pwr_mgt/IOPowerConnection.h"
 #include "IOPMPowerStateQueue.h"
 #include <IOKit/IOCatalogue.h>
 #include <IOKit/IOHibernatePrivate.h>
+#include "IOPMWorkArbiter.h"
 
 #ifdef __ppc__
 #include <ppc/pms.h>
 #endif
 
-extern "C" void kprintf(const char *, ...);
+extern "C" {
+IOReturn OSMetaClassSystemSleepOrWake( UInt32 );
+}
 
 extern const IORegistryPlane * gIOPowerPlane;
 
@@ -53,6 +57,14 @@ IOReturn broadcast_aggressiveness ( OSObject *, void *, void *, void *, void * )
 static void sleepTimerExpired(thread_call_param_t);
 static void wakeupClamshellTimerExpired ( thread_call_param_t us);
 
+// "IOPMSetSleepSupported"  callPlatformFunction name
+static const OSSymbol *sleepSupportedPEFunction = NULL;
+
+#define kIOSleepSupportedKey  "IOSleepSupported"
+
+#define kRD_AllPowerSources (kIOPMSupportedOnAC \
+                           | kIOPMSupportedOnBatt \
+                           | kIOPMSupportedOnUPS)
 
 #define number_of_power_states 5
 #define OFF_STATE 0
@@ -66,25 +78,52 @@ static void wakeupClamshellTimerExpired ( thread_call_param_t us);
 #define SLEEP_POWER kIOPMAuxPowerOn
 #define DOZE_POWER kIOPMDoze
 
+#define kLocalEvalClamshellCommand        (1 << 15)
+
 static IOPMPowerState ourPowerStates[number_of_power_states] = {
-    {1,0,			0,		0,0,0,0,0,0,0,0,0},		// state 0, off
-    {1,kIOPMRestartCapability,	kIOPMRestart,	RESTART_POWER,0,0,0,0,0,0,0,0},	// state 1, restart
-    {1,kIOPMSleepCapability,	kIOPMSleep,	SLEEP_POWER,0,0,0,0,0,0,0,0},	// state 2, sleep
-    {1,kIOPMDoze,		kIOPMDoze,	DOZE_POWER,0,0,0,0,0,0,0,0},	// state 3, doze
-    {1,kIOPMPowerOn,		kIOPMPowerOn,	ON_POWER,0,0,0,0,0,0,0,0},	// state 4, on
+    // state 0, off
+    {1,0,			0,		0,0,0,0,0,0,0,0,0},
+    // state 1, restart
+    {1,kIOPMRestartCapability,	kIOPMRestart,	RESTART_POWER,0,0,0,0,0,0,0,0},	
+    // state 2, sleep
+    {1,kIOPMSleepCapability,	kIOPMSleep,	SLEEP_POWER,0,0,0,0,0,0,0,0},	
+    // state 3, doze
+    {1,kIOPMDoze,		kIOPMDoze,	DOZE_POWER,0,0,0,0,0,0,0,0},	
+    // state 4, on
+    {1,kIOPMPowerOn,		kIOPMPowerOn,	ON_POWER,0,0,0,0,0,0,0,0},	
 };
 
-// RESERVED IOPMrootDomain class variables
-#define diskSyncCalloutEntry                _reserved->diskSyncCalloutEntry
-#define _settingController                  _reserved->_settingController
-#define _batteryLocationNotifier            _reserved->_batteryLocationNotifier
-#define _displayWranglerNotifier            _reserved->_displayWranglerNotifier
-
-
 static IOPMrootDomain * gRootDomain;
 static UInt32           gSleepOrShutdownPending = 0;
 
 
+class PMSettingObject : public OSObject
+{
+    OSDeclareDefaultStructors(PMSettingObject)
+private:
+    IOPMrootDomain                  *parent;
+    IOPMSettingControllerCallback   func;
+    OSObject                        *target;
+    uintptr_t                       refcon;
+    uint32_t                        *publishedFeatureID;
+    int                             releaseAtCount;
+public:
+    static PMSettingObject *pmSettingObject(
+                IOPMrootDomain      *parent_arg,
+                IOPMSettingControllerCallback   handler_arg,
+                OSObject    *target_arg,
+                uintptr_t   refcon_arg,
+                uint32_t    supportedPowerSources,
+                const OSSymbol *settings[]);
+
+    void setPMSetting(const OSSymbol *type, OSObject *obj);
+
+    void taggedRelease(const void *tag, const int when) const;
+    void free(void);
+};
+
+
+
 #define super IOService
 OSDefineMetaClassAndStructors(IOPMrootDomain,IOService)
 
@@ -173,7 +212,7 @@ one power state from Sleep to Doze, and any objects in the tree for which this i
 ADB will turn on again so that they can wake the system out of Doze (keyboard/mouse activity will cause the Display Wrangler
 to be tickled)).
 */
-
+    
 // **********************************************************************************
 
 IOPMrootDomain * IOPMrootDomain::construct( void )
@@ -199,6 +238,13 @@ static void disk_sync_callout(thread_call_param_t p0, thread_call_param_t p1)
     rootDomain->allowPowerChange(pmRef);
 }
 
+// **********************************************************************************
+IOPMWorkArbiter *IOPMrootDomain::getPMArbiter(void)
+{
+    return pmArbiter;
+}
+
+
 // **********************************************************************************
 // start
 //
@@ -206,10 +252,29 @@ static void disk_sync_callout(thread_call_param_t p0, thread_call_param_t p1)
 // expert informs us we are the root.
 // **********************************************************************************
 
+#define kRootDomainSettingsCount        13
 
 bool IOPMrootDomain::start ( IOService * nub )
 {
-    OSDictionary                            *tmpDict;
+    OSIterator      *psIterator;
+    OSDictionary    *tmpDict;
+    const OSSymbol  *settingsArr[kRootDomainSettingsCount] = 
+        {
+            OSSymbol::withCString(kIOPMSettingSleepOnPowerButtonKey),
+            OSSymbol::withCString(kIOPMSettingAutoWakeSecondsKey),
+            OSSymbol::withCString(kIOPMSettingAutoPowerSecondsKey),
+            OSSymbol::withCString(kIOPMSettingAutoWakeCalendarKey),
+            OSSymbol::withCString(kIOPMSettingAutoPowerCalendarKey),
+            OSSymbol::withCString(kIOPMSettingDebugWakeRelativeKey),
+            OSSymbol::withCString(kIOPMSettingDebugPowerRelativeKey),
+            OSSymbol::withCString(kIOPMSettingWakeOnRingKey),
+            OSSymbol::withCString(kIOPMSettingRestartOnPowerLossKey),
+            OSSymbol::withCString(kIOPMSettingWakeOnClamshellKey),
+            OSSymbol::withCString(kIOPMSettingWakeOnACChangeKey),
+            OSSymbol::withCString(kIOPMSettingTimeZoneOffsetKey),
+            OSSymbol::withCString(kIOPMSettingMobileMotionModuleKey)
+        };
+    
 
     pmPowerStateQueue = 0;
 
@@ -221,33 +286,60 @@ bool IOPMrootDomain::start ( IOService * nub )
     gRootDomain = this;
 
     PMinit();
-    setProperty("IOSleepSupported","");
+    
+    sleepSupportedPEFunction = OSSymbol::withCString("IOPMSetSleepSupported");
+    canSleep = true;
+    setProperty(kIOSleepSupportedKey,true);
 
     allowSleep = true;
     sleepIsSupported = true;
     systemBooting = true;
-    ignoringClamshell = true;
     sleepSlider = 0;
     idleSleepPending = false;
-    canSleep = true;
     wrangler = NULL;
     sleepASAP = false;
-    _settingController = NULL;
+    clamshellIsClosed = false;
+    clamshellExists = false;
+    ignoringClamshell = true;
     ignoringClamshellDuringWakeup = false;
+    acAdaptorConnect = true;
     
     tmpDict = OSDictionary::withCapacity(1);
     setProperty(kRootDomainSupportedFeatures, tmpDict);
     tmpDict->release();
     
+    settingsCallbacks = OSDictionary::withCapacity(1);
+
+    // Create a list of the valid PM settings that we'll relay to
+    // interested clients in setProperties() => setPMSetting()
+    allowedPMSettings = OSArray::withObjects(
+                    (const OSObject **)settingsArr,
+                    kRootDomainSettingsCount,
+                    0);
+                    
+    fPMSettingsDict = OSDictionary::withCapacity(5);
+            
     pm_vars->PMworkloop = IOWorkLoop::workLoop();				
     pmPowerStateQueue = IOPMPowerStateQueue::PMPowerStateQueue(this);
     pm_vars->PMworkloop->addEventSource(pmPowerStateQueue);
     
+    /* Initialize PM arbiter */
+    pmArbiter = IOPMWorkArbiter::pmWorkArbiter(this);
+    arbiterWorkLoop = IOWorkLoop::workLoop();
+    arbiterWorkLoop->addEventSource(pmArbiter);
+    
     featuresDictLock = IOLockAlloc();
+    settingsCtrlLock = IORecursiveLockAlloc();
     
-    extraSleepTimer = thread_call_allocate((thread_call_func_t)sleepTimerExpired, (thread_call_param_t) this);
-    clamshellWakeupIgnore = thread_call_allocate((thread_call_func_t)wakeupClamshellTimerExpired, (thread_call_param_t) this);
-    diskSyncCalloutEntry = thread_call_allocate(&disk_sync_callout, (thread_call_param_t) this);
+    extraSleepTimer = thread_call_allocate(
+                        (thread_call_func_t)sleepTimerExpired, 
+                        (thread_call_param_t) this);
+    clamshellWakeupIgnore = thread_call_allocate(
+                        (thread_call_func_t)wakeupClamshellTimerExpired, 
+                        (thread_call_param_t) this);
+    diskSyncCalloutEntry = thread_call_allocate(
+                        &disk_sync_callout, 
+                        (thread_call_param_t) this);
 
     // create our parent
     patriarch = new IORootParent;
@@ -268,28 +360,32 @@ bool IOPMrootDomain::start ( IOService * nub )
     registerPrioritySleepWakeInterest( &sysPowerDownHandler, this, 0);
 
     // Register for a notification when IODisplayWrangler is published
-    _displayWranglerNotifier = addNotification( gIOPublishNotification, 
-                                                serviceMatching("IODisplayWrangler"), 
-                                                &displayWranglerPublished, this, 0);
+    _displayWranglerNotifier = addNotification( 
+                gIOPublishNotification, serviceMatching("IODisplayWrangler"), 
+                &displayWranglerPublished, this, 0);
 
-    _batteryLocationNotifier = addNotification( gIOPublishNotification, 
-                                                resourceMatching("battery"), 
-                                                &batteryLocationPublished, this, this);
+    // Battery location published - ApplePMU support only
+    _batteryPublishNotifier = addNotification( 
+                gIOPublishNotification, serviceMatching("IOPMPowerSource"), 
+                &batteryPublished, this, this);
+     
 
     const OSSymbol *ucClassName = OSSymbol::withCStringNoCopy("RootDomainUserClient");
     setProperty(gIOUserClientClassKey, (OSObject *) ucClassName);
     ucClassName->release();
 
-    IORegistryEntry     *temp_entry = NULL;
-    if( (temp_entry = IORegistryEntry::fromPath("mac-io/battery", gIODTPlane)) ||
-        (temp_entry = IORegistryEntry::fromPath("mac-io/via-pmu/battery", gIODTPlane)))
+    // IOBacklightDisplay can take a long time to load at boot, or it may
+    // not load at all if you're booting with clamshell closed. We publish
+    // 'DisplayDims' here redundantly to get it published early and at all.
+    psIterator = getMatchingServices( serviceMatching("IOPMPowerSource") );
+    if( psIterator && psIterator->getNextObject() )
     {
-        // If this machine has a battery, publish the fact that the backlight
-        //    supports dimming.
-        // Notice similar call in IOPMrootDomain::batteryLocationPublished() to 
-        //    detect batteries on SMU machines.
+        // There's at least one battery on the system, so we publish
+        // 'DisplayDims' support for the LCD.
         publishFeature("DisplayDims");
-        temp_entry->release();
+    }
+    if(psIterator) {
+        psIterator->release();
     }
 
     IOHibernateSystemInit(this);
@@ -299,17 +395,6 @@ bool IOPMrootDomain::start ( IOService * nub )
     return true;
 }
 
-IOReturn     IOPMrootDomain::setPMSetting(int type, OSNumber *n)
-{
-    if(_settingController && _settingController->func) {
-        int         seconds;
-        seconds = n->unsigned32BitValue();
-        return (*(_settingController->func))(type, seconds, _settingController->refcon);
-    } else {
-        return kIOReturnNoDevice;
-    }   
-}
-
 // **********************************************************************************
 // setProperties
 //
@@ -318,25 +403,29 @@ IOReturn     IOPMrootDomain::setPMSetting(int type, OSNumber *n)
 // **********************************************************************************
 IOReturn IOPMrootDomain::setProperties ( OSObject *props_obj)
 {
-    IOReturn                            return_value = kIOReturnSuccess;
-    OSDictionary                        *dict = OSDynamicCast(OSDictionary, props_obj);
-    OSBoolean                           *b;
-    OSNumber                            *n;
-    OSString                            *str;
-    const OSSymbol                      *boot_complete_string = OSSymbol::withCString("System Boot Complete");
-    const OSSymbol                            *power_button_string = OSSymbol::withCString("DisablePowerButtonSleep");
-    const OSSymbol                            *stall_halt_string = OSSymbol::withCString("StallSystemAtHalt");
-    const OSSymbol                            *auto_wake_string = OSSymbol::withCString("wake");
-    const OSSymbol                            *auto_power_string = OSSymbol::withCString("poweron");
-    const OSSymbol                            *wakeonring_string = OSSymbol::withCString("WakeOnRing");
-    const OSSymbol                            *fileserver_string = OSSymbol::withCString("AutoRestartOnPowerLoss");
-    const OSSymbol                            *wakeonlid_string = OSSymbol::withCString("WakeOnLid");
-    const OSSymbol                            *wakeonac_string = OSSymbol::withCString("WakeOnACChange");
-    const OSSymbol                            *hibernatemode_string = OSSymbol::withCString(kIOHibernateModeKey);
-    const OSSymbol                            *hibernatefile_string = OSSymbol::withCString(kIOHibernateFileKey);
-    const OSSymbol                            *hibernatefreeratio_string = OSSymbol::withCString(kIOHibernateFreeRatioKey);
-    const OSSymbol                            *hibernatefreetime_string = OSSymbol::withCString(kIOHibernateFreeTimeKey);
-    const OSSymbol                            *timezone_string = OSSymbol::withCString("TimeZoneOffsetSeconds");
+    IOReturn                        return_value = kIOReturnSuccess;
+    OSDictionary                    *dict = OSDynamicCast(OSDictionary, props_obj);
+    OSBoolean                       *b;
+    OSNumber                        *n;
+    OSString                        *str;
+    OSSymbol                        *type;
+    OSObject                        *obj;
+    unsigned int                    i;
+
+    const OSSymbol *boot_complete_string = 
+                OSSymbol::withCString("System Boot Complete");
+    const OSSymbol *sys_shutdown_string = 
+                OSSymbol::withCString("System Shutdown");
+    const OSSymbol *stall_halt_string = 
+                OSSymbol::withCString("StallSystemAtHalt");
+    const OSSymbol *hibernatemode_string = 
+                OSSymbol::withCString(kIOHibernateModeKey);
+    const OSSymbol *hibernatefile_string = 
+                OSSymbol::withCString(kIOHibernateFileKey);
+    const OSSymbol *hibernatefreeratio_string = 
+                OSSymbol::withCString(kIOHibernateFreeRatioKey);
+    const OSSymbol *hibernatefreetime_string = 
+                OSSymbol::withCString(kIOHibernateFreeTimeKey);
     
     if(!dict) 
     {
@@ -350,14 +439,42 @@ IOReturn IOPMrootDomain::setProperties ( OSObject *props_obj)
     {
         systemBooting = false;
         adjustPowerState();
+
+        // If lid is closed, re-send lid closed notification
+        // now that booting is complete.
+        if( clamshellIsClosed )
+        {
+            this->receivePowerNotification(kLocalEvalClamshellCommand);
+        }
     }
     
-    if( power_button_string
-        && (b = OSDynamicCast(OSBoolean, dict->getObject(power_button_string))) ) 
+    if( sys_shutdown_string 
+        && (b = OSDynamicCast(OSBoolean, dict->getObject(sys_shutdown_string)))) 
     {
-        setProperty(power_button_string, b);
+    
+        if(kOSBooleanTrue == b)
+        {
+            /* We set systemShutdown = true during shutdown
+               to prevent sleep at unexpected times while loginwindow is trying
+               to shutdown apps and while the OS is trying to transition to
+               complete power of.
+               
+               Set to true during shutdown, as soon as loginwindow shows
+               the "shutdown countdown dialog", through individual app
+               termination, and through black screen kernel shutdown.
+             */
+             kprintf("systemShutdown true\n");
+            systemShutdown = true;
+        } else {
+            /*
+             A shutdown was initiated, but then the shutdown
+             was cancelled, clearing systemShutdown to false here.
+            */
+            kprintf("systemShutdown false\n");
+            systemShutdown = false;            
+        }        
     }
-
+    
     if( stall_halt_string
         && (b = OSDynamicCast(OSBoolean, dict->getObject(stall_halt_string))) ) 
     {
@@ -365,94 +482,43 @@ IOReturn IOPMrootDomain::setProperties ( OSObject *props_obj)
     }
 
     if ( hibernatemode_string
-	&& (n = OSDynamicCast(OSNumber, dict->getObject(hibernatemode_string))))
+    && (n = OSDynamicCast(OSNumber, dict->getObject(hibernatemode_string))))
     {
-	setProperty(hibernatemode_string, n);
+    	setProperty(hibernatemode_string, n);
     }
     if ( hibernatefreeratio_string
-	&& (n = OSDynamicCast(OSNumber, dict->getObject(hibernatefreeratio_string))))
+    && (n = OSDynamicCast(OSNumber, dict->getObject(hibernatefreeratio_string))))
     {
-	setProperty(hibernatefreeratio_string, n);
+        setProperty(hibernatefreeratio_string, n);
     }
     if ( hibernatefreetime_string
-	&& (n = OSDynamicCast(OSNumber, dict->getObject(hibernatefreetime_string))))
+    && (n = OSDynamicCast(OSNumber, dict->getObject(hibernatefreetime_string))))
     {
-	setProperty(hibernatefreetime_string, n);
+        setProperty(hibernatefreetime_string, n);
     }
     if ( hibernatefile_string
-	&& (str = OSDynamicCast(OSString, dict->getObject(hibernatefile_string))))
+    && (str = OSDynamicCast(OSString, dict->getObject(hibernatefile_string))))
     {
-	setProperty(hibernatefile_string, str);
+        setProperty(hibernatefile_string, str);
     }
 
-    // Relay AutoWake setting to its controller
-    if( auto_wake_string
-        && (n = OSDynamicCast(OSNumber, dict->getObject(auto_wake_string))) )
-    {
-        return_value = setPMSetting(kIOPMAutoWakeSetting, n);
-        if(kIOReturnSuccess != return_value) goto exit;
-    }
+    // Relay our allowed PM settings onto our registered PM clients
+    for(i = 0; i < allowedPMSettings->getCount(); i++) {
 
-    // Relay AutoPower setting to its controller
-    if( auto_power_string
-        && (n = OSDynamicCast(OSNumber, dict->getObject(auto_power_string))) )
-    {
-        return_value = setPMSetting(kIOPMAutoPowerOnSetting, n);
-        if(kIOReturnSuccess != return_value) goto exit;
-    }
+        type = (OSSymbol *)allowedPMSettings->getObject(i);
+        if(!type) continue;
 
-    // Relay WakeOnRing setting to its controller
-    if( wakeonring_string
-        && (n = OSDynamicCast(OSNumber, dict->getObject(wakeonring_string))) )
-    {
-        return_value = setPMSetting(kIOPMWakeOnRingSetting, n);
-        if(kIOReturnSuccess != return_value) goto exit;
-    }
-
-    // Relay FileServer setting to its controller
-    if( fileserver_string
-        && (n = OSDynamicCast(OSNumber, dict->getObject(fileserver_string))) )
-    {
-        return_value = setPMSetting(kIOPMAutoRestartOnPowerLossSetting, n);
-        if(kIOReturnSuccess != return_value) goto exit;
-    }
-
-    // Relay WakeOnLid setting to its controller
-    if( wakeonlid_string 
-        && (n = OSDynamicCast(OSNumber, dict->getObject(wakeonlid_string))) )
-    {
-        return_value = setPMSetting(kIOPMWakeOnLidSetting, n);
-        if(kIOReturnSuccess != return_value) goto exit;
-    }
-    
-    // Relay WakeOnACChange setting to its controller
-    if( wakeonac_string
-        && (n = OSDynamicCast(OSNumber, dict->getObject(wakeonac_string))) )
-    {
-        return_value = setPMSetting(kIOPMWakeOnACChangeSetting, n);
-        if(kIOReturnSuccess != return_value) goto exit;
-    }
-
-    // Relay timezone offset in seconds to SMU
-    if( timezone_string
-        && (n = OSDynamicCast(OSNumber, dict->getObject(timezone_string))) )
-    {
-        return_value = setPMSetting(kIOPMTimeZoneSetting, n);
+        obj = dict->getObject(type);
+        if(!obj) continue;
+        
+        return_value = setPMSetting(type, obj);
+        
         if(kIOReturnSuccess != return_value) goto exit;
     }
 
-
     exit:
     if(boot_complete_string) boot_complete_string->release();
-    if(power_button_string) power_button_string->release();
     if(stall_halt_string) stall_halt_string->release();
-    if(auto_wake_string) auto_wake_string->release();
-    if(auto_power_string) auto_power_string->release();
-    if(wakeonring_string) wakeonring_string->release();
-    if(fileserver_string) fileserver_string->release();
-    if(wakeonlid_string) wakeonlid_string->release();
-    if(wakeonac_string) wakeonac_string->release();
-    if(timezone_string) timezone_string->release();
     return return_value;
 }
 
@@ -567,13 +633,12 @@ void IOPMrootDomain::handleSleepTimerExpiration ( void )
 
 void IOPMrootDomain::stopIgnoringClamshellEventsDuringWakeup(void)
 {
-    OSObject *  state;
-
     // Allow clamshell-induced sleep now
     ignoringClamshellDuringWakeup = false;
 
-    if ((state = getProperty(kAppleClamshellStateKey)))
-        publishResource(kAppleClamshellStateKey, state);
+    // Re-send clamshell event, in case it causes a sleep
+    if(clamshellIsClosed) 
+        this->receivePowerNotification( kLocalEvalClamshellCommand );
 }
 
 //*********************************************************************************
@@ -617,17 +682,20 @@ IOReturn IOPMrootDomain::setAggressiveness ( unsigned long type, unsigned long n
 // **********************************************************************************
 IOReturn IOPMrootDomain::sleepSystem ( void )
 {
-    //kprintf("sleep demand received\n");
-    if ( !systemBooting && allowSleep && sleepIsSupported ) {
+    if(systemShutdown) {
+        kprintf("Preventing system sleep on grounds of systemShutdown.\n");
+    }
+
+    if ( !systemBooting && !systemShutdown && allowSleep && sleepIsSupported ) {
         patriarch->sleepSystem();
 
         return kIOReturnSuccess;
     }
-    if ( !systemBooting && allowSleep && !sleepIsSupported ) {
+    if ( !systemBooting && !systemShutdown && allowSleep && !sleepIsSupported ) {
         patriarch->dozeSystem();
         return kIOReturnSuccess;
     }
-    return kIOReturnSuccess;
+    return kIOReturnError;
 }
 
 
@@ -688,7 +756,7 @@ void IOPMrootDomain::powerChangeDone ( unsigned long previousState )
                 // code will resume execution here.
 
                 // Now we're waking...
-		IOHibernateSystemWake();
+                IOHibernateSystemWake();
 
                 // stay awake for at least 30 seconds
                 clock_interval_to_deadline(30, kSecondScale, &deadline);	
@@ -722,7 +790,9 @@ void IOPMrootDomain::powerChangeDone ( unsigned long previousState )
                 if(getProperty(kIOREMSleepEnabledKey)) {
                     // clamshellWakeupIgnore callout clears ignoreClamshellDuringWakeup bit   
                     clock_interval_to_deadline(5, kSecondScale, &deadline);
-                    if(clamshellWakeupIgnore) thread_call_enter_delayed(clamshellWakeupIgnore, deadline);
+                    if(clamshellWakeupIgnore)  {
+                        thread_call_enter_delayed(clamshellWakeupIgnore, deadline);
+                    }
                 } else ignoringClamshellDuringWakeup = false;
                 
                 // Find out what woke us
@@ -787,8 +857,11 @@ void IOPMrootDomain::wakeFromDoze( void )
 {
     if ( pm_vars->myCurrentState == DOZE_STATE ) 
     {
-        // reset this till next attempt
-        canSleep = true;
+        // Reset sleep support till next sleep attempt.
+        // A machine's support of sleep vs. doze can change over the course of
+        // a running system, so we recalculate it before every sleep.
+        setSleepSupported(0);
+
         powerOverrideOffPriv();
 
         // early wake notification
@@ -800,43 +873,263 @@ void IOPMrootDomain::wakeFromDoze( void )
 }
 
 
-// **********************************************************************************
+// *****************************************************************************
 // publishFeature
 //
 // Adds a new feature to the supported features dictionary
 // 
 // 
-// **********************************************************************************
+// *****************************************************************************
 void IOPMrootDomain::publishFeature( const char * feature )
 {
+    publishFeature(feature, kIOPMSupportedOnAC 
+                                  | kIOPMSupportedOnBatt 
+                                  | kIOPMSupportedOnUPS,
+                            NULL);
+    return;
+}
+
+
+// *****************************************************************************
+// publishFeature (with supported power source specified)
+//
+// Adds a new feature to the supported features dictionary
+// 
+// 
+// *****************************************************************************
+void IOPMrootDomain::publishFeature( 
+    const char *feature, 
+    uint32_t supportedWhere,
+    uint32_t *uniqueFeatureID)
+{
+    static uint16_t     next_feature_id = 500;
+
+    OSNumber            *new_feature_data = NULL;
+    OSNumber            *existing_feature = NULL;
+    OSArray             *existing_feature_arr = NULL;
+    OSObject            *osObj = NULL;
+    uint32_t            feature_value = 0;
+
+    supportedWhere &= kRD_AllPowerSources; // mask off any craziness!
+
+    if(!supportedWhere) {
+        // Feature isn't supported anywhere!
+        return;
+    }
+    
+    if(next_feature_id > 5000) {
+        // Far, far too many features!
+        return;
+    }
+
     if(featuresDictLock) IOLockLock(featuresDictLock);
+
     OSDictionary *features =
         (OSDictionary *) getProperty(kRootDomainSupportedFeatures);
     
-    if ( features && OSDynamicCast(OSDictionary, features))
+    // Create new features dict if necessary
+    if ( features && OSDynamicCast(OSDictionary, features)) {
         features = OSDictionary::withDictionary(features);
-    else
+    } else {
         features = OSDictionary::withCapacity(1);
+    }
+    
+    // Create OSNumber to track new feature
+    
+    next_feature_id += 1;
+    if( uniqueFeatureID ) {
+        // We don't really mind if the calling kext didn't give us a place
+        // to stash their unique id. Many kexts don't plan to unload, and thus
+        // have no need to remove themselves later.
+        *uniqueFeatureID = next_feature_id;
+    }
+    
+    feature_value = supportedWhere + (next_feature_id << 16);
+    new_feature_data = OSNumber::withNumber(
+                                (unsigned long long)feature_value, 32);
+
+    // Does features object already exist?
+    if( (osObj = features->getObject(feature)) )
+    {
+        if(( existing_feature = OSDynamicCast(OSNumber, osObj) ))
+        {
+            // We need to create an OSArray to hold the now 2 elements.
+            existing_feature_arr = OSArray::withObjects(
+                            (const OSObject **)&existing_feature, 1, 2);
+            existing_feature_arr->setObject(new_feature_data);
+            features->setObject(feature, existing_feature_arr);
+        } else if(( existing_feature_arr = OSDynamicCast(OSArray, osObj) ))
+        {
+            // Add object to existing array
+            existing_feature_arr->setObject(new_feature_data);        
+        }
+    } else {
+        // The easy case: no previously existing features listed. We simply
+        // set the OSNumber at key 'feature' and we're on our way.
+        features->setObject(feature, new_feature_data);        
+    }
+    
+    new_feature_data->release();
 
-    features->setObject(feature, kOSBooleanTrue);
     setProperty(kRootDomainSupportedFeatures, features);
+
     features->release();
-    if(featuresDictLock) IOLockUnlock(featuresDictLock);
+
+    // Notify EnergySaver and all those in user space so they might
+    // re-populate their feature specific UI
+    messageClients(kIOPMMessageFeatureChange, this);
+
+    if(featuresDictLock) IOLockUnlock(featuresDictLock);    
 }
 
+// *****************************************************************************
+// removePublishedFeature
+//
+// Removes previously published feature
+// 
+// 
+// *****************************************************************************
+IOReturn IOPMrootDomain::removePublishedFeature( uint32_t removeFeatureID )
+{
+    IOReturn                ret = kIOReturnError;
+    uint32_t                feature_value = 0;
+    uint16_t                feature_id = 0;
+    bool                    madeAChange = false;
+    
+    OSSymbol                *dictKey = NULL;
+    OSCollectionIterator    *dictIterator = NULL;
+    OSArray                 *arrayMember  = NULL;
+    OSNumber                *numberMember = NULL;
+    OSObject                *osObj        = NULL;
+    OSNumber                *osNum        = NULL;
+
+    if(featuresDictLock) IOLockLock(featuresDictLock);
 
+    OSDictionary *features =
+        (OSDictionary *) getProperty(kRootDomainSupportedFeatures);
+    
+    if ( features && OSDynamicCast(OSDictionary, features) )
+    {
+        // Any modifications to the dictionary are made to the copy to prevent
+        // races & crashes with userland clients. Dictionary updated
+        // automically later.
+        features = OSDictionary::withDictionary(features);
+    } else {
+        features = NULL;
+        ret = kIOReturnNotFound;
+        goto exit;
+    }
+    
+    // We iterate 'features' dictionary looking for an entry tagged
+    // with 'removeFeatureID'. If found, we remove it from our tracking
+    // structures and notify the OS via a general interest message.
+    
+    dictIterator = OSCollectionIterator::withCollection(features);
+    if(!dictIterator) {
+        goto exit;
+    }
+    
+    while( (dictKey = OSDynamicCast(OSSymbol, dictIterator->getNextObject())) )
+    {
+        osObj = features->getObject(dictKey);
+        
+        // Each Feature is either tracked by an OSNumber
+        if( osObj && (numberMember = OSDynamicCast(OSNumber, osObj)) )
+        {
+            feature_value = numberMember->unsigned32BitValue();
+            feature_id = (uint16_t)(feature_value >> 16);
+
+            if( feature_id == (uint16_t)removeFeatureID )
+            {
+                // Remove this node
+                features->removeObject(dictKey);
+                madeAChange = true;
+                break;
+            }
+        
+        // Or tracked by an OSArray of OSNumbers
+        } else if( osObj && (arrayMember = OSDynamicCast(OSArray, osObj)) )
+        {
+            unsigned int arrayCount = arrayMember->getCount();
+            
+            for(unsigned int i=0; i<arrayCount; i++)
+            {
+                osNum = OSDynamicCast(OSNumber, arrayMember->getObject(i));
+                if(!osNum) {
+                    continue;
+                }
+                
+                feature_value = osNum->unsigned32BitValue();
+                feature_id = (uint16_t)(feature_value >> 16);
+
+                if( feature_id == (uint16_t)removeFeatureID )
+                {
+                    // Remove this node
+                    if( 1 == arrayCount ) {
+                        // If the array only contains one element, remove
+                        // the whole thing.
+                        features->removeObject(dictKey);
+                    } else {
+                        // Otherwise just remove the element in question.
+                        arrayMember->removeObject(i);                    
+                    }
+
+                    madeAChange = true;
+                    break;
+                }
+            }
+        }    
+    }
+    
+    
+    dictIterator->release();
+    
+    if( madeAChange )
+    {
+        ret = kIOReturnSuccess;    
+
+        setProperty(kRootDomainSupportedFeatures, features);
+    
+        // Notify EnergySaver and all those in user space so they might
+        // re-populate their feature specific UI
+        messageClients(kIOPMMessageFeatureChange, this);
+    } else {
+        ret = kIOReturnNotFound;
+    }
+    
+exit:
+    if(features)    features->release();
+    if(featuresDictLock) IOLockUnlock(featuresDictLock);    
+    return ret;
+}
+
+
+// **********************************************************************************
+// unIdleDevice
+//
+// Enqueues unidle event to be performed later in a serialized context.
+// 
+// **********************************************************************************
 void IOPMrootDomain::unIdleDevice( IOService *theDevice, unsigned long theState )
 {
     if(pmPowerStateQueue)
         pmPowerStateQueue->unIdleOccurred(theDevice, theState);
 }
 
+// **********************************************************************************
+// announcePowerSourceChange
+//
+// Notifies "interested parties" that the batteries have changed state
+// 
+// **********************************************************************************
 void IOPMrootDomain::announcePowerSourceChange( void )
 {
-    IORegistryEntry                 *_batteryRegEntry = (IORegistryEntry *) getProperty("BatteryEntry");
+    IORegistryEntry *_batteryRegEntry = (IORegistryEntry *) getProperty("BatteryEntry");
+
+    // (if possible) re-publish power source state under IOPMrootDomain;
+    // only do so if the battery controller publishes an IOResource 
+    // defining battery location. Called from ApplePMU battery driver.
 
-    // (if possible) re-publish power source state under IOPMrootDomain
-    // (only done if the battery controller publishes an IOResource defining battery location)
     if(_batteryRegEntry)
     {
         OSArray             *batt_info;
@@ -845,57 +1138,236 @@ void IOPMrootDomain::announcePowerSourceChange( void )
             setProperty(kIOBatteryInfoKey, batt_info);
     }
 
-    messageClients(kIOPMMessageBatteryStatusHasChanged);
 }
 
-IOReturn IOPMrootDomain::registerPMSettingController
-        (IOPMSettingControllerCallback func, void *info)
+
+// *****************************************************************************
+// setPMSetting (private)
+//
+// Internal helper to relay PM settings changes from user space to individual
+// drivers. Should be called only by IOPMrootDomain::setProperties.
+// 
+// *****************************************************************************
+IOReturn     IOPMrootDomain::setPMSetting(
+    const OSSymbol *type, 
+    OSObject *obj)
 {
-    if(_settingController) return kIOReturnExclusiveAccess;
-    
-    _settingController = (PMSettingCtrl *)IOMalloc(sizeof(PMSettingCtrl));
-    if(!_settingController) return kIOReturnNoMemory;
+    OSArray             *arr = NULL;
+    PMSettingObject     *p_obj = NULL;
+    int                 count;
+    int                 i;
+
+    if(NULL == type) return kIOReturnBadArgument;
+
+    IORecursiveLockLock(settingsCtrlLock);
     
-    _settingController->func = func;
-    _settingController->refcon = info;
+    fPMSettingsDict->setObject(type, obj);
+
+    arr = (OSArray *)settingsCallbacks->getObject(type);
+    if(NULL == arr) goto exit;
+    count = arr->getCount();
+    for(i=0; i<count; i++) {
+        p_obj = (PMSettingObject *)OSDynamicCast(PMSettingObject, arr->getObject(i));
+        if(p_obj) p_obj->setPMSetting(type, obj);
+    }
+
+exit:
+    IORecursiveLockUnlock(settingsCtrlLock);
     return kIOReturnSuccess;
 }
 
+// *****************************************************************************
+// copyPMSetting (public)
+//
+// Allows kexts to safely read setting values, without being subscribed to
+// notifications.
+// 
+// *****************************************************************************
+OSObject * IOPMrootDomain::copyPMSetting(
+    OSSymbol *whichSetting)
+{
+    OSObject *obj = NULL;
 
-//*********************************************************************************
-// receivePowerNotification
+    if(!whichSetting) return NULL;
+
+    IORecursiveLockLock(settingsCtrlLock);
+    obj = fPMSettingsDict->getObject(whichSetting);
+    if(obj) {
+        obj->retain();
+    }
+    IORecursiveLockUnlock(settingsCtrlLock);
+    
+    return obj;
+}
+
+// *****************************************************************************
+// registerPMSettingController (public)
 //
-// The power controller is notifying us of a hardware-related power management
-// event that we must handle. This is a result of an 'environment' interrupt from
-// the power mgt micro.
-//*********************************************************************************
+// direct wrapper to registerPMSettingController with uint32_t power source arg
+// *****************************************************************************
+IOReturn IOPMrootDomain::registerPMSettingController(
+    const OSSymbol *                settings[],
+    IOPMSettingControllerCallback   func,
+    OSObject                        *target,
+    uintptr_t                       refcon,
+    OSObject                        **handle)
+{
+    return registerPMSettingController( 
+            settings,
+            (kIOPMSupportedOnAC | kIOPMSupportedOnBatt | kIOPMSupportedOnUPS),
+            func, target, refcon, handle);
+}
 
-IOReturn IOPMrootDomain::receivePowerNotification (UInt32 msg)
+// *****************************************************************************
+// registerPMSettingController (public)
+//
+// Kexts may register for notifications when a particular setting is changed.
+// A list of settings is available in IOPM.h.
+// Arguments:
+//  * settings - An OSArray containing OSSymbols. Caller should populate this
+//          array with a list of settings caller wants notifications from.
+//  * func - A C function callback of the type IOPMSettingControllerCallback
+//  * target - caller may provide an OSObject *, which PM will pass as an 
+//          target to calls to "func"
+//  * refcon - caller may provide an void *, which PM will pass as an 
+//          argument to calls to "func"
+//  * handle - This is a return argument. We will populate this pointer upon
+//          call success. Hold onto this and pass this argument to
+//          IOPMrootDomain::deRegisterPMSettingCallback when unloading your kext
+// Returns:
+//      kIOReturnSuccess on success
+// *****************************************************************************
+IOReturn IOPMrootDomain::registerPMSettingController(
+    const OSSymbol *                settings[],
+    uint32_t                        supportedPowerSources,
+    IOPMSettingControllerCallback   func,
+    OSObject                        *target,
+    uintptr_t                       refcon,
+    OSObject                        **handle)
 {
-    if (msg & kIOPMOverTemp) 
+    PMSettingObject     *pmso = NULL;
+    OSArray             *list = NULL;
+    IOReturn            ret = kIOReturnSuccess;
+    int                 i;
+
+    if( NULL == settings ||
+        NULL == func ||
+        NULL == handle)
     {
-        IOLog("Power Management received emergency overtemp signal. Going to sleep.");
-        (void) sleepSystem ();
+        return kIOReturnBadArgument;
     }
-    if (msg & kIOPMSetDesktopMode) 
+
+
+    pmso = PMSettingObject::pmSettingObject(
+                (IOPMrootDomain *)this, func, target, 
+                refcon, supportedPowerSources, settings);
+
+    if(!pmso) {
+        ret = kIOReturnInternalError;
+        goto bail_no_unlock;
+    }
+
+    IORecursiveLockLock(settingsCtrlLock);
+    for(i=0; settings[i]; i++) 
     {
-        desktopMode = (0 != (msg & kIOPMSetValue));
-        msg &= ~(kIOPMSetDesktopMode | kIOPMSetValue);
+        list = (OSArray *)settingsCallbacks->getObject(settings[i]);
+        if(!list) {
+            // New array of callbacks for this setting
+            list = OSArray::withCapacity(1);
+            settingsCallbacks->setObject(settings[i], list);
+            list->release();
+        }
+
+        // Add caller to the callback list
+        list->setObject(pmso);
     }
-    if (msg & kIOPMSetACAdaptorConnected) 
+
+    ret = kIOReturnSuccess;
+
+    // Track this instance by its OSData ptr from now on  
+    *handle = pmso;
+    
+    IORecursiveLockUnlock(settingsCtrlLock);
+
+bail_no_unlock:
+    if(kIOReturnSuccess != ret) 
     {
-        acAdaptorConnect = (0 != (msg & kIOPMSetValue));
-        msg &= ~(kIOPMSetACAdaptorConnected | kIOPMSetValue);
+        // Error return case
+        if(pmso) pmso->release();
+        if(handle) *handle = NULL;
     }
-    if (msg & kIOPMEnableClamshell) 
+    return ret;
+}
+
+//******************************************************************************
+// sleepOnClamshellClosed
+//
+// contains the logic to determine if the system should sleep when the clamshell
+// is closed.
+//******************************************************************************
+
+bool IOPMrootDomain::shouldSleepOnClamshellClosed ( void )
+{
+    return ( !ignoringClamshell 
+          && !ignoringClamshellDuringWakeup 
+          && !(desktopMode && acAdaptorConnect) );
+}
+
+void IOPMrootDomain::sendClientClamshellNotification ( void )
+{
+    /* Only broadcast clamshell alert if clamshell exists. */
+    if(!clamshellExists)
+        return;
+        
+    setProperty(kAppleClamshellStateKey, 
+                    clamshellIsClosed ? kOSBooleanTrue : kOSBooleanFalse);
+
+    setProperty(kAppleClamshellCausesSleepKey, 
+       shouldSleepOnClamshellClosed() ? kOSBooleanTrue : kOSBooleanFalse);
+
+
+    /* Argument to message is a bitfield of 
+     *      ( kClamshellStateBit | kClamshellSleepBit )
+     * Carry on the clamshell state change notification from an 
+     * independent thread.
+     */
+    pmArbiter->clamshellStateChangeOccurred(
+            (uint32_t) ( (clamshellIsClosed ? kClamshellStateBit : 0)
+             | ( shouldSleepOnClamshellClosed() ? kClamshellSleepBit : 0)));
+}
+
+//******************************************************************************
+// receivePowerNotification
+//
+// The power controller is notifying us of a hardware-related power management
+// event that we must handle. This is a result of an 'environment' interrupt from
+// the power mgt micro.
+//******************************************************************************
+
+IOReturn IOPMrootDomain::receivePowerNotification (UInt32 msg)
+{
+    bool        eval_clamshell = false;
+
+    /*
+     * Local (IOPMrootDomain only) eval clamshell command
+     */
+    if (msg & kLocalEvalClamshellCommand)
     {
-        ignoringClamshell = false;
+        eval_clamshell = true;
     }
-    if (msg & kIOPMDisableClamshell) 
+
+    /*
+     * Overtemp
+     */
+    if (msg & kIOPMOverTemp) 
     {
-        ignoringClamshell = true;
+        IOLog("PowerManagement emergency overtemp signal. Going to sleep!");
+        (void) sleepSystem ();
     }
 
+    /*
+     * PMU Processor Speed Change
+     */
     if (msg & kIOPMProcessorSpeedChange) 
     {
         IOService *pmu = waitForService(serviceMatching("ApplePMU"));
@@ -904,26 +1376,138 @@ IOReturn IOPMrootDomain::receivePowerNotification (UInt32 msg)
         pmu->callPlatformFunction("recoverFromSleep", false, 0, 0, 0, 0);
     }
 
+    /*
+     * Sleep Now!
+     */
     if (msg & kIOPMSleepNow) 
     {
       (void) sleepSystem ();
     }
     
+    /*
+     * Power Emergency
+     */
     if (msg & kIOPMPowerEmergency) 
     {
       (void) sleepSystem ();
     }
 
-    if (msg & kIOPMClamshellClosed) 
+    
+    /*
+     * Clamshell OPEN
+     */
+    if (msg & kIOPMClamshellOpened) 
+    {
+        // Received clamshel open message from clamshell controlling driver
+        // Update our internal state and tell general interest clients
+        clamshellIsClosed = false;
+        clamshellExists = true;
+                
+        sendClientClamshellNotification();
+    } 
+
+    /* 
+     * Clamshell CLOSED
+     * Send the clamshell interest notification since the lid is closing. 
+     */
+    if (msg & kIOPMClamshellClosed)
     {
-        if ( !ignoringClamshell && !ignoringClamshellDuringWakeup 
-                    && (!desktopMode || !acAdaptorConnect) ) 
+        // Received clamshel open message from clamshell controlling driver
+        // Update our internal state and tell general interest clients
+        clamshellIsClosed = true;
+        clamshellExists = true;
+
+        sendClientClamshellNotification();
+        
+        // And set eval_clamshell = so we can attempt 
+        eval_clamshell = true;
+    }
+
+    /*
+     * Set Desktop mode (sent from graphics)
+     *
+     *  -> reevaluate lid state
+     */
+    if (msg & kIOPMSetDesktopMode) 
+    {
+        desktopMode = (0 != (msg & kIOPMSetValue));
+        msg &= ~(kIOPMSetDesktopMode | kIOPMSetValue);
+
+        sendClientClamshellNotification();
+
+        // Re-evaluate the lid state
+        if( clamshellIsClosed )
         {
+            eval_clamshell = true;
+        }
+    }
+    
+    /*
+     * AC Adaptor connected
+     *
+     *  -> reevaluate lid state
+     */
+    if (msg & kIOPMSetACAdaptorConnected) 
+    {
+        acAdaptorConnect = (0 != (msg & kIOPMSetValue));
+        msg &= ~(kIOPMSetACAdaptorConnected | kIOPMSetValue);
 
-             (void) sleepSystem ();
+        sendClientClamshellNotification();
+
+        // Re-evaluate the lid state
+        if( clamshellIsClosed )
+        {
+            eval_clamshell = true;
         }
+
     }
+    
+    /*
+     * Enable Clamshell (external display disappear)
+     *
+     *  -> reevaluate lid state
+     */
+    if (msg & kIOPMEnableClamshell) 
+    {
+        // Re-evaluate the lid state
+        // System should sleep on external display disappearance
+        // in lid closed operation.
+        if( clamshellIsClosed && (true == ignoringClamshell) )        
+        {
+            eval_clamshell = true;
+        }
 
+        ignoringClamshell = false;
+
+        sendClientClamshellNotification();
+    }
+    
+    /*
+     * Disable Clamshell (external display appeared)
+     * We don't bother re-evaluating clamshell state. If the system is awake,
+     * the lid is probably open. 
+     */
+    if (msg & kIOPMDisableClamshell) 
+    {
+        ignoringClamshell = true;
+
+        sendClientClamshellNotification();
+    }
+
+    /*
+     * Evaluate clamshell and SLEEP if appropiate
+     */
+    if ( eval_clamshell && shouldSleepOnClamshellClosed() ) 
+    {
+
+
+        // SLEEP!
+        sleepSystem();
+    }
+
+    /*
+     * Power Button
+     */
     if (msg & kIOPMPowerButton) 
     {				
         // toggle state of sleep/wake
@@ -936,23 +1520,29 @@ IOReturn IOPMrootDomain::receivePowerNotification (UInt32 msg)
             reportUserInput();					
         }
         else {
+            OSString *pbs = OSString::withCString("DisablePowerButtonSleep");
             // Check that power button sleep is enabled
-            if(kOSBooleanTrue != getProperty(OSString::withCString("DisablePowerButtonSleep")))
+            if( pbs ) {
+                if( kOSBooleanTrue != getProperty(pbs))
                 sleepSystem();
+            }
         }
     }
 
-    // if the case has been closed, we allow
-    // the machine to be put to sleep or to idle sleep
-
+    /*
+     * Allow Sleep
+     *
+     */
     if ( (msg & kIOPMAllowSleep) && !allowSleep ) 
     {
         allowSleep = true;
         adjustPowerState();
     }
 
-    // if the case has been opened, we disallow sleep/doze
-
+    /*
+     * Prevent Sleep
+     *
+     */
     if (msg & kIOPMPreventSleep) {
         allowSleep = false;
 	    // are we dozing?
@@ -984,9 +1574,12 @@ void IOPMrootDomain::setSleepSupported( IOOptionBits flags )
     {
         canSleep = false;
     } else {
+        canSleep = true;
         platformSleepSupport = flags;
     }
 
+    setProperty(kIOSleepSupportedKey, canSleep);
+
 }
 
 //*********************************************************************************
@@ -1084,10 +1677,23 @@ bool IOPMrootDomain::tellChangeDown ( unsigned long stateNum )
     switch ( stateNum ) {
         case DOZE_STATE:
         case SLEEP_STATE:
+	
+            // Direct callout into OSMetaClass so it can disable kmod unloads
+            // during sleep/wake to prevent deadlocks.
+            OSMetaClassSystemSleepOrWake( kIOMessageSystemWillSleep );
+
             return super::tellClientsWithResponse(kIOMessageSystemWillSleep);
         case RESTART_STATE:
+            // Unsupported shutdown ordering hack on RESTART only
+            // For Bluetooth and USB (4368327)
+            super::tellClients(iokit_common_msg(0x759));
+
             return super::tellClientsWithResponse(kIOMessageSystemWillRestart);
         case OFF_STATE:
+            // Unsupported shutdown ordering hack on SHUTDOWN only
+            // For Bluetooth and USB (4554440)
+            super::tellClients(iokit_common_msg(0x749));
+
             return super::tellClientsWithResponse(kIOMessageSystemWillPowerOff);
     }
     // this shouldn't execute
@@ -1141,6 +1747,10 @@ void IOPMrootDomain::tellChangeUp ( unsigned long stateNum)
 {
     if ( stateNum == ON_STATE ) 
     {
+        // Direct callout into OSMetaClass so it can disable kmod unloads
+        // during sleep/wake to prevent deadlocks.
+        OSMetaClassSystemSleepOrWake( kIOMessageSystemHasPoweredOn );
+
 	IOHibernateSystemPostWake();
         return tellClients(kIOMessageSystemHasPoweredOn);
     }
@@ -1202,6 +1812,27 @@ IOReturn IOPMrootDomain::changePowerStateTo ( unsigned long ordinal )
 
 IOReturn IOPMrootDomain::changePowerStateToPriv ( unsigned long ordinal )
 {
+    IOReturn    ret;
+
+    if( (systemBooting || systemShutdown) && (ordinal == SLEEP_STATE) )
+    {
+        kprintf("DANGER DANGER DANGER unexpected code path. aborting SLEEPSTATE change.\n");
+        super::changePowerStateToPriv(ON_STATE);
+    }
+
+    if( (SLEEP_STATE == ordinal) && sleepSupportedPEFunction )
+    {
+
+        // Determine if the machine supports sleep, or must doze.
+        ret = getPlatform()->callPlatformFunction(
+                            sleepSupportedPEFunction, false,
+                            NULL, NULL, NULL, NULL);
+    
+        // If the machine only supports doze, the callPlatformFunction call
+        // boils down toIOPMrootDomain::setSleepSupported(kPCICantSleep), 
+        // otherwise nothing.
+    }
+
     return super::changePowerStateToPriv(ordinal);
 }
 
@@ -1361,10 +1992,13 @@ IOReturn IOPMrootDomain::displayWranglerNotification( void * target, void * refC
 //
 //*********************************************************************************
 
-bool IOPMrootDomain::displayWranglerPublished( void * target, void * refCon,
-                                    IOService * newService)
+bool IOPMrootDomain::displayWranglerPublished( 
+    void * target, 
+    void * refCon,
+    IOService * newService)
 {
-    IOPMrootDomain *                rootDomain = OSDynamicCast(IOPMrootDomain, (IOService *)target);
+    IOPMrootDomain *rootDomain = 
+            OSDynamicCast(IOPMrootDomain, (IOService *)target);
 
     if(!rootDomain)
         return false;
@@ -1372,40 +2006,35 @@ bool IOPMrootDomain::displayWranglerPublished( void * target, void * refCon,
     rootDomain->wrangler = newService;
     
     // we found the display wrangler, now install a handler
-    if( !rootDomain->wrangler->registerInterest( gIOGeneralInterest, &displayWranglerNotification, target, 0) ) {
-        IOLog("IOPMrootDomain::displayWranglerPublished registerInterest failed\n");
+    if( !rootDomain->wrangler->registerInterest( gIOGeneralInterest, 
+                            &displayWranglerNotification, target, 0) ) 
+    {
         return false;
     }
     
     return true;
 }
 
+
 //*********************************************************************************
-// batteryLocationPublished
+// batteryPublished
 //
-// Notification on AppleSMU publishing location of battery data
+// Notification on battery class IOPowerSource appearance
 //
-//*********************************************************************************
-
-bool IOPMrootDomain::batteryLocationPublished( void * target, void * root_domain,
-        IOService * resourceService )
-{
-    IORegistryEntry                     *battery_location;
-
-    battery_location = (IORegistryEntry *) resourceService->getProperty("battery");
-    if (!battery_location || !OSDynamicCast(IORegistryEntry, battery_location))
-        return (true);
-        
-    ((IOPMrootDomain *)root_domain)->setProperty("BatteryEntry", battery_location);
-    
-    // rdar://2936060
+//******************************************************************************
+
+bool IOPMrootDomain::batteryPublished( 
+    void * target, 
+    void * root_domain,
+    IOService * resourceService )
+{    
+    // rdar://2936060&4435589    
     // All laptops have dimmable LCD displays
     // All laptops have batteries
     // So if this machine has a battery, publish the fact that the backlight
     // supports dimming.
     ((IOPMrootDomain *)root_domain)->publishFeature("DisplayDims");
-    
-    ((IOPMrootDomain *)root_domain)->announcePowerSourceChange();
+
     return (true);
 }
 
@@ -1435,9 +2064,15 @@ bool IOPMrootDomain::batteryLocationPublished( void * target, void * root_domain
 
 void IOPMrootDomain::adjustPowerState( void )
 {
-    if ( (sleepSlider == 0) ||
-        ! allowSleep ||
-        systemBooting ) {
+    if ( (sleepSlider == 0) 
+        || !allowSleep 
+        || systemBooting 
+        || systemShutdown )
+    {
+        if(systemBooting || systemShutdown) {
+            kprintf("adjusting power state to ON_STATE [2063] on grounds of systemBooting.\n");
+        }
+        
         changePowerStateToPriv(ON_STATE);
     } else {
         if ( sleepASAP ) 
@@ -1453,6 +2088,117 @@ void IOPMrootDomain::adjustPowerState( void )
     }
 }
 
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+
+#undef super
+#define super OSObject
+OSDefineMetaClassAndStructors(PMSettingObject, OSObject)
+
+void PMSettingObject::setPMSetting(const OSSymbol *type, OSObject *obj)
+{
+        (*func)(target, type, obj, refcon);
+}
+
+/* 
+ * Static constructor/initializer for PMSettingObject
+ */
+PMSettingObject *PMSettingObject::pmSettingObject(
+    IOPMrootDomain                      *parent_arg,
+    IOPMSettingControllerCallback       handler_arg,
+    OSObject                            *target_arg,
+    uintptr_t                           refcon_arg,
+    uint32_t                            supportedPowerSources,
+    const OSSymbol *                    settings[])
+{
+    uint32_t                            objCount = 0;
+    PMSettingObject                     *pmso;
+
+    if( !parent_arg || !handler_arg || !settings ) return NULL;
+
+     // count OSSymbol entries in NULL terminated settings array
+    while( settings[objCount] ) {
+        objCount++;
+    }
+    if(0 == objCount) return NULL;
+
+    pmso = new PMSettingObject;
+    if(!pmso || !pmso->init()) return NULL;
+
+    pmso->parent = parent_arg;
+    pmso->func = handler_arg;
+    pmso->target = target_arg;
+    pmso->refcon = refcon_arg;
+    pmso->releaseAtCount = objCount + 1; // release when it has count+1 retains
+ 
+    pmso->publishedFeatureID = (uint32_t *)IOMalloc(sizeof(uint32_t)*objCount);
+    if(pmso->publishedFeatureID) {
+        for(unsigned int i=0; i<objCount; i++) {
+            // Since there is now at least one listener to this setting, publish
+            // PM root domain support for it.
+            parent_arg->publishFeature( settings[i]->getCStringNoCopy(), 
+                    supportedPowerSources, &pmso->publishedFeatureID[i] );
+        }
+    }
+    
+    return pmso;
+}
+
+void PMSettingObject::free(void)
+{
+    OSCollectionIterator    *settings_iter;
+    OSSymbol                *sym;
+    OSArray                 *arr;
+    int                     arr_idx;
+    int                     i;
+    int                     objCount = releaseAtCount - 1;
+    
+    if(publishedFeatureID) {
+        for(i=0; i<objCount; i++) {
+            if(0 != publishedFeatureID[i]) {
+                parent->removePublishedFeature( publishedFeatureID[i] );
+            }
+        }
+    
+        IOFree(publishedFeatureID, sizeof(uint32_t) * objCount);
+    }
+            
+    IORecursiveLockLock(parent->settingsCtrlLock);        
+    
+    // Search each PM settings array in the kernel.
+    settings_iter = OSCollectionIterator::withCollection(parent->settingsCallbacks);
+    if(settings_iter) 
+    {
+        while(( sym = OSDynamicCast(OSSymbol, settings_iter->getNextObject()) ))
+        {
+            arr = (OSArray *)parent->settingsCallbacks->getObject(sym);
+            arr_idx = arr->getNextIndexOfObject(this, 0);
+            if(-1 != arr_idx) {
+                // 'this' was found in the array; remove it                
+                arr->removeObject(arr_idx);
+            }
+        }
+    
+        settings_iter->release();
+    }
+    
+    IORecursiveLockUnlock(parent->settingsCtrlLock);
+    
+    super::free();
+}
+
+void PMSettingObject::taggedRelease(const void *tag, const int when) const
+{     
+    // We have n+1 retains - 1 per array that this PMSettingObject is a member
+    // of, and 1 retain to ourself. When we get a release with n+1 retains
+    // remaining, we go ahead and free ourselves, cleaning up array pointers
+    // in free();
+
+    super::taggedRelease(tag, releaseAtCount);    
+}
+
+
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
@@ -1532,3 +2278,23 @@ void IORootParent::wakeSystem ( void )
     changePowerStateToPriv(ON_STATE);
 }
 
+IOReturn IORootParent::changePowerStateToPriv ( unsigned long ordinal )
+{
+    IOReturn        ret;
+
+    if( (SLEEP_STATE == ordinal) && sleepSupportedPEFunction )
+    {
+
+        // Determine if the machine supports sleep, or must doze.
+        ret = getPlatform()->callPlatformFunction(
+                            sleepSupportedPEFunction, false,
+                            NULL, NULL, NULL, NULL);
+    
+        // If the machine only supports doze, the callPlatformFunction call
+        // boils down toIOPMrootDomain::setSleepSupported(kPCICantSleep), 
+        // otherwise nothing.
+    }
+
+    return super::changePowerStateToPriv(ordinal);
+}
+
diff --git a/iokit/Kernel/IOPlatformExpert.cpp b/iokit/Kernel/IOPlatformExpert.cpp
index f298bbd4d..d5b9f7f5b 100644
--- a/iokit/Kernel/IOPlatformExpert.cpp
+++ b/iokit/Kernel/IOPlatformExpert.cpp
@@ -51,6 +51,9 @@ extern "C" {
 #include <pexpert/pexpert.h>
 }
 
+/* Delay period for UPS halt */
+#define kUPSDelayHaltCPU_msec   (1000*60*5)
+
 void printDictionaryKeys (OSDictionary * inDictionary, char * inMsg);
 static void getCStringForObject (OSObject * inObj, char * outStr);
 
@@ -246,20 +249,18 @@ int (*PE_halt_restart)(unsigned int type) = 0;
 
 int IOPlatformExpert::haltRestart(unsigned int type)
 {
-  IOPMrootDomain *rd = getPMRootDomain();
-  OSBoolean   *b = 0;
-    
-  if(rd) b = (OSBoolean *)OSDynamicCast(OSBoolean, rd->getProperty(OSString::withCString("StallSystemAtHalt")));
-
   if (type == kPEHangCPU) while (1);
 
-  if (kOSBooleanTrue == b) {
-    // Stall shutdown for 5 minutes, and if no outside force has removed our power, continue with
-    // a reboot.
-    IOSleep(1000*60*5);
+  if (type == kPEUPSDelayHaltCPU) {
+    // Stall shutdown for 5 minutes, and if no outside force has 
+    // removed our power at that point, proceed with a reboot.
+    IOSleep( kUPSDelayHaltCPU_msec );
+
+    // Ideally we never reach this point.
+
     type = kPERestartCPU;
   }
-  
+  kprintf("platform halt restart\n");
   if (PE_halt_restart) return (*PE_halt_restart)(type);
   else return -1;
 }
@@ -730,15 +731,17 @@ static void getCStringForObject (OSObject * inObj, char * outStr)
    }
 }
 
-/* IOPMPanicOnShutdownHang
+/* IOShutdownNotificationsTimedOut
  * - Called from a timer installed by PEHaltRestart
  */
-static void IOPMPanicOnShutdownHang(thread_call_param_t p0, thread_call_param_t p1)
+static void IOShutdownNotificationsTimedOut(
+    thread_call_param_t p0, 
+    thread_call_param_t p1)
 {
     int type = (int)p0;
 
     /* 30 seconds has elapsed - resume shutdown */
-    gIOPlatform->haltRestart(type);
+    if(gIOPlatform) gIOPlatform->haltRestart(type);
 }
 
 
@@ -778,8 +781,9 @@ int PEHaltRestart(unsigned int type)
   bool              noWaitForResponses;
   AbsoluteTime      deadline;
   thread_call_t     shutdown_hang;
+  unsigned int      tell_type;
   
-  if(type == kPEHaltCPU || type == kPERestartCPU)
+  if(type == kPEHaltCPU || type == kPERestartCPU || type == kPEUPSDelayHaltCPU)
   {
     /* Notify IOKit PM clients of shutdown/restart
        Clients subscribe to this message with a call to
@@ -790,11 +794,19 @@ int PEHaltRestart(unsigned int type)
        If all goes well the machine will be off by the time
        the timer expires.
      */
-    shutdown_hang = thread_call_allocate( &IOPMPanicOnShutdownHang, (thread_call_param_t) type);
+    shutdown_hang = thread_call_allocate( &IOShutdownNotificationsTimedOut, 
+                        (thread_call_param_t) type);
     clock_interval_to_deadline( 30, kSecondScale, &deadline );
     thread_call_enter1_delayed( shutdown_hang, 0, deadline );
     
-    noWaitForResponses = pmRootDomain->tellChangeDown2(type); 
+
+    if( kPEUPSDelayHaltCPU == type ) {
+        tell_type = kPEHaltCPU;
+    } else {
+        tell_type = type;
+    }
+
+    noWaitForResponses = pmRootDomain->tellChangeDown2(tell_type); 
     /* This notification should have few clients who all do 
        their work synchronously.
              
@@ -818,16 +830,18 @@ UInt32 PESavePanicInfo(UInt8 *buffer, UInt32 length)
 
 long PEGetGMTTimeOfDay(void)
 {
+	long	result = 0;
+
     if( gIOPlatform)
-	return( gIOPlatform->getGMTTimeOfDay());
-    else
-	return( 0 );
+		result = gIOPlatform->getGMTTimeOfDay();
+
+	return (result);
 }
 
 void PESetGMTTimeOfDay(long secs)
 {
     if( gIOPlatform)
-	gIOPlatform->setGMTTimeOfDay(secs);
+		gIOPlatform->setGMTTimeOfDay(secs);
 }
 
 } /* extern "C" */
@@ -1220,6 +1234,12 @@ void IOPlatformExpertDevice::free()
         workLoop->release();
 }
 
+bool IOPlatformExpertDevice::attachToChild( IORegistryEntry * child,
+                                        const IORegistryPlane * plane )
+{
+    return IOService::attachToChild( child, plane );
+}
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #undef super
diff --git a/iokit/Kernel/IORegistryEntry.cpp b/iokit/Kernel/IORegistryEntry.cpp
index 0fd1f423b..16ee95a7d 100644
--- a/iokit/Kernel/IORegistryEntry.cpp
+++ b/iokit/Kernel/IORegistryEntry.cpp
@@ -93,9 +93,9 @@ OSDefineMetaClassAndStructors(IORegistryPlane, OSObject)
 static IORecursiveLock *	gPropertiesLock;
 static SInt32			gIORegistryGenerationCount;
 
-#define UNLOCK	s_lock_done( &gIORegistryLock )
-#define RLOCK	s_lock_read( &gIORegistryLock )
-#define WLOCK	s_lock_write( &gIORegistryLock );	\
+#define UNLOCK	lck_rw_done( &gIORegistryLock )
+#define RLOCK	lck_rw_lock_shared( &gIORegistryLock )
+#define WLOCK	lck_rw_lock_exclusive( &gIORegistryLock );	\
 		gIORegistryGenerationCount++
 		// make atomic
 
@@ -114,170 +114,11 @@ static SInt32			gIORegistryGenerationCount;
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-struct s_lock_t {
-	lck_spin_t interlock;	/* "hardware" interlock field */
-	volatile unsigned int
-		read_count:16,	/* No. of accepted readers */
-		want_upgrade:1,	/* Read-to-write upgrade waiting */
-		want_write:1,	/* Writer is waiting, or
-				   locked for write */
-		waiting:1,	/* Someone is sleeping on lock */
-		can_sleep:1;	/* Can attempts to lock go to sleep? */
-};
-
-static struct s_lock_t	gIORegistryLock;
-
-/* Time we loop without holding the interlock. 
- * The former is for when we cannot sleep, the latter
- * for when our thread can go to sleep (loop less)
- * we shouldn't retake the interlock at all frequently
- * if we cannot go to sleep, since it interferes with
- * any other processors. In particular, 100 is too small
- * a number for powerpc MP systems because of cache
- * coherency issues and differing lock fetch times between
- * the processors
- */
-static unsigned int lock_wait_time[2] = { (unsigned int)-1, 100 } ;
-	  
-static void
-s_lock_init(
-	s_lock_t	*l,
-	boolean_t	can_sleep)
-{
-	(void) memset((void *) l, 0, sizeof(s_lock_t));
-
-	lck_spin_init(&l->interlock, IOLockGroup, LCK_ATTR_NULL);
-	l->want_write = FALSE;
-	l->want_upgrade = FALSE;
-	l->read_count = 0;
-	l->can_sleep = can_sleep;
-}
-
-static void
-s_lock_write(
-	register s_lock_t	* l)
-{
-        register int	   i;
-
-	lck_spin_lock(&l->interlock);
-
-	/*
-	 *	Try to acquire the want_write bit.
-	 */
-	while (l->want_write) {
-
-		i = lock_wait_time[l->can_sleep ? 1 : 0];
-		if (i != 0) {
-			lck_spin_unlock(&l->interlock);
-			while (--i != 0 && l->want_write)
-				continue;
-			lck_spin_lock(&l->interlock);
-		}
-
-		if (l->can_sleep && l->want_write) {
-			l->waiting = TRUE;
-			lck_spin_sleep( &l->interlock, LCK_SLEEP_DEFAULT,
-					(event_t) l, THREAD_UNINT);
-			/* interlock relocked */
-		}
-	}
-	l->want_write = TRUE;
-
-	/* Wait for readers (and upgrades) to finish */
-
-	while ((l->read_count != 0) || l->want_upgrade) {
-
-		i = lock_wait_time[l->can_sleep ? 1 : 0];
-		if (i != 0) {
-			lck_spin_unlock(&l->interlock);
-			while (--i != 0 && (l->read_count != 0 ||
-					    l->want_upgrade))
-				continue;
-			lck_spin_lock(&l->interlock);
-		}
-
-		if (l->can_sleep && (l->read_count != 0 || l->want_upgrade)) {
-			l->waiting = TRUE;
-			lck_spin_sleep( &l->interlock, LCK_SLEEP_DEFAULT,
-					(event_t) l, THREAD_UNINT);
-			/* interlock relocked */
-		}
-	}
-
-	lck_spin_unlock(&l->interlock);
-}
-
-static void
-s_lock_done(
-	register s_lock_t	* l)
-{
-	boolean_t	  do_wakeup = FALSE;
-
-	lck_spin_lock(&l->interlock);
-
-	if (l->read_count != 0) {
-		l->read_count -= 1;
-	}
-	else	{
-		if (l->want_upgrade) {
-			l->want_upgrade = FALSE;
-		}
-                else {
-                        l->want_write = FALSE;
-                }
-        }
-
-	/*
-	 *	There is no reason to wakeup a waiting thread
-	 *	if the read-count is non-zero.  Consider:
-	 *		we must be dropping a read lock
-	 *		threads are waiting only if one wants a write lock
-	 *		if there are still readers, they can't proceed
-	 */
-	if (l->waiting && (l->read_count == 0)) {
-		l->waiting = FALSE;
-		do_wakeup = TRUE;
-	}
-
-	lck_spin_unlock(&l->interlock);
-
-	if (do_wakeup)
-		thread_wakeup((event_t) l);
-}
-
-static void
-s_lock_read(
-	register s_lock_t	* l)
-{
-	register int	    i;
+lck_rw_t	gIORegistryLock;
+lck_grp_t       *gIORegistryLockGrp;
+lck_grp_attr_t  *gIORegistryLockGrpAttr;
+lck_attr_t      *gIORegistryLockAttr;
 
-	lck_spin_lock(&l->interlock);
-
-	while ( l->want_upgrade || ((0 == l->read_count) && l->want_write )) {
-
-		i = lock_wait_time[l->can_sleep ? 1 : 0];
-
-		if (i != 0) {
-			lck_spin_unlock(&l->interlock);
-			while (--i != 0 && 
-                            (l->want_upgrade || ((0 == l->read_count) && l->want_write )))
-				continue;
-			lck_spin_lock(&l->interlock);
-		}
-
-		if (l->can_sleep &&
-                    (l->want_upgrade || ((0 == l->read_count) && l->want_write ))) {
-			l->waiting = TRUE;
-			lck_spin_sleep( &l->interlock, LCK_SLEEP_DEFAULT,
-					(event_t) l, THREAD_UNINT);
-			/* interlock relocked */
-		}
-	}
-
-	l->read_count += 1;
-	lck_spin_unlock(&l->interlock);
-
-}
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
@@ -287,7 +128,15 @@ IORegistryEntry * IORegistryEntry::initialize( void )
 
     if( !gRegistryRoot) {
 
-        s_lock_init( &gIORegistryLock, true );
+
+	gIORegistryLockGrpAttr = lck_grp_attr_alloc_init();
+	//lck_grp_attr_setstat(gIORegistryLockGrpAttr);
+	gIORegistryLockGrp = lck_grp_alloc_init("IORegistryLock",  gIORegistryLockGrpAttr);
+	gIORegistryLockAttr = lck_attr_alloc_init();
+	lck_attr_rw_shared_priority(gIORegistryLockAttr);
+	//lck_attr_setdebug(gIORegistryLockAttr);
+	lck_rw_init( &gIORegistryLock, gIORegistryLockGrp, gIORegistryLockAttr);
+
 	gRegistryRoot = new IORegistryEntry;
 	gPropertiesLock = IORecursiveLockAlloc();
 	gIORegistryPlanes = OSDictionary::withCapacity( 1 );
diff --git a/iokit/Kernel/IOService.cpp b/iokit/Kernel/IOService.cpp
index 7feca38f6..b5991ba4e 100644
--- a/iokit/Kernel/IOService.cpp
+++ b/iokit/Kernel/IOService.cpp
@@ -199,6 +199,22 @@ bool IOService::isInactive( void ) const
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
+#if __i386__
+
+// Only used by the intel implementation of
+//     IOService::requireMaxBusStall(UInt32 __unused ns)
+struct BusStallEntry
+{
+    const IOService *fService;
+    UInt32 fMaxDelay;
+};
+
+static OSData *sBusStall     = OSData::withCapacity(8 * sizeof(BusStallEntry));
+static IOLock *sBusStallLock = IOLockAlloc();
+#endif /* __i386__ */
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
 void IOService::initialize( void )
 {
     kern_return_t	err;
@@ -341,6 +357,7 @@ void IOService::stop( IOService * provider )
 
 void IOService::free( void )
 {
+    requireMaxBusStall(0);
     if( getPropertyTable())
         unregisterAllInterest();
     PMfree();
@@ -495,6 +512,8 @@ void IOService::startMatching( IOOptionBits options )
 	|| ((provider = getProvider())
 		&& (provider->__state[1] & kIOServiceSynchronousState));
 
+	if ( options & kIOServiceAsynchronous )
+		sync = false;
 
     needConfig =  (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigState)))
 	       && (0 == (__state[0] & kIOServiceInactiveState));
@@ -4023,6 +4042,9 @@ IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
     IOUserClient *client;
     OSObject *temp;
 
+    if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler ))
+	return kIOReturnSuccess;
+
     // First try my own properties for a user client class name
     temp = getProperty(gIOUserClientClassKey);
     if (temp) {
@@ -4040,6 +4062,7 @@ IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
     if (!userClientClass)
         return kIOReturnUnsupported;
 
+    // This reference is consumed by the IOServiceOpen call
     temp = OSMetaClass::allocClassWithName(userClientClass);
     if (!temp)
         return kIOReturnNoMemory;
@@ -4074,7 +4097,7 @@ IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
 IOReturn IOService::newUserClient( task_t owningTask, void * securityID,
                                     UInt32 type, IOUserClient ** handler )
 {
-    return( newUserClient( owningTask, securityID, type, 0, handler ));
+    return( kIOReturnUnsupported );
 }
 
 IOReturn IOService::requestProbe( IOOptionBits options )
@@ -4309,6 +4332,92 @@ void IOService::setDeviceMemory( OSArray * array )
     setProperty( gIODeviceMemoryKey, array);
 }
 
+/*
+ * For machines where the transfers on an I/O bus can stall because
+ * the CPU is in an idle mode, These APIs allow a driver to specify
+ * the maximum bus stall that they can handle.  0 indicates no limit.
+ */
+void IOService::
+setCPUSnoopDelay(UInt32 __unused ns)
+{
+#if __i386__
+    ml_set_maxsnoop(ns); 
+#endif /* __i386__ */
+}
+
+UInt32 IOService::
+getCPUSnoopDelay()
+{
+#if __i386__
+    return ml_get_maxsnoop(); 
+#else
+    return 0;
+#endif /* __i386__ */
+}
+
+void IOService::
+requireMaxBusStall(UInt32 __unused ns)
+{
+#if __i386__
+    static const UInt kNoReplace = -1U;	// Must be an illegal index
+    UInt replace = kNoReplace;
+
+    IOLockLock(sBusStallLock);
+
+    UInt count = sBusStall->getLength() / sizeof(BusStallEntry);
+    BusStallEntry *entries = (BusStallEntry *) sBusStall->getBytesNoCopy();
+
+    if (ns) {
+	const BusStallEntry ne = {this, ns};
+
+	// Set Maximum bus delay.
+	for (UInt i = 0; i < count; i++) {
+	    const IOService *thisService = entries[i].fService;
+	    if (this == thisService)
+		replace = i;
+	    else if (!thisService) {
+		if (kNoReplace == replace)
+		    replace = i;
+	    }
+	    else {
+		const UInt32 thisMax = entries[i].fMaxDelay;
+		if (thisMax < ns)
+		    ns = thisMax;
+	    }
+	}
+
+	// Must be safe to call from locked context
+	ml_set_maxbusdelay(ns);
+
+	if (kNoReplace == replace)
+	    sBusStall->appendBytes(&ne, sizeof(ne));
+	else
+	    entries[replace] = ne;
+    }
+    else {
+	ns = -1U;	// Set to max unsigned, i.e. no restriction
+
+	for (UInt i = 0; i < count; i++) {
+	    // Clear a maximum bus delay.
+	    const IOService *thisService = entries[i].fService;
+	    UInt32 thisMax = entries[i].fMaxDelay;
+	    if (this == thisService)
+		replace = i;
+	    else if (thisService && thisMax < ns)
+		ns = thisMax;
+	}
+
+	// Check if entry found
+	if (kNoReplace != replace) {
+	    entries[replace].fService = 0;	// Null the entry
+	    ml_set_maxbusdelay(ns);
+	}
+    }
+
+    IOLockUnlock(sBusStallLock);
+#endif /* __i386__ */
+}
+
 /*
  * Device interrupts
  */
@@ -4514,6 +4623,8 @@ OSMetaClassDefineReservedUnused(IOService, 44);
 OSMetaClassDefineReservedUnused(IOService, 45);
 OSMetaClassDefineReservedUnused(IOService, 46);
 OSMetaClassDefineReservedUnused(IOService, 47);
+
+#ifdef __ppc__
 OSMetaClassDefineReservedUnused(IOService, 48);
 OSMetaClassDefineReservedUnused(IOService, 49);
 OSMetaClassDefineReservedUnused(IOService, 50);
@@ -4530,3 +4641,4 @@ OSMetaClassDefineReservedUnused(IOService, 60);
 OSMetaClassDefineReservedUnused(IOService, 61);
 OSMetaClassDefineReservedUnused(IOService, 62);
 OSMetaClassDefineReservedUnused(IOService, 63);
+#endif
diff --git a/iokit/Kernel/IOServicePM.cpp b/iokit/Kernel/IOServicePM.cpp
index 1f818780e..7993b7458 100644
--- a/iokit/Kernel/IOServicePM.cpp
+++ b/iokit/Kernel/IOServicePM.cpp
@@ -44,6 +44,7 @@
 #include <IOKit/pwr_mgt/IOPowerConnection.h>
 #include <IOKit/pwr_mgt/RootDomain.h>
 
+
 // Required for notification instrumentation
 #include "IOServicePrivate.h"
 
@@ -287,8 +288,13 @@ void IOService::PMinit ( void )
         pm_vars->responseFlags = NULL;
         pm_vars->doNotPowerDown = true;
         pm_vars->PMcommandGate = NULL;
-        priv->ackTimer = thread_call_allocate((thread_call_func_t)ack_timer_expired, (thread_call_param_t)this);
-        priv->settleTimer = thread_call_allocate((thread_call_func_t)settle_timer_expired, (thread_call_param_t)this);
+        priv->ackTimer = thread_call_allocate(
+                        (thread_call_func_t)ack_timer_expired, 
+                        (thread_call_param_t)this);
+        priv->settleTimer = thread_call_allocate(
+                        (thread_call_func_t)settle_timer_expired, 
+                        (thread_call_param_t)this);
+        
         initialized = true;
     }
 }
@@ -591,7 +597,7 @@ IOReturn IOService::addPowerChild ( IOService * theChild )
     // Put ourselves into a usable power state.
     // We must be in an "on" power state, as our children must be able to access
     // our hardware after joining the power plane.
-    makeUsable();
+    temporaryMakeUsable();
     
     // make a nub
     connection = new IOPowerConnection;			
@@ -637,14 +643,13 @@ IOReturn IOService::addPowerChild ( IOService * theChild )
 }
 
 
-//*********************************************************************************
+//******************************************************************************
 // removePowerChild
 //
-//*********************************************************************************
+//******************************************************************************
 IOReturn IOService::removePowerChild ( IOPowerConnection * theNub )
 {
     IORegistryEntry                         *theChild;
-    OSIterator                              *iter;
 
     pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogRemoveChild,0,0);
 
@@ -670,7 +675,7 @@ IOReturn IOService::removePowerChild ( IOPowerConnection * theNub )
             if (priv->head_note_pendingAcks != 0 ) 
             {
                 // that's one fewer ack to worry about
-                priv->head_note_pendingAcks -= 1;
+                OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
                 // is that the last?
                 if ( priv->head_note_pendingAcks == 0 ) 
                 {
@@ -699,25 +704,13 @@ IOReturn IOService::removePowerChild ( IOPowerConnection * theNub )
         return IOPMNoErr;				
     }
 
-    // Perhaps the departing child was holding up idle or system sleep - we need to re-evaluate our
-    // childrens' requests. Clear and re-calculate our kIOPMChildClamp and kIOPMChildClamp2 bits.
+    // Perhaps the departing child was holding up idle or system sleep - we 
+    // need to re-evaluate our childrens' requests. 
+    // Clear and re-calculate our kIOPMChildClamp and kIOPMChildClamp2 bits.
     rebuildChildClampBits();
     
-    if(!priv->clampOn)
-    {
-        // count children
-        iter = getChildIterator(gIOPowerPlane);
-        if ( !iter || !iter->getNextObject()  ) 
-        {
-            // paired to match the makeUsable() call in addPowerChild()
-            changePowerStateToPriv(0);
-        }
-        if(iter) iter->release();
-    }
-    
-    // this may be different now
+    // Change state if we can now tolerate lower power
     computeDesiredState();
-    // change state if we can now tolerate lower power
     changeState();
 
     return IOPMNoErr;
@@ -952,7 +945,7 @@ IOReturn IOService::acknowledgePowerChange ( IOService * whichObject )
                 // mark it acked
                 ackingObject->timer = 0;
                 // that's one fewer to worry about
-                priv->head_note_pendingAcks -= 1;
+                OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
                 // is that the last?
                 if ( priv->head_note_pendingAcks == 0 ) 
                 {
@@ -974,7 +967,7 @@ IOReturn IOService::acknowledgePowerChange ( IOService * whichObject )
             if ( ((IOPowerConnection *)whichObject)->getAwaitingAck() ) 
             {
                 // that's one fewer to worry about
-                priv->head_note_pendingAcks -= 1;
+                OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
                 ((IOPowerConnection *)whichObject)->setAwaitingAck(false);
                 theChild = (IOService *)whichObject->copyChildEntry(gIOPowerPlane);
                 if ( theChild ) 
@@ -1057,6 +1050,7 @@ IOReturn IOService::acknowledgeSetPowerState ( void )
 
 void IOService::driver_acked ( void )
 {
+
     switch (priv->machine_state) {
         case kIOPM_OurChangeWaitForPowerSettle:
             OurChangeWaitForPowerSettle();
@@ -1464,6 +1458,40 @@ IOReturn IOService::makeUsable ( void )
     return IOPMNoErr;
 }
 
+//******************************************************************************
+// temporaryMakeUsable
+//
+// Private function, called by IOService::addPowerChild to ensure that the
+// device is temporarily in a usable power state so that attached power  
+// children may properly initialize.
+//******************************************************************************
+
+IOReturn IOService::temporaryMakeUsable ( void )
+{
+    IOReturn        ret = kIOReturnSuccess;
+    unsigned long   tempDesire;
+    
+    pm_vars->thePlatform->PMLog( pm_vars->ourName,
+                                 PMlogMakeUsable,
+                                 PMlogMakeUsable,
+                                 priv->deviceDesire);
+
+    if ( pm_vars->theControllingDriver == NULL ) 
+    {
+        priv->need_to_become_usable = true;
+        return IOPMNoErr;
+    }
+    tempDesire = priv->deviceDesire;
+    priv->deviceDesire = pm_vars->theNumberOfPowerStates - 1;
+    computeDesiredState();
+    if ( inPlane(gIOPowerPlane) && (pm_vars->parentsKnowState) ) 
+    {
+        ret = changeState();
+    }
+    priv->deviceDesire = tempDesire;
+    return ret;
+}
+
 
 //*********************************************************************************
 // currentCapability
@@ -1868,9 +1896,13 @@ void IOService::PM_idle_timer_expiration ( void )
         }
         if ( pm_vars->myCurrentState > 0 ) 
         {
+        	
+        	unsigned long	newState = pm_vars->myCurrentState - 1;
+        	
             IOUnlock(priv->activityLock);
-            changePowerStateToPriv(pm_vars->myCurrentState - 1);
-            start_PM_idle_timer();
+            changePowerStateToPriv(newState);
+            if ( newState >= priv->ourDesiredPowerState )
+            	start_PM_idle_timer();
             return;
         }
         IOUnlock(priv->activityLock);
@@ -2103,7 +2135,12 @@ IOReturn IOService::powerOverrideOffPriv ( void )
 // needn't perform the previous change, so we collapse the list a little.
 //*********************************************************************************
 
-IOReturn IOService::enqueuePowerChange ( unsigned long flags,  unsigned long whatStateOrdinal, unsigned long domainState, IOPowerConnection * whichParent, unsigned long singleParentState )
+IOReturn IOService::enqueuePowerChange ( 
+    unsigned long flags,  
+    unsigned long whatStateOrdinal, 
+    unsigned long domainState, 
+    IOPowerConnection * whichParent, 
+    unsigned long singleParentState )
 {
     long                            newNote;
     long                            previousNote;
@@ -2192,26 +2229,35 @@ IOReturn IOService::notifyAll ( bool is_prechange )
     OSObject *			next;
     IOPowerConnection *	connection;
 
-    // To prevent acknowledgePowerChange from finishing the change note and removing it from the queue if
-    // some driver calls it, we inflate the number of pending acks so it cannot become zero.  We'll fix it later.
-
-    priv->head_note_pendingAcks =1;
+    // To prevent acknowledgePowerChange from finishing the change note and 
+    // removing it from the queue if
+    // some driver calls it, we inflate the number of pending acks so it 
+    // cannot become zero.  We'll fix it later.
+    
+    if(!acquire_lock()) return IOPMAckImplied;
 
-    // OK, we will go through the lists of interested drivers and power domain children
-    // and notify each one of this change.
+    OSAddAtomic(1, (SInt32*)&priv->head_note_pendingAcks);
 
+    // OK, we will go through the lists of interested drivers and 
+    // power domain children and notify each one of this change.
+    
     nextObject =  priv->interestedDrivers->firstInList();
     while (  nextObject != NULL ) {
-        priv->head_note_pendingAcks +=1;
-        if (! inform(nextObject, is_prechange) ) 
+
+        OSAddAtomic(1, (SInt32*)&priv->head_note_pendingAcks);
+
+        IOUnlock(priv->our_lock);
+
+        inform(nextObject, is_prechange);
+        
+        if(!acquire_lock())
         {
+            goto exit;
         }
+        
         nextObject  =  priv->interestedDrivers->nextInList(nextObject);
     }
 
-    if (! acquire_lock() ) {
-        return IOPMNoErr;
-    }
     // did they all ack?
     if ( priv->head_note_pendingAcks > 1 ) {
         // no
@@ -2226,24 +2272,34 @@ IOReturn IOService::notifyAll ( bool is_prechange )
     // summing their power consumption
     pm_vars->thePowerStates[priv->head_note_state].staticPower = 0;
 
-    if ( iter ) 
+    if ( iter && acquire_lock()) 
     {
         while ( (next = iter->getNextObject()) ) 
         {
             if ( (connection = OSDynamicCast(IOPowerConnection,next)) ) 
             {
-                priv->head_note_pendingAcks +=1;
+                OSAddAtomic(1, (SInt32*)&priv->head_note_pendingAcks);
+                
+                IOUnlock(priv->our_lock);
+
                 notifyChild(connection, is_prechange);
+            
+                if(!acquire_lock())
+                {
+                    goto exit;
+                }
             }
         }
         iter->release();
+        IOUnlock(priv->our_lock);
     }
 
     if (! acquire_lock() ) {
         return IOPMNoErr;
     }
     // now make this real
-    priv->head_note_pendingAcks -= 1;
+    OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
+
     // is it all acked?
     if (priv->head_note_pendingAcks == 0 ) {
         // yes, all acked
@@ -2254,6 +2310,9 @@ IOReturn IOService::notifyAll ( bool is_prechange )
 
     // not all acked
     IOUnlock(priv->our_lock);
+
+exit:           // unable to acquire_lock exit case
+
     return IOPMWillAckLater;
 }
 
@@ -2278,7 +2337,12 @@ bool IOService::notifyChild ( IOPowerConnection * theNub, bool is_prechange )
         // The child has been detached since we grabbed the child iterator.
         // Decrement pending_acks, already incremented in notifyAll,
         // to account for this unexpected departure.
-        priv->head_note_pendingAcks--;
+
+        if( acquire_lock() ) 
+        {
+            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
+            IOUnlock(priv->our_lock); 
+        }
         return true;
     }
     
@@ -2297,7 +2361,12 @@ bool IOService::notifyChild ( IOPowerConnection * theNub, bool is_prechange )
     if ( k == IOPMAckImplied ) 
     {
         // yes
-        priv->head_note_pendingAcks--;
+        if( acquire_lock() ) 
+        {
+            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
+            IOUnlock(priv->our_lock); 
+        }
+        
         theNub->setAwaitingAck(false);
         childPower = theChild->currentPowerConsumption();
         if ( childPower == kIOPMUnknown ) 
@@ -2350,19 +2419,17 @@ bool IOService::inform ( IOPMinformee * nextObject, bool is_prechange )
         return true;
     }
     
-    // no, did the return code ack?
-    if ( k ==IOPMAckImplied ) 
+    if ( (k ==IOPMAckImplied)   // no, did the return code ack?
+      || (k < 0) )              // somebody goofed
     {
         // yes
         nextObject->timer = 0;
-        priv->head_note_pendingAcks -= 1;
-        return true;
-    }
-    if ( k<0 ) 
-    {
-        // somebody goofed
-        nextObject->timer = 0;
-        priv-> head_note_pendingAcks -= 1;
+
+        if( acquire_lock() ) 
+        {
+            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
+            IOUnlock(priv->our_lock); 
+        }
         return true;
     }
     
@@ -2459,14 +2526,18 @@ void IOService::OurChangeSetPowerState ( void )
 {
     priv->machine_state = kIOPM_OurChangeWaitForPowerSettle;
 
+    IOLockLock(priv->our_lock);
+
     if ( instruct_driver(priv->head_note_state) == IOPMAckImplied ) 
     {
         // it's done, carry on
+        IOLockUnlock(priv->our_lock);
         OurChangeWaitForPowerSettle();
     } else {
         // it's not, wait for it
         pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogStartAckTimer,0,0);
         start_ack_timer();
+        IOLockUnlock(priv->our_lock);
         // execution will resume via ack_timer_ticked()
     }
 }
@@ -2646,14 +2717,18 @@ IOReturn IOService::ParentDownSetPowerState_Immediate ( void )
 {
     priv->machine_state = kIOPM_ParentDownWaitForPowerSettle_Delayed;
 
+    IOLockLock(priv->our_lock);
+
     if ( instruct_driver(priv->head_note_state) == IOPMAckImplied ) 
     {
         // it's done, carry on
+        IOLockUnlock(priv->our_lock);
         return ParentDownWaitForPowerSettleAndNotifyDidChange_Immediate();
     }
     // it's not, wait for it
     pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogStartAckTimer,0,0);
     start_ack_timer();
+    IOLockUnlock(priv->our_lock);
     return IOPMWillAckLater;
 }
 
@@ -2674,14 +2749,18 @@ void IOService::ParentDownSetPowerState_Delayed ( void )
 {
     priv-> machine_state = kIOPM_ParentDownWaitForPowerSettle_Delayed;
 
+    IOLockLock(priv->our_lock);
+
     if ( instruct_driver(priv->head_note_state) == IOPMAckImplied ) 
     {
         // it's done, carry on
+        IOLockUnlock(priv->our_lock);
         ParentDownWaitForPowerSettle_Delayed();
     } else {
         // it's not, wait for it
         pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogStartAckTimer,0,0);
         start_ack_timer();
+        IOLockUnlock(priv->our_lock);
     }
 }
 
@@ -2826,14 +2905,18 @@ void IOService::ParentUpSetPowerState_Delayed ( void )
 {
     priv->machine_state = kIOPM_ParentUpWaitForSettleTime_Delayed;
  
+    IOLockLock(priv->our_lock);
+ 
     if ( instruct_driver(priv->head_note_state) == IOPMAckImplied ) 
     {
         // it did it, carry on
+        IOLockUnlock(priv->our_lock);
         ParentUpWaitForSettleTime_Delayed();
     } else {
         // it didn't, wait for it
         pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogStartAckTimer,0,0);
         start_ack_timer();
+        IOLockUnlock(priv->our_lock);
     }
 }
 
@@ -2853,15 +2936,19 @@ IOReturn IOService::ParentUpSetPowerState_Immediate ( void )
 {
     priv->machine_state = kIOPM_ParentUpWaitForSettleTime_Delayed;
 
+    IOLockLock(priv->our_lock);
+
     if ( instruct_driver(priv->head_note_state) == IOPMAckImplied ) 
     {
         // it did it, carry on
+        IOLockUnlock(priv->our_lock);
         return ParentUpWaitForSettleTime_Immediate();
     }
     else {
         // it didn't, wait for it
         pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogStartAckTimer,0,0);
         start_ack_timer();
+        IOLockUnlock(priv->our_lock);
         return IOPMWillAckLater;
     }
 }
@@ -3075,6 +3162,7 @@ void IOService::all_done ( void )
             start_our_change(priv->head_note);
         } else {
             nub = priv->changeList->changeNote[priv->head_note].parent;
+            if (nub) nub->retain(); // might be released by start_parent_change()
             if ( start_parent_change(priv->head_note) == IOPMAckImplied ) 
             {
                 parent = (IOService *)nub->copyParentEntry(gIOPowerPlane);
@@ -3084,6 +3172,7 @@ void IOService::all_done ( void )
                     parent->release();
                 }
             }
+            if (nub) nub->release();
         }
     } else {
         IOLockUnlock(priv->queue_lock);
@@ -3101,7 +3190,9 @@ void IOService::all_done ( void )
 //
 //*********************************************************************************
 
-void IOService::all_acked ( void )
+
+
+void IOService::all_acked( void )
 {
     switch (priv->machine_state) {
        case kIOPM_OurChangeSetPowerState:
@@ -3286,7 +3377,7 @@ void IOService::ack_timer_ticked ( void )
                         {
                             pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogIntDriverTardy,0,0);
                             //kprintf("interested driver tardy: %s\n",nextObject->whatObject->getName());
-                            priv->head_note_pendingAcks -= 1;
+                            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
                         }
                     }
                     nextObject  =  priv->interestedDrivers->nextInList(nextObject);
@@ -3464,18 +3555,19 @@ IOReturn IOService::add_child_to_active_change ( IOPowerConnection * newObject )
         case kIOPM_ParentDownSetPowerState_Delayed:
         case kIOPM_ParentUpSetPowerState_Delayed:
             // one for this child and one to prevent
-            priv->head_note_pendingAcks += 2;
+            OSAddAtomic(2, (SInt32*)&priv->head_note_pendingAcks);
             // incoming acks from changing our state
             IOUnlock(priv->our_lock);
             notifyChild(newObject, true);
             if (! acquire_lock() ) 
             {
                 // put it back
-                --priv->head_note_pendingAcks;
+                OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
                 return IOPMNoErr;
             }
             // are we still waiting for acks?
-            if ( --priv->head_note_pendingAcks == 0 ) 
+            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
+            if ( priv->head_note_pendingAcks == 0 ) 
             {
                 // no, stop the timer
                 stop_ack_timer();
@@ -3490,18 +3582,19 @@ IOReturn IOService::add_child_to_active_change ( IOPowerConnection * newObject )
         case kIOPM_ParentDownAcknowledgeChange_Delayed:
         case kIOPM_ParentUpAcknowledgePowerChange_Delayed:
             // one for this child and one to prevent
-            priv->head_note_pendingAcks += 2;
+            OSAddAtomic(2, (SInt32*)&priv->head_note_pendingAcks);
             // incoming acks from changing our state
             IOUnlock(priv->our_lock);
             notifyChild(newObject, false);
             if (! acquire_lock() ) 
             {
                 // put it back
-                --priv->head_note_pendingAcks;
+                OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
                 return IOPMNoErr;
             }
             // are we still waiting for acks?
-            if ( --priv->head_note_pendingAcks == 0 ) 
+            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
+            if ( priv->head_note_pendingAcks == 0 ) 
             {
                 // no, stop the timer
                 stop_ack_timer();
@@ -3539,7 +3632,7 @@ IOReturn IOService::add_driver_to_active_change ( IOPMinformee * newObject )
         case kIOPM_ParentDownSetPowerState_Delayed:
         case kIOPM_ParentUpSetPowerState_Delayed:
             // one for this driver and one to prevent
-            priv->head_note_pendingAcks += 2;
+            OSAddAtomic(2, (SInt32*)&priv->head_note_pendingAcks);
             // incoming acks from changing our state
             IOUnlock(priv->our_lock);
             // inform the driver
@@ -3547,11 +3640,12 @@ IOReturn IOService::add_driver_to_active_change ( IOPMinformee * newObject )
             if (! acquire_lock() ) 
             {
                 // put it back
-                --priv->head_note_pendingAcks;
+                OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
                 return IOPMNoErr;
             }
             // are we still waiting for acks?
-            if ( --priv->head_note_pendingAcks == 0 ) 
+            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
+            if ( priv->head_note_pendingAcks == 0 ) 
             {
                 // no, stop the timer
                 stop_ack_timer();
@@ -3566,18 +3660,19 @@ IOReturn IOService::add_driver_to_active_change ( IOPMinformee * newObject )
         case kIOPM_ParentDownAcknowledgeChange_Delayed:
         case kIOPM_ParentUpAcknowledgePowerChange_Delayed:
             // one for this driver and one to prevent
-            priv->head_note_pendingAcks += 2;
+            OSAddAtomic(2, (SInt32*)&priv->head_note_pendingAcks);
             // incoming acks from changing our state
             IOUnlock(priv->our_lock);
             // inform the driver
             inform(newObject, false);
             if (! acquire_lock() ) {
                 // put it back
-                --priv->head_note_pendingAcks;
+                OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
                 return IOPMNoErr;
             }
             // are we still waiting for acks?
-            if ( --priv->head_note_pendingAcks == 0 ) {
+            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
+            if ( priv->head_note_pendingAcks == 0 ) {
                 // no, stop the timer
                 stop_ack_timer();
                 IOUnlock(priv->our_lock);
@@ -3887,9 +3982,11 @@ IOReturn IOService::instruct_driver ( unsigned long newState )
     priv->driver_timer = -1;
 
     // yes, instruct it
+    IOLockUnlock(priv->our_lock);
     OUR_PMLog(          kPMLogProgramHardware, (UInt32) this, newState);
     delay = pm_vars->theControllingDriver->setPowerState( newState,this );
     OUR_PMLog((UInt32) -kPMLogProgramHardware, (UInt32) this, (UInt32) delay);
+    IOLockLock(priv->our_lock);
 
     // it finished
     if ( delay == IOPMAckImplied ) 
diff --git a/iokit/Kernel/IOStartIOKit.cpp b/iokit/Kernel/IOStartIOKit.cpp
index 6494f247c..1245c46ce 100644
--- a/iokit/Kernel/IOStartIOKit.cpp
+++ b/iokit/Kernel/IOStartIOKit.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -43,6 +43,7 @@
 #include <IOKit/IOLib.h>
 #include <IOKit/IOKitKeys.h>
 #include <IOKit/IOKitDebug.h>
+#include <IOKit/pwr_mgt/IOPMinformeeList.h>
 
 #include <IOKit/assert.h>
 
@@ -51,6 +52,7 @@
 extern "C" {
 
 extern void OSlibkernInit (void);
+extern void ml_hpet_cfg(uint32_t, uint32_t);
 
 #include <kern/clock.h>
 
@@ -65,6 +67,7 @@ void IOKitResetTime( void )
 #ifndef i386
 	IOService::waitForService(
 		IOService::resourceMatching("IONVRAM"), &t );
+
 #endif
 
     clock_initialize_calendar();
@@ -156,6 +159,9 @@ void StartIOKit( void * p1, void * p2, void * p3, void * p4 )
     IOUserClient::initialize();
     IOMemoryDescriptor::initialize();
 
+    // Initializes IOPMinformeeList class-wide shared lock
+    IOPMinformeeList::getSharedRecursiveLock();
+
     obj = OSString::withCString( version );
     assert( obj );
     if( obj ) {
diff --git a/iokit/Kernel/IOStringFuncs.c b/iokit/Kernel/IOStringFuncs.c
index 85a3e326f..76172e518 100644
--- a/iokit/Kernel/IOStringFuncs.c
+++ b/iokit/Kernel/IOStringFuncs.c
@@ -363,11 +363,10 @@ strtoq(nptr, endptr, base)
  * Ignores `locale' stuff.  Assumes that the upper and lower case
  * alphabets and digits are each contiguous.
  */
-u_quad_t
-strtouq(nptr, endptr, base)
-	const char *nptr;
-	char **endptr;
-	register int base;
+uint64_t
+strtouq(const char *nptr,
+	char **endptr,
+	register int base)
 {
 	register const char *s = nptr;
 	register u_quad_t acc;
diff --git a/iokit/Kernel/IOUserClient.cpp b/iokit/Kernel/IOUserClient.cpp
index 7506a0cc4..45a4f3c22 100644
--- a/iokit/Kernel/IOUserClient.cpp
+++ b/iokit/Kernel/IOUserClient.cpp
@@ -31,7 +31,6 @@
 #include <IOKit/IOKitKeysPrivate.h>
 #include <IOKit/IOUserClient.h>
 #include <IOKit/IOService.h>
-#include <IOKit/IOService.h>
 #include <IOKit/IORegistryEntry.h>
 #include <IOKit/IOCatalogue.h>
 #include <IOKit/IOMemoryDescriptor.h>
@@ -837,6 +836,22 @@ IOReturn IOUserClient::clientHasPrivilege( void * securityToken,
     return (kr);
 }
 
+bool IOUserClient::init()
+{
+    if( getPropertyTable())
+        return true;
+    else
+        return super::init();
+}
+
+bool IOUserClient::init(OSDictionary * dictionary)
+{
+    if( getPropertyTable())
+        return true;
+    else
+        return super::init(dictionary);
+}
+
 bool IOUserClient::initWithTask(task_t owningTask,
                                 void * securityID,
                                 UInt32 type )
@@ -1760,6 +1775,7 @@ kern_return_t is_io_registry_entry_get_property_bytes(
     return( ret );
 }
 
+
 /* Routine io_registry_entry_get_property */
 kern_return_t is_io_registry_entry_get_property(
 	io_object_t registry_entry,
@@ -1993,7 +2009,7 @@ kern_return_t is_io_service_open(
     CHECK( IOService, _service, service );
 
     err = service->newUserClient( owningTask, (void *) owningTask,
-		connect_type, &client );
+		connect_type, 0, &client );
 
     if( err == kIOReturnSuccess) {
 	assert( OSDynamicCast(IOUserClient, client) );
@@ -2003,6 +2019,101 @@ kern_return_t is_io_service_open(
     return( err);
 }
 
+/* Routine io_service_open_ndr */
+kern_return_t is_io_service_open_extended(
+	io_object_t _service,
+	task_t owningTask,
+	int connect_type,
+	NDR_record_t ndr,
+	io_buf_ptr_t properties,
+	mach_msg_type_number_t propertiesCnt,
+        natural_t * result,
+	io_object_t *connection )
+{
+    IOUserClient * client = 0;
+    kern_return_t  err = KERN_SUCCESS;
+    IOReturn	   res = kIOReturnSuccess;
+    OSDictionary * propertiesDict = 0;
+    bool	   crossEndian;
+    bool	   disallowAccess;
+
+    CHECK( IOService, _service, service );
+
+    do
+    {
+	if (properties)
+	{
+	    OSObject *	    obj;
+	    vm_offset_t     data;
+	    vm_map_offset_t map_data;
+
+	    err = vm_map_copyout( kernel_map, &map_data, (vm_map_copy_t) properties );
+	    res = err;
+	    data = CAST_DOWN(vm_offset_t, map_data);
+	    if (KERN_SUCCESS == err)
+	    {
+		// must return success after vm_map_copyout() succeeds
+		obj = OSUnserializeXML( (const char *) data );
+		vm_deallocate( kernel_map, data, propertiesCnt );
+		propertiesDict = OSDynamicCast(OSDictionary, obj);
+		if (!propertiesDict)
+		{
+		    res = kIOReturnBadArgument;
+		    if (obj)
+			obj->release();
+		}
+	    }
+	    if (kIOReturnSuccess != res)
+		break;
+	}
+
+	crossEndian = (ndr.int_rep != NDR_record.int_rep);
+	if (crossEndian)
+	{
+	    if (!propertiesDict)
+		propertiesDict = OSDictionary::withCapacity(4);
+	    OSData * data = OSData::withBytes(&ndr, sizeof(ndr));
+	    if (data)
+	    {
+		if (propertiesDict)
+		    propertiesDict->setObject(kIOUserClientCrossEndianKey, data);
+		data->release();
+	    }
+	}
+
+	res = service->newUserClient( owningTask, (void *) owningTask,
+		    connect_type, propertiesDict, &client );
+
+	if (propertiesDict)
+	    propertiesDict->release();
+
+	if (res == kIOReturnSuccess)
+	{
+	    assert( OSDynamicCast(IOUserClient, client) );
+
+	    disallowAccess = (crossEndian
+		&& (kOSBooleanTrue != service->getProperty(kIOUserClientCrossEndianCompatibleKey))
+		&& (kOSBooleanTrue != client->getProperty(kIOUserClientCrossEndianCompatibleKey)));
+
+	    if (disallowAccess)
+	    {
+		client->clientClose();
+		client->release();
+		client = 0;
+		res = kIOReturnUnsupported;
+		break;
+	    }
+	    client->sharedInstance = (0 != client->getProperty(kIOUserClientSharedInstanceKey));
+	}
+    }
+    while (false);
+
+    *connection = client;
+    *result = res;
+
+    return (err);
+}
+
 /* Routine io_service_close */
 kern_return_t is_io_service_close(
 	io_object_t connection )
@@ -2069,7 +2180,8 @@ kern_return_t is_io_connect_map_memory(
         if( mapSize)
             *mapSize = map->getLength();
 
-        if( task != current_task()) {
+        if( client->sharedInstance
+	    || (task != current_task())) {
             // push a name out to the task owning the map,
             // so we can clean up maps
 #if IOASSERT
diff --git a/iokit/Kernel/IOWorkLoop.cpp b/iokit/Kernel/IOWorkLoop.cpp
index f2c11d948..c5f757160 100644
--- a/iokit/Kernel/IOWorkLoop.cpp
+++ b/iokit/Kernel/IOWorkLoop.cpp
@@ -44,8 +44,8 @@ OSDefineMetaClassAndStructors(IOWorkLoop, OSObject);
 
 // Block of unused functions intended for future use
 OSMetaClassDefineReservedUsed(IOWorkLoop, 0);
+OSMetaClassDefineReservedUsed(IOWorkLoop, 1);
 
-OSMetaClassDefineReservedUnused(IOWorkLoop, 1);
 OSMetaClassDefineReservedUnused(IOWorkLoop, 2);
 OSMetaClassDefineReservedUnused(IOWorkLoop, 3);
 OSMetaClassDefineReservedUnused(IOWorkLoop, 4);
@@ -76,7 +76,8 @@ bool IOWorkLoop::init()
         return false;
 
     controlG = IOCommandGate::
-	commandGate(this, (IOCommandGate::Action) &IOWorkLoop::_maintRequest);
+	commandGate(this, OSMemberFunctionCast(IOCommandGate::Action,
+					this, &IOWorkLoop::_maintRequest));
     if ( !controlG )
         return false;
 
@@ -90,7 +91,9 @@ bool IOWorkLoop::init()
     if (addEventSource(controlG) != kIOReturnSuccess)
         return false;
 
-    workThread = IOCreateThread((thread_continue_t)threadMainContinuation, this);
+    IOThreadFunc cptr =
+	OSMemberFunctionCast(IOThreadFunc, this, &IOWorkLoop::threadMain);
+    workThread = IOCreateThread(cptr, this);
     if (!workThread)
         return false;
 
@@ -246,52 +249,55 @@ do {									\
 
 #endif /* KDEBUG */
 
-void IOWorkLoop::threadMainContinuation(IOWorkLoop *self)
+/* virtual */ bool IOWorkLoop::runEventSources()
 {
-	self->threadMain();
-}
-
-void IOWorkLoop::threadMain()
-{
-    CLRP(&fFlags, kLoopRestart);
-
-    for (;;) {
-        bool more;
-	IOInterruptState is;
+    bool res = false;
+    closeGate();
+    if (ISSETP(&fFlags, kLoopTerminate))
+	goto abort;
 
     IOTimeWorkS();
+    bool more;
+    do {
+	CLRP(&fFlags, kLoopRestart);
+	workToDo = more = false;
+	for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) {
+
+	    IOTimeClientS();
+	    more |= evnt->checkForWork();
+	    IOTimeClientE();
+
+	    if (ISSETP(&fFlags, kLoopTerminate))
+		goto abort;
+	    else if (fFlags & kLoopRestart) {
+		more = true;
+		break;
+	    }
+	}
+    } while (more);
 
-        closeGate();
-        if (ISSETP(&fFlags, kLoopTerminate))
-	    goto exitThread;
-
-        do {
-            workToDo = more = false;
-            for (IOEventSource *event = eventChain; event; event = event->getNext()) {
-
-            IOTimeClientS();
-                more |= event->checkForWork();
-            IOTimeClientE();
-
-		if (ISSETP(&fFlags, kLoopTerminate))
-		    goto exitThread;
-                else if (fFlags & kLoopRestart) {
-		    CLRP(&fFlags, kLoopRestart);
-                    continue;
-                }
-            }
-        } while (more);
-
+    res = true;
     IOTimeWorkE();
 
-        openGate();
+abort:
+    openGate();
+    return res;
+}
+
+/* virtual */ void IOWorkLoop::threadMain()
+{
+    do {
+	if ( !runEventSources() )
+	    goto exitThread;
 
-	is = IOSimpleLockLockDisableInterrupt(workToDoLock);
+	IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
         if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) {
 	    assert_wait((void *) &workToDo, false);
 	    IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
 
-	    thread_block_parameter((thread_continue_t)threadMainContinuation, this);
+	    thread_continue_t cptr = OSMemberFunctionCast(
+		    thread_continue_t, this, &IOWorkLoop::threadMain);
+	    thread_block_parameter(cptr, this);
 	    /* NOTREACHED */
 	}
 
@@ -299,11 +305,7 @@ void IOWorkLoop::threadMain()
 	// to commit suicide.  But no matter 
 	// Clear the simple lock and retore the interrupt state
 	IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
-	if (workToDo)
-	    continue;
-	else
-	    break;
-    }
+    } while(workToDo);
 
 exitThread:
     workThread = 0;	// Say we don't have a loop and free ourselves
diff --git a/iokit/Kernel/RootDomainUserClient.cpp b/iokit/Kernel/RootDomainUserClient.cpp
index d55003ded..c6c812c54 100644
--- a/iokit/Kernel/RootDomainUserClient.cpp
+++ b/iokit/Kernel/RootDomainUserClient.cpp
@@ -32,6 +32,7 @@
 
 #include <IOKit/assert.h>
 #include <IOKit/IOLib.h>
+#include <IOKit/IOKitKeys.h>
 #include <IOKit/IOBufferMemoryDescriptor.h>
 #include "RootDomainUserClient.h"
 #include <IOKit/pwr_mgt/IOPMLibDefs.h>
@@ -44,9 +45,13 @@ OSDefineMetaClassAndStructors(RootDomainUserClient, IOUserClient)
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-bool RootDomainUserClient::initWithTask(task_t owningTask, void *security_id, UInt32)
+bool RootDomainUserClient::initWithTask(task_t owningTask, void *security_id, 
+					UInt32 type, OSDictionary * properties)
 {
-    if (!super::init())
+    if (properties)
+	properties->setObject(kIOUserClientCrossEndianCompatibleKey, kOSBooleanTrue);
+
+    if (!super::initWithTask(owningTask, security_id, type, properties))
 	return false;
 
     fOwningTask = owningTask;
@@ -131,13 +136,13 @@ RootDomainUserClient::getTargetAndMethodForIndex( IOService ** targetP, UInt32 i
 {
     static IOExternalMethod sMethods[] = {
         { // kPMSetAggressiveness, 0
-            1, (IOMethod)&RootDomainUserClient::secureSetAggressiveness, kIOUCScalarIScalarO, 2, 1
+            (IOService *)1, (IOMethod)&RootDomainUserClient::secureSetAggressiveness, kIOUCScalarIScalarO, 2, 1
         },
         { // kPMGetAggressiveness, 1
             0, (IOMethod)&IOPMrootDomain::getAggressiveness, kIOUCScalarIScalarO, 1, 1
         },
         { // kPMSleepSystem, 2
-            1, (IOMethod)&RootDomainUserClient::secureSleepSystem, kIOUCScalarIScalarO, 0, 1
+            (IOService *)1, (IOMethod)&RootDomainUserClient::secureSleepSystem, kIOUCScalarIScalarO, 0, 1
         },
         { // kPMAllowPowerChange, 3
             0, (IOMethod)&IOPMrootDomain::allowPowerChange, kIOUCScalarIScalarO, 1, 0
@@ -152,7 +157,7 @@ RootDomainUserClient::getTargetAndMethodForIndex( IOService ** targetP, UInt32 i
             0, (IOMethod)&IOPMrootDomain::restartSystem, kIOUCScalarIScalarO, 0, 0
         },
         { // kPMSetPreventative, 7
-            1, (IOMethod) &RootDomainUserClient::setPreventative, kIOUCScalarIScalarO, 2, 0
+            (IOService *)1, (IOMethod)&RootDomainUserClient::setPreventative, kIOUCScalarIScalarO, 2, 0
         },
     };
     
diff --git a/iokit/Kernel/RootDomainUserClient.h b/iokit/Kernel/RootDomainUserClient.h
index 481f4ed18..548745c13 100644
--- a/iokit/Kernel/RootDomainUserClient.h
+++ b/iokit/Kernel/RootDomainUserClient.h
@@ -61,7 +61,8 @@ public:
 
     virtual bool start( IOService * provider );
 
-    virtual bool initWithTask(task_t owningTask, void *security_id, UInt32);
+    virtual bool initWithTask(task_t owningTask, void *security_id, 
+					UInt32 type, OSDictionary * properties);
 
     void setPreventative(UInt32 on_off, UInt32 types_of_sleep);
 
diff --git a/iokit/KernelConfigTables.cpp b/iokit/KernelConfigTables.cpp
index ad2627acd..71e022fca 100644
--- a/iokit/KernelConfigTables.cpp
+++ b/iokit/KernelConfigTables.cpp
@@ -77,13 +77,5 @@ const char * gIOKernelConfigTables =
 "       'IONameMatch'           = nvram;"
 "   }"
 #endif /* PPC */
-#ifdef i386
-"   ,"
-"   {"
-"       'IOClass'           = AppleIntelClock;"
-"       'IOProviderClass'   = IOPlatformDevice;"
-"       'IONameMatch'       = intel-clock;"
-"   }"
-#endif /* i386 */
 ")";
 
diff --git a/iokit/bsddev/DINetBootHook.cpp b/iokit/bsddev/DINetBootHook.cpp
index 9ba7b5b91..d93e326c1 100644
--- a/iokit/bsddev/DINetBootHook.cpp
+++ b/iokit/bsddev/DINetBootHook.cpp
@@ -8,10 +8,16 @@
  *	Revision History
  *
  *	$Log: DINetBootHook.cpp,v $
+ *	Revision 1.3.1558.1  2005/06/24 01:47:25  lindak
+ *	Bringing over all of the Karma changes into chardonnay.
+ *
+ *	Revision 1.1.1.1  2005/02/24 21:48:06  akosut
+ *	Import xnu-764 from Tiger8A395
+ *	
  *	Revision 1.3  2002/06/16 20:36:02  lindak
  *	Merged PR-2957314 into Jaguar (siegmund: netboot kernel code needs to set
  *	com.apple.AppleDiskImageController.load to boolean Yes)
- *
+ *	
  *	Revision 1.2.40.2  2002/06/15 03:50:38  dieter
  *	- corrected com.apple.AppleDiskImageController.load string
  *	
diff --git a/iokit/bsddev/DINetBootHook.h b/iokit/bsddev/DINetBootHook.h
index ed1f54381..8f2ffe411 100644
--- a/iokit/bsddev/DINetBootHook.h
+++ b/iokit/bsddev/DINetBootHook.h
@@ -8,13 +8,19 @@
  *	Revision History
  *
  *	$Log: DINetBootHook.h,v $
+ *	Revision 1.3.1582.1  2005/06/24 01:47:25  lindak
+ *	Bringing over all of the Karma changes into chardonnay.
+ *
+ *	Revision 1.1.1.1  2005/02/24 21:48:06  akosut
+ *	Import xnu-764 from Tiger8A395
+ *	
  *	Revision 1.3  2002/05/22 18:50:49  aramesh
  *	Kernel API Cleanup
  *	Bug #: 2853781
  *	Changes from Josh(networking), Rick(IOKit), Jim & David(osfmk), Umesh, Dan & Ramesh(BSD)
  *	Submitted by: Ramesh
  *	Reviewed by: Vincent
- *
+ *	
  *	Revision 1.2.12.1  2002/05/21 23:08:14  aramesh
  *	Kernel API Cleanup
  *	Bug #: 2853781
diff --git a/iokit/bsddev/IOKitBSDInit.cpp b/iokit/bsddev/IOKitBSDInit.cpp
index e7ec120be..48d9399a9 100644
--- a/iokit/bsddev/IOKitBSDInit.cpp
+++ b/iokit/bsddev/IOKitBSDInit.cpp
@@ -471,6 +471,14 @@ kern_return_t IOFindBSDRoot( char * rootName,
 
     do {
 	if( (regEntry = IORegistryEntry::fromPath( "/chosen", gIODTPlane ))) {
+            data = OSDynamicCast(OSData, regEntry->getProperty( "root-matching" ));
+            if (data) {
+               matching = OSDynamicCast(OSDictionary, OSUnserializeXML((char *)data->getBytesNoCopy()));
+                if (matching) {
+                    continue;
+                }
+            }
+
 	    data = (OSData *) regEntry->getProperty( "boot-uuid" );
 	    if( data) {
 		uuidStr = (const char*)data->getBytesNoCopy();
@@ -570,10 +578,12 @@ kern_return_t IOFindBSDRoot( char * rootName,
 	// from OpenFirmware path
 	IOLog("From path: \"%s\", ", look);
 
-	if( forceNet || (0 == strncmp( look, "enet", strlen( "enet" ))) ) {
-            matching = IONetworkMatching( look, str, kMaxPathBuf );
-        } else {
-            matching = IODiskMatching( look, str, kMaxPathBuf );
+        if (!matching) {
+            if( forceNet || (0 == strncmp( look, "enet", strlen( "enet" ))) ) {
+                matching = IONetworkMatching( look, str, kMaxPathBuf );
+            } else {
+                matching = IODiskMatching( look, str, kMaxPathBuf );
+            }
         }
     }
     
@@ -675,7 +685,7 @@ kern_return_t IOFindBSDRoot( char * rootName,
         IOService * subservice = IOFindMatchingChild( service );
         if ( subservice ) service = subservice;
     } else if ( service && mediaProperty ) {
-        service = service->getProperty(mediaProperty);
+        service = (IOService *)service->getProperty(mediaProperty);
     }
 
     major = 0;
diff --git a/iokit/conf/Makefile.i386 b/iokit/conf/Makefile.i386
index 1f8be4145..029888c26 100644
--- a/iokit/conf/Makefile.i386
+++ b/iokit/conf/Makefile.i386
@@ -10,18 +10,33 @@ CWARNFLAGS= $(filter-out -Wbad-function-cast, $(CWARNFLAGS_STD))
 OBJS_NO_WERROR=				\
 	ioconf.o			\
 	UNDRequest.o			\
-	IOLib.o				\
+	IOLib.cpo				\
 	IOStringFuncs.o			\
+	IOCatalogue.cpo		\
 	IOCPU.cpo			\
+	IOCommandGate.cpo		\
 	IOCommandPool.cpo		\
 	IOCommandQueue.cpo		\
-	IOKitBSDInit.cpo		\
+	IOConditionLock.cpo		\
+	IOFilterInterruptEventSource.cpo	\
+	IOHibernateIO.cpo		\
 	IOInterruptController.cpo	\
 	IOInterruptEventSource.cpo	\
+	IOKitBSDInit.cpo		\
+	IOMapper.cpo			\
+	IOMemoryCursor.cpo		\
+	IOMemoryDescriptor.cpo	\
+	IOPlatformExpert.cpo	\
 	IOPMPowerStateQueue.cpo		\
 	IOPMchangeNoteList.cpo		\
 	IOPMrootDomain.cpo		\
+	IORangeAllocator.cpo	\
+	IORegistryEntry.cpo		\
+	IOService.cpo			\
 	IOServicePM.cpo			\
+	IOSyncer.cpo			\
+	IOTimerEventSource.cpo	\
+	IOUserClient.cpo		\
 	IOWorkLoop.cpo			\
 	RootDomainUserClient.cpo
  
diff --git a/iokit/conf/files b/iokit/conf/files
index ab719eb34..45c5cc070 100644
--- a/iokit/conf/files
+++ b/iokit/conf/files
@@ -11,7 +11,7 @@ iokit/Kernel/WKdmDecompress.c				optional iokitcpp
 iokit/Kernel/IOHibernateIO.cpp				optional iokitcpp
 iokit/Kernel/IOHibernateRestoreKernel.c			optional iokitcpp
 
-iokit/Kernel/IOLib.c					optional iokitcpp
+iokit/Kernel/IOLib.cpp					optional iokitcpp
 iokit/Kernel/IOLocks.cpp				optional iokitcpp
 iokit/Kernel/IOConditionLock.cpp			optional iokitcpp
 iokit/Kernel/IOSyncer.cpp				optional iokitcpp
@@ -21,6 +21,7 @@ iokit/Kernel/IOStartIOKit.cpp				optional iokitcpp
 
 iokit/Kernel/IODeviceTreeSupport.cpp			optional iokitcpp
 
+iokit/Kernel/IOPMWorkArbiter.cpp                        optional iokitcpp
 iokit/Kernel/IORegistryEntry.cpp			optional iokitcpp
 iokit/Kernel/IOService.cpp				optional iokitcpp
 iokit/Kernel/IOServicePM.cpp				optional iokitcpp
@@ -42,11 +43,14 @@ iokit/Kernel/IOCommandQueue.cpp				optional iokitcpp
 iokit/Kernel/IOFilterInterruptEventSource.cpp		optional iokitcpp
 iokit/Kernel/IOTimerEventSource.cpp			optional iokitcpp
 
+# Memory system
+iokit/Kernel/IOBufferMemoryDescriptor.cpp		optional iokitcpp
+iokit/Kernel/IODMACommand.cpp				optional iokitcpp
 iokit/Kernel/IODeviceMemory.cpp				optional iokitcpp
 iokit/Kernel/IOMapper.cpp				optional iokitcpp
-iokit/Kernel/IOMemoryDescriptor.cpp			optional iokitcpp
+iokit/Kernel/IOCopyMapper.cpp				optional iokitcpp
 iokit/Kernel/IOMemoryCursor.cpp				optional iokitcpp
-iokit/Kernel/IOBufferMemoryDescriptor.cpp		optional iokitcpp
+iokit/Kernel/IOMemoryDescriptor.cpp			optional iokitcpp
 iokit/Kernel/IOMultiMemoryDescriptor.cpp		optional iokitcpp
 iokit/Kernel/IORangeAllocator.cpp			optional iokitcpp
 
diff --git a/iokit/conf/files.i386 b/iokit/conf/files.i386
index a604778e1..2193ae37a 100644
--- a/iokit/conf/files.i386
+++ b/iokit/conf/files.i386
@@ -4,8 +4,5 @@
 iokit/Kernel/i386/IOSharedLock.s                            standard
 iokit/Kernel/i386/IOAsmSupport.s                            standard
 
-# Real Time Clock hack
-iokit/Drivers/platform/drvAppleIntelClock/IntelClock.cpp    optional iokitcpp
-
 # Power Domains
 iokit/Kernel/IOPMrootDomain.cpp    				optional iokitcpp
diff --git a/iokit/include/architecture/i386/pio.h b/iokit/include/architecture/i386/pio.h
deleted file mode 100644
index 9dd02c03e..000000000
--- a/iokit/include/architecture/i386/pio.h
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the License
- * may not be used to create, or enable the creation or redistribution of,
- * unlawful or unlicensed copies of an Apple operating system, or to
- * circumvent, violate, or enable the circumvention or violation of, any
- * terms of an Apple operating system software license agreement.
- * 
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- * 
- * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/*
- * HISTORY
- * 
- * Revision 1.1.2.1  1998/10/13 00:40:44  ehewitt
- * Added support for Intel.
- *
- * Revision 1.1.1.1  1998/09/22 21:05:37  wsanchez
- * Import of Mac OS X kernel (~semeria)
- *
- * Revision 1.1.1.1  1998/03/07 02:25:38  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.1.8.2  1996/07/31  09:46:36  paire
- * 	Merged with nmk20b7_shared (1.1.11.2 -> 1.1.11.1)
- * 	[96/06/10            paire]
- *
- * Revision 1.1.11.2  1996/06/13  12:38:25  bernadat
- * 	Do not use inline macros when MACH_ASSERT is configured.
- * 	[96/05/24            bernadat]
- * 
- * Revision 1.1.11.1  1996/05/14  13:50:23  paire
- * 	Added new linl and loutl __inline__.
- * 	Added conditional compilation for [l]{in|oub}[bwl]() __inline__.
- * 	[95/11/24            paire]
- * 
- * Revision 1.1.8.1  1994/09/23  02:00:28  ezf
- * 	change marker to not FREE
- * 	[1994/09/22  21:25:52  ezf]
- * 
- * Revision 1.1.4.5  1993/08/09  19:40:41  dswartz
- * 	Add ANSI prototypes - CR#9523
- * 	[1993/08/06  17:45:57  dswartz]
- * 
- * Revision 1.1.4.4  1993/06/11  15:17:37  jeffc
- * 	CR9176 - ANSI C violations: inb/outb macros must be changed from
- * 	({ ... }) to inline functions, with proper type definitions. Callers
- * 	must pass proper types to these functions: 386 I/O port addresses
- * 	are unsigned shorts (not pointers).
- * 	[1993/06/10  14:26:10  jeffc]
- * 
- * Revision 1.1.4.3  1993/06/07  22:09:28  jeffc
- * 	CR9176 - ANSI C violations: trailing tokens on CPP
- * 	directives, extra semicolons after decl_ ..., asm keywords
- * 	[1993/06/07  19:00:26  jeffc]
- * 
- * Revision 1.1.4.2  1993/06/04  15:28:45  jeffc
- * 	CR9176 - ANSI problems -
- * 	Added casts to get macros to take caddr_t as an I/O space address.
- * 	[1993/06/04  13:45:55  jeffc]
- * 
- * Revision 1.1  1992/09/30  02:25:51  robert
- * 	Initial revision
- * 
- * $EndLog$
- */
-/* CMU_HIST */
-/*
- * Revision 2.5  91/05/14  16:14:20  mrt
- * 	Correcting copyright
- * 
- * Revision 2.4  91/02/05  17:13:56  mrt
- * 	Changed to new Mach copyright
- * 	[91/02/01  17:37:08  mrt]
- * 
- * Revision 2.3  90/12/20  16:36:37  jeffreyh
- * 	changes for __STDC__
- * 	[90/12/07            jeffreyh]
- * 
- * Revision 2.2  90/11/26  14:48:41  rvb
- * 	Pulled from 2.5
- * 	[90/11/22  10:09:38  rvb]
- * 
- * 	[90/08/14            mg32]
- * 
- * 	Now we know how types are factor in.
- * 	Cleaned up a bunch: eliminated ({ for output and flushed unused
- * 	output variables.
- * 	[90/08/14            rvb]
- * 
- * 	This is how its done in gcc:
- * 		Created.
- * 	[90/03/26            rvb]
- * 
- */
-/* CMU_ENDHIST */
-/* 
- * Mach Operating System
- * Copyright (c) 1991,1990 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
-/* 
- */
-#ifndef I386_PIO_H
-#define I386_PIO_H
-//#include <cpus.h>
-//#include <mach_assert.h>
-#define MACH_ASSERT 0
-
-typedef unsigned short i386_ioport_t;
-
-/* read a longword */
-extern unsigned long	inl(
-				i386_ioport_t	port);
-/* read a shortword */
-extern unsigned short	inw(
-				i386_ioport_t	port);
-/* read a byte */
-extern unsigned char	inb(
-				i386_ioport_t	port);
-/* write a longword */
-extern void		outl(
-				i386_ioport_t	port,
-				unsigned long	datum);
-/* write a word */
-extern void		outw(
-				i386_ioport_t	port,
-				unsigned short	datum);
-/* write a longword */
-extern void		outb(
-				i386_ioport_t	port,
-				unsigned char	datum);
-
-/* input an array of longwords */
-extern void		linl(
-				i386_ioport_t	port,
-				int		* data,
-				int		count);
-/* output an array of longwords */
-extern void		loutl(
-				i386_ioport_t	port,
-				int		* data,
-				int		count);
-
-/* input an array of words */
-extern void		linw(
-				i386_ioport_t	port,
-				int		* data,
-				int		count);
-/* output an array of words */
-extern void		loutw(
-				i386_ioport_t	port,
-				int		* data,
-				int		count);
-
-/* input an array of bytes */
-extern void		linb(
-				i386_ioport_t	port,
-				char		* data,
-				int		count);
-/* output an array of bytes */
-extern void		loutb(
-				i386_ioport_t	port,
-				char		* data,
-				int		count);
-
-#if defined(__GNUC__) && (!MACH_ASSERT)
-extern __inline__ unsigned long	inl(
-				i386_ioport_t port)
-{
-	unsigned long datum;
-	__asm__ volatile("inl %1, %0" : "=a" (datum) : "d" (port));
-	return(datum);
-}
-
-extern __inline__ unsigned short inw(
-				i386_ioport_t port)
-{
-	unsigned short datum;
-	__asm__ volatile(".byte 0x66; inl %1, %0" : "=a" (datum) : "d" (port));
-	return(datum);
-}
-
-extern __inline__ unsigned char inb(
-				i386_ioport_t port)
-{
-	unsigned char datum;
-	__asm__ volatile("inb %1, %0" : "=a" (datum) : "d" (port));
-	return(datum);
-}
-
-extern __inline__ void outl(
-				i386_ioport_t port,
-				unsigned long datum)
-{
-	__asm__ volatile("outl %0, %1" : : "a" (datum), "d" (port));
-}
-
-extern __inline__ void outw(
-				i386_ioport_t port,
-				unsigned short datum)
-{
-	__asm__ volatile(".byte 0x66; outl %0, %1" : : "a" (datum), "d" (port));
-}
-
-extern __inline__ void outb(
-				i386_ioport_t port,
-				unsigned char datum)
-{
-	__asm__ volatile("outb %0, %1" : : "a" (datum), "d" (port));
-}
-#endif /* defined(__GNUC__) && (!MACH_ASSERT) */
-#endif /* I386_PIO_H */
diff --git a/kgmacros b/kgmacros
index 2356ffe9e..d1a666c74 100644
--- a/kgmacros
+++ b/kgmacros
@@ -6,26 +6,8 @@
 #
 #  All the convenience variables used by these macros begin with $kgm_
 
-define showversion
-#Display version string, a pointer to which is pinned at 0x501C in the kernel's
-#low memory globals
-       p (char *) *0x501c
-end
-
-document showversion
-Syntax: showversion
-| Read the kernel version string from a fixed address in low
-| memory. Useful if you don't know which kernel is on the other end,
-| and need to find the appropriate symbols. Beware that if you've
-| loaded a symbol file, but aren't connected to a remote target,
-| the version string from the symbol file will be displayed instead.
-| This macro expects to be connected to the remote kernel to function
-| correctly.
-end
-
-set $kgm_dummy = &proc0
-set $kgm_dummy = &kmod
-set $kgm_mtype = ((struct mach_header)_mh_execute_header).cputype
+set print asm-demangle on
+set cp-abi gnu-v2
 
 echo Loading Kernel GDB Macros package.  Type "help kgm" for more info.\n
 
@@ -54,7 +36,13 @@ document kgm
 |     showallipc     Display a summary listing of all the ipc spaces
 |     showallrights  Display a summary listing of all the ipc rights
 |     showallkmods   Display a summary listing of all the kernel modules
-|     showallclasses Display info about all OSObject subclasses in the system
+|
+|     showallclasses    Display info about all OSObject subclasses in the system
+|     showobject        Show info about an OSObject - its vtable ptr and retain count, & more info for simple container classes.
+|     showregistry      Show info about all registry entries in the current plane
+|     showregistryprops Show info about all registry entries in the current plane, and their properties
+|     showregistryentry Show info about a registry entry; its properties and descendants in the current plane
+|     setregistryplane  Set the plane to be used for the iokit registry macros (pass zero for list)
 |
 |     showtask       Display info about the specified task
 |     showtaskthreads      Display info about the threads in the task
@@ -104,13 +92,45 @@ document kgm
 |     disablecore    Configure the kernel to disable coredump transmission
 |     switchtocorethread Corefile version of "switchtoact"
 |     resetcorectx   Corefile version of "resetctx"
-|     
+|
+|     readphys       Reads the specified untranslated address
+|     readphys64     Reads the specified untranslated 64-bit address
+|
 |     kdp-reboot     Restart remote target
 |	
 | Type "help <macro>" for more specific help on a particular macro.
 | Type "show user <macro>" to see what the macro is really doing.
 end
 
+# This macro should appear before any symbol references, to facilitate
+# a gdb "source" without a loaded symbol file.
+define showversion
+       printf "%s\n", *(char **)0x501C
+end
+
+document showversion
+Syntax: showversion
+| Read the kernel version string from a fixed address in low
+| memory. Useful if you don't know which kernel is on the other end,
+| and need to find the appropriate symbols. Beware that if you've
+| loaded a symbol file, but aren't connected to a remote target,
+| the version string from the symbol file will be displayed instead.
+| This macro expects to be connected to the remote kernel to function
+| correctly.
+end
+
+set $kgm_dummy = &proc0
+set $kgm_dummy = &kmod
+set $kgm_mtype = ((struct mach_header)_mh_execute_header).cputype
+
+set $kgm_reg_depth = 0
+set $kgm_reg_plane = (void **) gIOServicePlane
+set $kgm_namekey = (OSSymbol *) 0
+set $kgm_childkey = (OSSymbol *) 0
+
+set $kgm_show_object_addrs = 0
+set $kgm_show_object_retain = 0
+set $kgm_show_props = 0
 
 define showkmodheader
     printf "kmod        address     size        "
@@ -135,7 +155,7 @@ set $kgm_fkmodmax = 0xffffffff
 set $kgm_pkmod = 0
 set $kgm_pkmodst = 0
 set $kgm_pkmoden = 0
-define showkmodaddr
+define showkmodaddrint
     printf "0x%x" , $arg0
     if ((unsigned int)$arg0 >= (unsigned int)$kgm_pkmodst) && ((unsigned int)$arg0 <= (unsigned int)$kgm_pkmoden)
 	set $kgm_off = ((unsigned int)$arg0 - (unsigned int)$kgm_pkmodst)
@@ -149,7 +169,7 @@ define showkmodaddr
 		    set $kgm_kmodmin = $kgm_kmod.address
 		end
 		if ($kgm_kmod.address + $kgm_kmod.size) > $kgm_kmodmax
-	    	    set $kgm_kmodmax = $kgm_kmod.address
+	    	    set $kgm_kmodmax = $kgm_kmod.address + $kgm_kmod.size
 	    	end
 	        set $kgm_off = ((unsigned int)$arg0 - (unsigned int)$kgm_kmod.address)
 	        if ($kgm_kmod.address <= $arg0) && ($kgm_off <= $kgm_kmod.size)
@@ -169,6 +189,12 @@ define showkmodaddr
 	end
     end
 end
+
+define showkmodaddr
+    showkmodaddrint $arg0
+    printf "\n"
+end
+
 document showkmodaddr
 | Given an address, print the offset and name for the kmod containing it
 | The following is the syntax:
@@ -235,10 +261,10 @@ define showactint
    	if $kgm_state & 0x01
 	    printf "W\t" 
 	    printf "0x%08x  ", $kgm_thread.wait_queue
-		
-		if ((unsigned)$kgm_thread.wait_event > (unsigned)sectPRELINKB)
-		    showkmodaddr $kgm_thread.wait_event
-		else
+	    	if (((unsigned)$kgm_thread.wait_event > (unsigned)sectPRELINKB) \
+		    && ($arg1 != 2))
+			showkmodaddr $kgm_thread.wait_event
+	        else
 			output /a (unsigned) $kgm_thread.wait_event
 		end
 	end
@@ -251,37 +277,45 @@ define showactint
 		if ($kgm_mtype == 18)
 			set $mysp = $kgm_thread.machine.pcb->save_r1
 		else
-			set $kgm_statep = (struct i386_kernel_state *) \
+			set $kgm_statep = (struct x86_kernel_state32 *) \
 				($kgm_thread->kernel_stack + 0x4000 \
-				 - sizeof(struct i386_kernel_state))
+				 - sizeof(struct x86_kernel_state32))
 			set $mysp = $kgm_statep->k_ebp
 		end
-		set $prevsp = 0
+		set $prevsp = $mysp - 16
 		printf "\n\t\tstacktop=0x%08x", $mysp
 		if ($kgm_mtype == 18)
 			set $stkmask = 0xf
-			set $stklimit = 0xb0000000
 		else
 			set $stkmask = 0x3
-			set $stklimit = 0xfc000000
 		end
+		set $kgm_return = 0
 	    	while ($mysp != 0) && (($mysp & $stkmask) == 0) \
-		       && ($mysp < $stklimit) \
-		       && ((unsigned)$mysp > (unsigned)$prevsp)
+		      && ($mysp != $prevsp) \
+		      && ((((unsigned) $mysp ^ (unsigned) $prevsp) < 0x2000) \
+		      || (((unsigned)$mysp < ((unsigned) ($kgm_thread->kernel_stack+0x4000))) \
+		      && ((unsigned)$mysp > (unsigned) ($kgm_thread->kernel_stack))))
+
+			if ((unsigned) $kgm_return > (unsigned) sectPRELINKB)
+			    showkmodaddr $kgm_return
+			else
+				if ((unsigned) $kgm_return > 0)
+				    output /a (unsigned) $kgm_return
+				end
+			end
 			printf "\n\t\t0x%08x  ", $mysp
 			if ($kgm_mtype == 18)
 				set $kgm_return = *($mysp + 8)
 			else
 				set $kgm_return = *($mysp + 4)
 			end
-			if ((unsigned) $kgm_return > (unsigned) sectPRELINKB)
-			    showkmodaddr $kgm_return
-			else
-			    output /a (unsigned) $kgm_return
-			end
 			set $prevsp = $mysp
 			set $mysp = * $mysp
 		end
+		if ((unsigned) $kgm_return > 0)
+		   output/a $kgm_return
+		end
+		set $kgm_return = 0
 		printf "\n\t\tstackbottom=0x%08x", $prevsp
 	    else
 		printf "\n\t\t\tcontinuation="
@@ -358,6 +392,7 @@ document showcurrentthreads
 |     (gdb) showcurrentthreads
 end
 
+set $decode_wait_events = 0
 define showallstacks
     set $kgm_head_taskp = &default_pset.tasks
     set $kgm_taskp = (struct task *)($kgm_head_taskp->next)
@@ -368,17 +403,25 @@ define showallstacks
         set $kgm_actp = (struct thread *)($kgm_taskp->threads.next)
 	while $kgm_actp != $kgm_head_actp
 	    showactheader
-	    showactint $kgm_actp 1
+	    if ($decode_wait_events > 0)
+	       showactint $kgm_actp 1
+	    else
+	       showactint $kgm_actp 2
+	    end
   	    set $kgm_actp = (struct thread *)($kgm_actp->task_threads.next)
         end
 	printf "\n"
     	set $kgm_taskp = (struct task *)($kgm_taskp->pset_tasks.next)
     end
 end
+
 document showallstacks
 | Routine to print out the stack for each thread in the system.
 | The following is the syntax:
 |     (gdb) showallstacks
+| If the variable $decode_wait_events is non-zero, the routine attempts to
+| interpret thread wait_events as kernel module offsets, which can add to
+| processing time.
 end
 
 define showcurrentstacks
@@ -395,6 +438,7 @@ set $kgm_prp = processor_list
 	set $kgm_prp = ($kgm_prp)->processor_list
     end
 end
+
 document showcurrentstacks
 | Routine to print out the thread running on each cpu (incl. its stack)
 | The following is the syntax:
@@ -1426,25 +1470,44 @@ end
 set $kdp_act_counter = 0
 
 define switchtoact
+	set $newact = (struct thread *) $arg0
+	if ($newact->kernel_stack == 0)
+		echo This activation does not have a stack.\n
+		echo continuation:
+		output/a (unsigned) $newact.continuation
+		echo \n
+	else
 	if ($kgm_mtype == 18)
 		if ($kdp_act_counter == 0)
-		set $kdpstate = (struct savearea *) kdp.saved_state
+			set $kdpstate = (struct savearea *) kdp.saved_state
 		end
 		set $kdp_act_counter = $kdp_act_counter + 1
 		set $newact = (struct thread *) $arg0
-		if ($newact->kernel_stack == 0)
-		   echo This activation does not have a stack.\n
-		   echo continuation:
-		   output/a (unsigned) $newact.continuation
-		   echo \n
-		else
-		   set (struct savearea *) kdp.saved_state=$newact->machine->pcb
-		   flush
-		   set $pc=$newact->machine->pcb.save_srr0
-		   update
-		end
+		set (struct savearea *) kdp.saved_state=$newact->machine->pcb
+		flushregs
+		flushstack
+		set $pc=$newact->machine->pcb.save_srr0
+		update
 	else
-		echo switchtoact not implemented for this architecture.\n
+		set $kdpstatep = (struct x86_saved_state32 *) kdp.saved_state
+		if ($kdp_act_counter == 0)
+			set $kdpstate = *($kdpstatep)
+		end	
+		set $kdp_act_counter = $kdp_act_counter + 1
+
+		set $kgm_statep = (struct x86_kernel_state32 *) \
+					($newact->kernel_stack + 0x4000 \
+					 - sizeof(struct x86_kernel_state32))
+		set $kdpstatep->ebx = $kgm_statep->k_ebx 
+		set $kdpstatep->ebp = $kgm_statep->k_ebp 
+		set $kdpstatep->edi = $kgm_statep->k_edi 
+		set $kdpstatep->esi = $kgm_statep->k_esi 
+		set $kdpstatep->eip = $kgm_statep->k_eip 
+		flushregs
+		flushstack
+		set $pc = $kgm_statep->k_eip
+		update
+	end
 	end
 end
 
@@ -1464,7 +1527,8 @@ define switchtoctx
 		end
 		set $kdp_act_counter = $kdp_act_counter + 1
 		set (struct savearea *) kdp.saved_state=(struct savearea *) $arg0
-		flush
+		flushregs
+		flushstack
 		set $pc=((struct savearea *) $arg0)->save_srr0
 		update
 	else
@@ -1483,12 +1547,19 @@ end
 define resetctx
 	if ($kgm_mtype == 18)
 		set (struct savearea *)kdp.saved_state=$kdpstate
-		flush
+		flushregs
+		flushstack
 		set $pc=((struct savearea *) kdp.saved_state)->save_srr0
 		update
 		set $kdp_act_counter = 0
 	else
-		echo resetctx not implemented for this architecture.\n
+		set $kdpstatep = (struct x86_saved_state32 *) kdp.saved_state
+		set *($kdpstatep)=$kdpstate
+		flushregs
+		flushstack
+		set $pc=$kdpstatep->eip
+		update
+		set $kdp_act_counter = 0
 	end
 end     
         
@@ -1600,7 +1671,8 @@ end
 define _kgm_flush_loop
        set $kgm_flush_loop_ctr = 0
        while ($kgm_flush_loop_ctr < 30)
-       	     flush
+       	     flushregs
+	     flushstack
 	     set $kgm_flush_loop_ctr = $kgm_flush_loop_ctr + 1
        end
 end
@@ -1613,6 +1685,41 @@ define _kgm_update_loop
        end
 end
 
+#This is necessary since gdb often doesn't do backtraces on x86 correctly
+#in the absence of symbols.The code below in showuserstack and 
+#showx86backtrace also contains several workarouds for the gdb bug where 
+#gdb stops macro evaluation because of spurious "Cannot read memory"
+#errors on x86. These errors appear on ppc as well, but they don't
+#always stop macro evaluation.
+
+set $kgm_cur_ebp = 0
+set $kgm_cur_eip = 0
+
+define showx86backtrace
+	if ($kgm_cur_ebp == 0)
+		set $kgm_cur_ebp = $ebp
+	end
+	if ($kgm_cur_eip == 0)
+		set $kgm_cur_eip = $eip
+	end
+	printf "0: EBP: 0x%08x EIP: 0x%08x\n", $kgm_cur_ebp, $kgm_cur_eip
+	x/i $kgm_cur_eip
+	set $kgm_prev_ebp = *((uint32_t *) $kgm_cur_ebp)
+	set $kgm_prev_eip = *((uint32_t *) ($kgm_cur_ebp + 4))
+	set $kgm_frameno = 1
+	while $kgm_prev_ebp != 0
+		printf "%d: saved EBP: 0x%08x saved EIP: 0x%08x\n", $kgm_frameno, $kgm_prev_ebp, $kgm_prev_eip
+		x/i $kgm_prev_eip
+		set $kgm_cur_ebp = $kgm_prev_ebp
+		set $kgm_prev_ebp = *((uint32_t *) $kgm_cur_ebp)
+		set $kgm_prev_eip = *((uint32_t *) ($kgm_cur_ebp + 4))
+		set $kgm_frameno = $kgm_frameno + 1
+	end
+	set $kgm_cur_ebp = 0
+	set $kgm_cur_eip = 0
+	set kdp_pmap = 0
+end
+
 define showuserstack
   		if ($kgm_mtype == 18)	
 		   if ($kdp_act_counter == 0)
@@ -1644,10 +1751,22 @@ define showuserstack
 		     _kgm_update_loop
 		   end
 		else
-		echo showuserstack not implemented for this architecture.\n
-	end		
+			set $newact = (struct thread *) $arg0
+			set $newiss = (x86_saved_state32_t *) ($newact->machine.pcb->iss)
+			set $checkpc = $newiss.eip
+			if ($checkpc == 0)
+			    echo This activation does not appear to have
+			    echo \20 a valid user context.\n
+			else
+			set $kgm_cur_ebp = $newiss.ebp
+			set $kgm_cur_eip = $checkpc
+			printf "You may now issue the showx86backtrace command to see the user space backtrace for this thread (0x%08x); you can also examine memory locations in this address space (pmap 0x%08x) before issuing the backtrace. This two-step process is necessary to work around various bugs in x86 gdb, which cause it to stop memory evaluation on spurious memory read errors. Additionally, you may need to issue a set kdp_pmap = 0 command after the showx86backtrace completes, to resume reading from the kernel address space.\n", $arg0, $newact->task->map->pmap
+			set kdp_pmap = $newact->task->map->pmap
+			_kgm_flush_loop
+			_kgm_update_loop
+			end			
+		end
 end
-
 document showuserstack
 Syntax: showuserstack <address of thread activation>
 |This command displays a numeric backtrace for the user space stack of
@@ -1843,9 +1962,9 @@ define showgdbthread
 		if ($kgm_mtype == 18)
 			set $mysp = $kgm_thread.machine.pcb->save_r1
 		else
-			set $kgm_statep = (struct i386_kernel_state *) \
+			set $kgm_statep = (struct x86_kernel_state32 *) \
 				($kgm_thread->kernel_stack + 0x4000 \
-				 - sizeof(struct i386_kernel_state))
+				 - sizeof(struct x86_kernel_state32))
 			set $mysp = $kgm_statep->k_ebp
 		end
 		set $prevsp = 0
@@ -1955,22 +2074,429 @@ Syntax: switchtouserthread <address of thread>
 end
 
 define showmetaclass
-    set cp-abi gnu-v2
     set $kgm_metaclassp = (OSMetaClass *)$arg0
     printf "%-5d", $kgm_metaclassp->instanceCount
     printf "x %5d bytes", $kgm_metaclassp->classSize
     printf " %s\n", $kgm_metaclassp->className->string
 end
 
-define showallclasses
-    set cp-abi gnu-v2
+define showstring
+    printf "\"%s\"", ((OSString *)$arg0)->string
+end
+
+define shownumber
+    printf "%lld", ((OSNumber *)$arg0)->value
+end
+
+define showboolean
+    if ($arg0 == gOSBooleanFalse)
+	printf "No"
+    else
+	printf "Yes"
+    end
+end
+
+define showdata
+    set $kgm_data = (OSData *)$arg0
+
+    printf "<"
+    set $kgm_datap = (const unsigned char *) $kgm_data->data
+
+    set $kgm_printstr = 0
+    if (0 == (3 & (unsigned int)$kgm_datap) && ($kgm_data->length >= 3))
+	set $kgm_bytes = *(unsigned int *) $kgm_datap
+	if (0xffff0000 & $kgm_bytes)
+	    set $kgm_idx = 0
+	    set $kgm_printstr = 1
+	    while ($kgm_idx++ < 4)
+		set $kgm_bytes = $kgm_bytes >> 8
+		set $kgm_char = 0xff & $kgm_bytes
+		if ($kgm_char && (($kgm_char < 0x20) || ($kgm_char > 0x7e)))
+		    set $kgm_printstr = 0
+		end
+	    end
+	end
+    end
+    
+    set $kgm_idx = 0
+    if ($kgm_printstr)
+	set $kgm_quoted = 0
+	while ($kgm_idx < $kgm_data->length)
+	    set $kgm_char = $kgm_datap[$kgm_idx++]
+	    if ($kgm_char)
+		if (0 == $kgm_quoted)
+		    set $kgm_quoted = 1
+		    if ($kgm_idx > 1)
+			printf ",\""
+		    else
+			printf "\""
+		    end
+		end
+		printf "%c", $kgm_char
+	    else
+		if ($kgm_quoted)
+		    set $kgm_quoted = 0
+		    printf "\""
+		end
+	    end
+	end
+	if ($kgm_quoted)
+	    printf "\""
+	end
+    else
+	if (0 == (3 & (unsigned int)$kgm_datap))
+	    while (($kgm_idx + 3) <= $kgm_data->length)
+		printf "%08x", *(unsigned int *) &$kgm_datap[$kgm_idx]
+		set $kgm_idx = $kgm_idx + 4
+	    end
+	end
+	while ($kgm_idx < $kgm_data->length)
+	    printf "%02x", $kgm_datap[$kgm_idx++]
+	end
+    end
+    printf ">"
+end
+
+define showdictionaryint
+    set $kgm$arg0_dict = (OSDictionary *)$arg1
+
+    printf "{"
+    set $kgm$arg0_idx = 0
+    while ($kgm$arg0_idx < $kgm$arg0_dict->count)
+	set $kgm_obj = $kgm$arg0_dict->dictionary[$kgm$arg0_idx].key
+	showobjectint _$arg0 $kgm_obj
+	printf "="
+	set $kgm_obj = $kgm$arg0_dict->dictionary[$kgm$arg0_idx++].value
+	showobjectint _$arg0 $kgm_obj
+	if ($kgm$arg0_idx < $kgm$arg0_dict->count)
+	    printf ","
+	end
+    end
+    printf "}"
+end
+
+define indent
+    set $kgm_idx = 0
+    while ($kgm_idx < $arg0)
+	if ($arg1 & (1 << $kgm_idx++))
+	    printf "| "
+	else
+	    printf "  "
+	end
+    end
+end
+
+define showregdictionary
+    indent $kgm_reg_depth+2 $arg1
+    printf "{\n"
+
+    set $kgm_reg_idx = 0
+    while ($kgm_reg_idx < $arg0->count)
+	indent $kgm_reg_depth+2 $arg1
+	printf "  "
+	set $kgm_obj = $arg0->dictionary[$kgm_reg_idx].key
+	showobjectint _ $kgm_obj
+	printf " = "
+
+	set $kgm_obj = $arg0->dictionary[$kgm_reg_idx++].value
+	showobjectint _ $kgm_obj
+	printf "\n"
+    end
+    indent $kgm_reg_depth+2 $arg1
+    printf "}\n"
+end
+
+
+define showarraysetint
+    set $kgm$arg0_array = (OSArray *)$arg1
+
+    set $kgm$arg0_idx = 0
+    while ($kgm$arg0_idx < $kgm$arg0_array->count)
+	set $kgm_obj = $kgm$arg0_array->array[$kgm$arg0_idx++]
+	showobjectint _$arg0 $kgm_obj
+	if ($kgm$arg0_idx < $kgm$arg0_array->count)
+	    printf ","
+	end
+    end
+end
+
+define showarrayint
+    printf "("
+    showarraysetint $arg0 $arg1
+    printf ")"
+end
+
+define showsetint
+    set $kgm_array = ((OSSet *)$arg1)->members
+    printf "["
+    showarraysetint $arg0 $kgm_array
+    printf "]"
+end
+
+
+define showobjectint
+    set $kgm_obj = (OSObject *) $arg1
+    set $kgm_vt = *((void **) $arg1)
+
+    if ($kgm_show_object_addrs)
+	printf "`object %p, vt ", $arg1
+	output /a (unsigned) $kgm_vt
+	if ($kgm_show_object_retain)
+	    printf ", retain count %d, container retain %d", (0xffff & $kgm_obj->retainCount), $kgm_obj->retainCount >> 16
+	end
+	printf "` "
+    end
+
+    if ($kgm_vt == _ZTV8OSString)
+	showstring $arg1
+    else
+	if ($kgm_vt == _ZTV8OSSymbol)
+	    showstring $arg1
+	else
+	    if ($kgm_vt == _ZTV8OSNumber)
+		shownumber $arg1
+	    else
+		if ($kgm_vt == _ZTV6OSData)
+		    showdata $arg1
+		else
+		    if ($kgm_vt == _ZTV9OSBoolean)
+			showboolean $arg1
+		    else
+			if ($kgm_vt == _ZTV12OSDictionary)
+			    showdictionaryint _$arg0 $arg1
+			else
+			    if ($kgm_vt == _ZTV7OSArray)
+				showarrayint _$arg0 $arg1
+			    else
+				if ($kgm_vt == _ZTV5OSSet)
+				    showsetint _$arg0 $arg1
+				else
+				    if ($kgm_show_object_addrs == 0)
+					printf "`object %p, vt ", $arg1
+					output /a (unsigned) $kgm_vt
+					printf "`"
+				    end
+				end
+			    end
+			end
+		    end
+		end
+	    end
+	end
+    end
+end
+
+define showobject
+    set $kgm_save = $kgm_show_object_addrs
+    set $kgm_show_object_addrs = 1
+    set $kgm_show_object_retain = 1
+    showobjectint _ $arg0
+    set $kgm_show_object_addrs = $kgm_save
+    set $kgm_show_object_retain = 0
+    printf "\n"
+end
+document showobject
+| Show info about an OSObject - its vtable ptr and retain count.
+| If the object is a simple container class, more info will be shown.
+| The following is the syntax:
+|     (gdb) showobject <object address>
+end
+
+define dictget 
+    set $kgm_dictp = (OSDictionary *)$arg0
+    set $kgm_keyp = (const OSSymbol *)$arg1
+    set $kgm_idx = 0
+    set $kgm_result = 0
+    while (($kgm_idx < $kgm_dictp->count) && ($kgm_result == 0))
+	if ($kgm_keyp == $kgm_dictp->dictionary[$kgm_idx].key)
+	    set $kgm_result = $kgm_dictp->dictionary[$kgm_idx].value
+	end
+	set $kgm_idx = $kgm_idx + 1
+    end
+end
+
+
+define showregistryentryrecurse
+    set $kgm_re         = (IOService *)$arg1
+    set $kgm$arg0_stack = (unsigned long long) $arg2
+
+    if ($arg3)
+	set $kgm$arg0_stack = $kgm$arg0_stack | (1ULL << $kgm_reg_depth)
+    else
+	set $kgm$arg0_stack = $kgm$arg0_stack & ~(1ULL << $kgm_reg_depth)
+    end
+
+    dictget $kgm_re->fRegistryTable $kgm_childkey
+    set $kgm$arg0_child_array = (OSArray *) $kgm_result
+
+    if ($kgm$arg0_child_array)
+	set $kgm$arg0_child_count = $kgm$arg0_child_array->count
+    else
+	set $kgm$arg0_child_count = 0
+    end
+
+    if ($kgm$arg0_child_count)
+	set $kgm$arg0_stack = $kgm$arg0_stack | (2ULL << $kgm_reg_depth)
+    else
+	set $kgm$arg0_stack = $kgm$arg0_stack & ~(2ULL << $kgm_reg_depth)
+    end
+
+    indent $kgm_reg_depth $kgm$arg0_stack
+    printf "+-o "
+
+    dictget $kgm_re->fRegistryTable $kgm_namekey
+    if ($kgm_result == 0)
+	dictget $kgm_re->fRegistryTable gIONameKey
+    end
+    if ($kgm_result == 0)
+	dictget $kgm_re->fPropertyTable gIOClassKey
+    end
+
+    if ($kgm_result != 0)
+	printf "%s", ((OSString *)$kgm_result)->string
+    else
+ 	if (((IOService*)$kgm_re)->pm_vars &&  ((IOService*)$kgm_re)->pm_vars->ourName)
+ 	    printf "%s", ((IOService*)$kgm_re)->pm_vars->ourName
+	else
+#	    printf ", guessclass "
+#	    guessclass $kgm_re
+	    printf "??"
+	end
+    end
+
+
+    printf "  <object %p, ", $kgm_re
+    printf "vtable "
+    set $kgm_vt = (unsigned) *(void**) $kgm_re
+    output /a $kgm_vt
+
+    if ($kgm_vt != _ZTV15IORegistryEntry)
+	printf ", "
+	set $kgm_state =  $kgm_re->__state[0]
+	# kIOServiceRegisteredState
+	if (0 == ($kgm_state & 2))
+	    printf "!"
+	end
+	printf "registered, "
+	# kIOServiceMatchedState
+	if (0 == ($kgm_state & 4))
+	    printf "!"
+	end
+	printf "matched, "
+	# kIOServiceInactiveState
+	if ($kgm_state & 1)
+	    printf "in"
+	end
+	printf "active, busy %d, retain count %d", (0xff & $kgm_re->__state[1]), (0xffff & $kgm_re->retainCount)
+    end
+    printf ">\n"
+
+    if ($kgm_show_props)
+	set $kgm_props = $kgm_re->fPropertyTable
+	showregdictionary $kgm_props $kgm$arg0_stack
+    end
+
+    # recurse
+    if ($kgm$arg0_child_count != 0)
+
+	set $kgm_reg_depth = $kgm_reg_depth + 1
+	set $kgm$arg0_child_idx = 0
+
+	while ($kgm$arg0_child_idx < $kgm$arg0_child_count)
+	    set $kgm_re = $kgm$arg0_child_array->array[$kgm$arg0_child_idx++]
+	    set $kgm_more_sib = ($kgm$arg0_child_idx < $kgm$arg0_child_count)
+	    showregistryentryrecurse _$arg0 $kgm_re $kgm$arg0_stack $kgm_more_sib
+	end
+
+	set $kgm_reg_depth = $kgm_reg_depth - 1
+    end
+end
+
+define showregistryentryint
+    set $kgm_namekey   = (OSSymbol *) $kgm_reg_plane[2]
+    set $kgm_childkey  = (OSSymbol *) $kgm_reg_plane[4]
+
+    showregistryentryrecurse _ $arg0 0 0
+end
+
+define showregistry
+    set $kgm_reg_depth  = 0
+    set $kgm_show_props = 0
+    showregistryentryint gRegistryRoot
+end
+document showregistry
+| Show info about all registry entries in the current plane.
+| The following is the syntax:
+|     (gdb) showregistry 
+end
+
+define showregistryprops
+    set $kgm_reg_depth  = 0
+    set $kgm_show_props = 1
+    showregistryentryint gRegistryRoot
+end
+document showregistryprops
+| Show info about all registry entries in the current plane, and their properties.
+| set $kgm_show_object_addrs = 1 and/or set $kgm_show_object_retain = 1 will display
+| more verbose information
+| The following is the syntax:
+|     (gdb) showregistryprops 
+end
+
+define showregistryentry
+    set $kgm_reg_depth  = 0
+    set $kgm_show_props = 1
+    showregistryentryint $arg0
+end
+document showregistryentry
+| Show info about a registry entry; its properties and descendants in the current plane.
+| The following is the syntax:
+|     (gdb) showregistryentry <object address>
+end
+
+define setregistryplane
+    if ($arg0)
+	set $kgm_reg_plane = (void **) $arg0
+    else
+	showobjectint _ gIORegistryPlanes
+	printf "\n"
+    end
+end
+document setregistryplane
+| Set the plane to be used for the iokit registry macros. An argument of zero will 
+| display known planes.
+| The following is the syntax:
+|     (gdb) setregistryplane <plane object address>
+end
+
+define guessclass
     set $kgm_classidx = 0
+    set $kgm_lookvt = *((void **) $arg0)
+    set $kgm_bestvt = (void *) 0
+    set $kgm_bestidx = 0
+    
     while $kgm_classidx < sAllClassesDict->count
 	set $kgm_meta = (OSMetaClass *) sAllClassesDict->dictionary[$kgm_classidx].value
-	showmetaclass $kgm_meta
+
+	set $kgm_vt = *((void **) $kgm_meta)
+
+	if (($kgm_vt > $kgm_bestvt) && ($kgm_vt < $kgm_lookvt))
+	    set $kgm_bestvt  = $kgm_vt
+	    set $kgm_bestidx = $kgm_classidx
+	end
 	set $kgm_classidx = $kgm_classidx + 1
     end
+    printf "%s", sAllClassesDict->dictionary[$kgm_bestidx].key->string
 end
+
+define showallclasses
+    set $kgm_classidx = 0
+    while $kgm_classidx < sAllClassesDict->count
+	set $kgm_meta = (OSMetaClass *) sAllClassesDict->dictionary[$kgm_classidx++].value
+	showmetaclass $kgm_meta
+    end
+end
+
 document showallclasses
 | Show the instance counts and ivar size of all OSObject subclasses. See ioclasscount man page for details.
 | The following is the syntax:
@@ -1989,3 +2515,31 @@ document showioalloc
 | The following is the syntax:
 |     (gdb) showioalloc
 end
+
+define readphys
+       set kdp_trans_off = 1
+       x/x $arg0
+       set kdp_trans_off = 0
+end
+
+define readphys64
+	if ($kgm_mtype == 18)
+	   set kdp_src_high32 = ((uint32_t) ($arg0)) >> 32
+	   x/x (uint32_t) (($arg0) & 0x00000000ffffffffUL)
+	   set kdp_src_high32 = 0
+	else
+		echo readphys64 not available on this architecture.\n
+	end
+end
+
+document readphys
+| The argument is interpreted as a physical address, and the word addressed is
+| displayed. While this fails if no physical page exists at the given address,
+| it must be used with caution.
+end
+
+document readphys64
+| The argument is interpreted as a 64-bit physical address, and the word
+| addressed is displayed. While this fails if no physical page exists at the
+| given address, it must be used with caution.
+end
diff --git a/libkern/Makefile b/libkern/Makefile
index c9ed0dba2..eca7e6882 100644
--- a/libkern/Makefile
+++ b/libkern/Makefile
@@ -4,9 +4,9 @@ export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
 export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
 
 
-export COMP_LDFLAGS_COMPONENT_PPC = -i_OSCompareAndSwap:_hw_compare_and_store \
-                                    -i_OSDequeueAtomic:_hw_dequeue_atomic \
-                                    -i_OSEnqueueAtomic:_hw_queue_atomic
+export COMP_LDFLAGS_COMPONENT_PPC = -Wl,-i_OSCompareAndSwap:_hw_compare_and_store \
+                                    -Wl,-i_OSDequeueAtomic:_hw_dequeue_atomic \
+                                    -Wl,-i_OSEnqueueAtomic:_hw_queue_atomic
 
 include $(MakeInc_cmd)
 include $(MakeInc_def)
diff --git a/libkern/c++/OSCollectionIterator.cpp b/libkern/c++/OSCollectionIterator.cpp
index cdc565865..d44e0d500 100644
--- a/libkern/c++/OSCollectionIterator.cpp
+++ b/libkern/c++/OSCollectionIterator.cpp
@@ -76,7 +76,7 @@ OSCollectionIterator::withCollection(const OSCollection *inColl)
 void OSCollectionIterator::free()
 {
     if (collIterator) {
-        kfree((vm_offset_t)collIterator, collection->iteratorSize());
+        kfree(collIterator, collection->iteratorSize());
 	ACCUMSIZE(-(collection->iteratorSize()));
         collIterator = 0;
     }
diff --git a/libkern/c++/OSData.cpp b/libkern/c++/OSData.cpp
index 873beae4d..47def143b 100644
--- a/libkern/c++/OSData.cpp
+++ b/libkern/c++/OSData.cpp
@@ -63,7 +63,7 @@ bool OSData::initWithCapacity(unsigned int inCapacity)
 
     if (data && (!inCapacity || capacity < inCapacity) ) {
         // clean out old data's storage if it isn't big enough
-        kfree((vm_address_t) data, capacity);
+        kfree(data, capacity);
         data = 0;
         ACCUMSIZE(-capacity);
     }
@@ -188,7 +188,7 @@ OSData *OSData::withData(const OSData *inData,
 void OSData::free()
 {
     if (capacity != EXTERNAL && data && capacity) {
-        kfree((vm_offset_t)data, capacity);
+        kfree(data, capacity);
         ACCUMSIZE( -capacity );
     }
     super::free();
@@ -223,7 +223,7 @@ unsigned int OSData::ensureCapacity(unsigned int newCapacity)
         bzero(newData + capacity, newCapacity - capacity);
         if (data) {
             bcopy(data, newData, capacity);
-            kfree((vm_offset_t)data, capacity);
+            kfree(data, capacity);
         }
         ACCUMSIZE( newCapacity - capacity );
         data = (void *) newData;
diff --git a/libkern/c++/OSDictionary.cpp b/libkern/c++/OSDictionary.cpp
index e614954ec..d09743c1d 100644
--- a/libkern/c++/OSDictionary.cpp
+++ b/libkern/c++/OSDictionary.cpp
@@ -240,7 +240,7 @@ void OSDictionary::free()
     (void) super::setOptions(0, kImmutable);
     flushCollection();
     if (dictionary) {
-        kfree((vm_offset_t)dictionary, capacity * sizeof(dictEntry));
+        kfree(dictionary, capacity * sizeof(dictEntry));
         ACCUMSIZE( -(capacity * sizeof(dictEntry)) );
     }
 
@@ -283,7 +283,7 @@ unsigned int OSDictionary::ensureCapacity(unsigned int newCapacity)
         bzero(&newDict[capacity], newSize - oldSize);
 
         ACCUMSIZE(newSize - oldSize);
-        kfree((vm_offset_t)dictionary, oldSize);
+        kfree(dictionary, oldSize);
 
         dictionary = newDict;
         capacity = newCapacity;
diff --git a/libkern/c++/OSMetaClass.cpp b/libkern/c++/OSMetaClass.cpp
index 6aa1b1e47..d4f771633 100644
--- a/libkern/c++/OSMetaClass.cpp
+++ b/libkern/c++/OSMetaClass.cpp
@@ -44,6 +44,9 @@
 #include <libkern/c++/OSLib.h>
 #include <libkern/OSAtomic.h>
 
+#include <IOKit/pwr_mgt/RootDomain.h>
+#include <IOKit/IOMessage.h>
+
 __BEGIN_DECLS
 
 #include <sys/systm.h>
@@ -76,7 +79,7 @@ static const int kClassCapacityIncrement = 40;
 static const int kKModCapacityIncrement = 10;
 static OSDictionary *sAllClassesDict, *sKModClassesDict, *sSortedByClassesDict;
 
-static mutex_t *loadLock;
+static mutex_t *loadLock = 0;
 static struct StalledData {
     const char *kmodName;
     OSReturn result;
@@ -86,6 +89,8 @@ static struct StalledData {
 } *sStalled;
 
 static unsigned int sConsiderUnloadDelay = 60;	/* secs */
+static bool unloadsEnabled = true;  // set to false when system going to sleep
+static thread_call_t unloadCallout = 0;
 
 static const char OSMetaClassBasePanicMsg[] =
     "OSMetaClassBase::_RESERVEDOSMetaClassBase%d called\n";
@@ -278,7 +283,7 @@ OSMetaClass::OSMetaClass(const char *inClassName,
 
 	    sStalled->capacity += kKModCapacityIncrement;
 	    memmove(sStalled->classes, oldStalled, oldSize);
-	    kfree((vm_offset_t)oldStalled, oldSize);
+	    kfree(oldStalled, oldSize);
 	    ACCUMSIZE(newSize - oldSize);
 	}
 
@@ -362,7 +367,7 @@ void *OSMetaClass::preModLoad(const char *kmodName)
 	sStalled->classes  = (OSMetaClass **)
 			kalloc(kKModCapacityIncrement * sizeof(OSMetaClass *));
 	if (!sStalled->classes) {
-	    kfree((vm_offset_t) sStalled, sizeof(*sStalled));
+	    kfree(sStalled, sizeof(*sStalled));
 	    return 0;
 	}
 	ACCUMSIZE((kKModCapacityIncrement * sizeof(OSMetaClass *)) + sizeof(*sStalled));
@@ -414,7 +419,7 @@ OSReturn OSMetaClass::postModLoad(void *loadHandle)
     case kCompletedBootstrap:
     {
         unsigned int i;
-        myname = OSSymbol::withCStringNoCopy(sStalled->kmodName);
+        myname = (OSSymbol *)OSSymbol::withCStringNoCopy(sStalled->kmodName);
 
 	if (!sStalled->count)
 	    break;	// Nothing to do so just get out
@@ -471,9 +476,9 @@ OSReturn OSMetaClass::postModLoad(void *loadHandle)
     if (sStalled) {
 	ACCUMSIZE(-(sStalled->capacity * sizeof(OSMetaClass *)
 		     + sizeof(*sStalled)));
-	kfree((vm_offset_t) sStalled->classes,
+	kfree(sStalled->classes,
 	      sStalled->capacity * sizeof(OSMetaClass *));
-	kfree((vm_offset_t) sStalled, sizeof(*sStalled));
+	kfree(sStalled, sizeof(*sStalled));
 	sStalled = 0;
     }
 
@@ -564,6 +569,34 @@ void OSMetaClass::reportModInstances(const char *kmodName)
     iter->release();
 }
 
+
+extern "C" {
+
+IOReturn OSMetaClassSystemSleepOrWake(UInt32 messageType)
+{
+    mutex_lock(loadLock);
+
+   /* If the system is going to sleep, cancel the reaper thread timer
+    * and mark unloads disabled in case it just fired but hasn't
+    * taken the lock yet. If we are coming back from sleep, just
+    * set unloads enabled; IOService's normal operation will cause
+    * unloads to be considered soon enough.
+    */
+    if (messageType == kIOMessageSystemWillSleep) {
+        if (unloadCallout) {
+            thread_call_cancel(unloadCallout);
+        }
+        unloadsEnabled = false;
+    } else if (messageType == kIOMessageSystemHasPoweredOn) {
+        unloadsEnabled = true;
+    }
+    mutex_unlock(loadLock);
+
+    return kIOReturnSuccess;
+}
+
+};
+
 extern "C" kern_return_t kmod_unload_cache(void);
 
 static void _OSMetaClassConsiderUnloads(thread_call_param_t p0,
@@ -580,6 +613,11 @@ static void _OSMetaClassConsiderUnloads(thread_call_param_t p0,
 
     mutex_lock(loadLock);
 
+    if (!unloadsEnabled) {
+        mutex_unlock(loadLock);
+        return;
+    }
+
     do {
 
 	kmods = OSCollectionIterator::withCollection(sKModClassesDict);
@@ -590,7 +628,7 @@ static void _OSMetaClassConsiderUnloads(thread_call_param_t p0,
         while ( (kmodName = (OSSymbol *) kmods->getNextObject()) ) {
 
             if (ki) {
-                kfree((vm_offset_t) ki, sizeof(kmod_info_t));
+                kfree(ki, sizeof(kmod_info_t));
                 ki = 0;
             }
 
@@ -632,7 +670,6 @@ static void _OSMetaClassConsiderUnloads(thread_call_param_t p0,
 
 void OSMetaClass::considerUnloads()
 {
-    static thread_call_t unloadCallout;
     AbsoluteTime when;
 
     mutex_lock(loadLock);
@@ -786,7 +823,7 @@ const OSMetaClass *OSMetaClass::getSuperClass() const
 
 const OSSymbol *OSMetaClass::getKmodName() const
 {	
-    return sSortedByClassesDict->getObject((const OSSymbol *)this);
+    return (const OSSymbol *)sSortedByClassesDict->getObject((OSSymbol *)this);
 }
 
 unsigned int OSMetaClass::getInstanceCount() const
diff --git a/libkern/c++/OSObject.cpp b/libkern/c++/OSObject.cpp
index 9223907fc..86c31e279 100644
--- a/libkern/c++/OSObject.cpp
+++ b/libkern/c++/OSObject.cpp
@@ -81,6 +81,8 @@ OSMetaClassDefineReservedUnused(OSObject, 12);
 OSMetaClassDefineReservedUnused(OSObject, 13);
 OSMetaClassDefineReservedUnused(OSObject, 14);
 OSMetaClassDefineReservedUnused(OSObject, 15);
+
+#ifdef __ppc__
 OSMetaClassDefineReservedUnused(OSObject, 16);
 OSMetaClassDefineReservedUnused(OSObject, 17);
 OSMetaClassDefineReservedUnused(OSObject, 18);
@@ -97,6 +99,7 @@ OSMetaClassDefineReservedUnused(OSObject, 28);
 OSMetaClassDefineReservedUnused(OSObject, 29);
 OSMetaClassDefineReservedUnused(OSObject, 30);
 OSMetaClassDefineReservedUnused(OSObject, 31);
+#endif
 
 static const char *getClassName(const OSObject *obj)
 {
@@ -277,7 +280,7 @@ void *OSObject::operator new(size_t size)
 
 void OSObject::operator delete(void *mem, size_t size)
 {
-    kfree((vm_offset_t) mem, size);
+    kfree(mem, size);
 
     ACCUMSIZE(-size);
 }
diff --git a/libkern/c++/OSOrderedSet.cpp b/libkern/c++/OSOrderedSet.cpp
index d14852933..cabf763bb 100644
--- a/libkern/c++/OSOrderedSet.cpp
+++ b/libkern/c++/OSOrderedSet.cpp
@@ -105,7 +105,7 @@ void OSOrderedSet::free()
     flushCollection();
 
     if (array) {
-        kfree((vm_offset_t)array, sizeof(_Element) * capacity);
+        kfree(array, sizeof(_Element) * capacity);
         ACCUMSIZE( -(sizeof(_Element) * capacity) );
     }
 
@@ -143,7 +143,7 @@ unsigned int OSOrderedSet::ensureCapacity(unsigned int newCapacity)
 
         bcopy(array, newArray, oldSize);
         bzero(&newArray[capacity], newSize - oldSize);
-        kfree((vm_offset_t)array, oldSize);
+        kfree(array, oldSize);
         array = newArray;
         capacity = newCapacity;
     }
diff --git a/libkern/c++/OSString.cpp b/libkern/c++/OSString.cpp
index fa02e32fc..4b2cd6da4 100644
--- a/libkern/c++/OSString.cpp
+++ b/libkern/c++/OSString.cpp
@@ -164,7 +164,7 @@ OSString *OSString::stringWithFormat(const char *format, ...)
 void OSString::free()
 {
     if ( !(flags & kOSStringNoCopy) && string) {
-        kfree((vm_offset_t)string, (vm_size_t)length);
+        kfree(string, (vm_size_t)length);
         ACCUMSIZE(-length);
     }
 
diff --git a/libkern/c++/OSSymbol.cpp b/libkern/c++/OSSymbol.cpp
index ebaebabfa..c21bd466b 100644
--- a/libkern/c++/OSSymbol.cpp
+++ b/libkern/c++/OSSymbol.cpp
@@ -119,7 +119,7 @@ void * OSSymbolPool::operator new(size_t size)
 
 void OSSymbolPool::operator delete(void *mem, size_t size)
 {
-    kfree((vm_offset_t)mem, size);
+    kfree(mem, size);
     ACCUMSIZE(-size);
 }
 
@@ -151,12 +151,12 @@ OSSymbolPool::OSSymbolPool(const OSSymbolPool *old)
 OSSymbolPool::~OSSymbolPool()
 {
     if (buckets) {
-        kfree((vm_offset_t)buckets, nBuckets * sizeof(Bucket));
+        kfree(buckets, nBuckets * sizeof(Bucket));
         ACCUMSIZE(-(nBuckets * sizeof(Bucket)));
     }
 
     if (poolGate)
-        kfree((vm_offset_t) poolGate, 36 * 4);
+        kfree(poolGate, 36 * 4);
 }
 
 unsigned long OSSymbolPool::log2(unsigned int x)
@@ -301,7 +301,7 @@ OSSymbol *OSSymbolPool::insertSymbol(OSSymbol *sym)
     /* @@@ gvdl: Zero test and panic if can't set up pool */
     list[0] = sym;
     bcopy(thisBucket->symbolP, list + 1, j * sizeof(OSSymbol *));
-    kfree((vm_offset_t)thisBucket->symbolP, j * sizeof(OSSymbol *));
+    kfree(thisBucket->symbolP, j * sizeof(OSSymbol *));
     ACCUMSIZE(-(j * sizeof(OSSymbol *)));
     thisBucket->symbolP = list;
     if (count > nBuckets)
@@ -340,7 +340,7 @@ void OSSymbolPool::removeSymbol(OSSymbol *sym)
         probeSymbol = list[0];
         if (probeSymbol == sym) {
             thisBucket->symbolP = (OSSymbol **) list[1];
-            kfree((vm_offset_t)list, 2 * sizeof(OSSymbol *));
+            kfree(list, 2 * sizeof(OSSymbol *));
 	    ACCUMSIZE(-(2 * sizeof(OSSymbol *)));
             count--;
             thisBucket->count--;
@@ -350,7 +350,7 @@ void OSSymbolPool::removeSymbol(OSSymbol *sym)
         probeSymbol = list[1];
         if (probeSymbol == sym) {
             thisBucket->symbolP = (OSSymbol **) list[0];
-            kfree((vm_offset_t)list, 2 * sizeof(OSSymbol *));
+            kfree(list, 2 * sizeof(OSSymbol *));
 	    ACCUMSIZE(-(2 * sizeof(OSSymbol *)));
             count--;
             thisBucket->count--;
@@ -373,7 +373,7 @@ void OSSymbolPool::removeSymbol(OSSymbol *sym)
                 bcopy(thisBucket->symbolP + thisBucket->count-j,
                       list + thisBucket->count-1-j,
                       j * sizeof(OSSymbol *));
-            kfree((vm_offset_t)thisBucket->symbolP, thisBucket->count * sizeof(OSSymbol *));
+            kfree(thisBucket->symbolP, thisBucket->count * sizeof(OSSymbol *));
 	    ACCUMSIZE(-(thisBucket->count * sizeof(OSSymbol *)));
             thisBucket->symbolP = list;
             count--;
diff --git a/libkern/c++/OSUnserializeXML.cpp b/libkern/c++/OSUnserializeXML.cpp
index 647e691b6..0934a7a2c 100644
--- a/libkern/c++/OSUnserializeXML.cpp
+++ b/libkern/c++/OSUnserializeXML.cpp
@@ -87,7 +87,7 @@
 
 #define YYSTYPE object_t *
 #define YYPARSE_PARAM	state
-#define YYLEX_PARAM	state
+#define YYLEX_PARAM	(parser_state_t *)state
 
 // this is the internal struct used to hold objects on parser stack
 // it represents objects both before and after they have been created
@@ -153,7 +153,7 @@ extern unsigned long	strtoul(const char *, char **, int);
 
 #define malloc(s) kern_os_malloc(s)
 #define realloc(a, s) kern_os_realloc(a, s)
-#define free(a) kern_os_free(a)
+#define free(a) kern_os_free((void *)a)
 
 #ifndef YYSTYPE
 #define YYSTYPE int
@@ -939,7 +939,7 @@ case 16:
 case 17:
 #line 192 "OSUnserializeXML.y"
 { yyval = yyvsp[-1];
-				  yyval->key = yyval->object;
+				  yyval->key = (OSString *)yyval->object;
 				  yyval->object = yyvsp[0]->object;
 				  yyval->next = NULL; 
 				  yyvsp[0]->object = 0;
diff --git a/libkern/c++/OSUnserializeXML.y b/libkern/c++/OSUnserializeXML.y
index b71000eaf..115eab38b 100644
--- a/libkern/c++/OSUnserializeXML.y
+++ b/libkern/c++/OSUnserializeXML.y
@@ -66,7 +66,7 @@
 
 #define YYSTYPE object_t *
 #define YYPARSE_PARAM	state
-#define YYLEX_PARAM	state
+#define YYLEX_PARAM	(parser_state_t *)state
 
 // this is the internal struct used to hold objects on parser stack
 // it represents objects both before and after they have been created
@@ -132,7 +132,7 @@ extern unsigned long	strtoul(const char *, char **, int);
 
 #define malloc(s) kern_os_malloc(s)
 #define realloc(a, s) kern_os_realloc(a, s)
-#define free(a) kern_os_free(a)
+#define free(a) kern_os_free((void *)a)
 
 %}
 %token ARRAY
@@ -196,7 +196,7 @@ pairs:	  pair
 	;
 
 pair:	  key object		{ $$ = $1;
-				  $$->key = $$->object;
+				  $$->key = (OSString *)$$->object;
 				  $$->object = $2->object;
 				  $$->next = NULL; 
 				  $2->object = 0;
diff --git a/libkern/conf/Makefile.i386 b/libkern/conf/Makefile.i386
index fa98396d8..712240adf 100644
--- a/libkern/conf/Makefile.i386
+++ b/libkern/conf/Makefile.i386
@@ -25,6 +25,9 @@ OBJS_NO_WERROR=				\
 	OSString.cpo			\
 	OSSymbol.cpo			\
 	OSUnserialize.cpo		\
+	OSIterator.cpo			\
+	OSSet.cpo				\
+	scanf.o					\
 	OSUnserializeXML.cpo
 
 OBJS_WERROR=$(filter-out $(OBJS_NO_WERROR),$(OBJS))
diff --git a/libkern/gen/OSAtomicOperations.c b/libkern/gen/OSAtomicOperations.c
index 4899091fa..2167974e5 100644
--- a/libkern/gen/OSAtomicOperations.c
+++ b/libkern/gen/OSAtomicOperations.c
@@ -34,7 +34,6 @@ enum {
 };
 #define	NULL 0L
 
-
 /*
  * atomic operations
  *	these are _the_ atomic operations, currently cast atop CompareAndSwap,
@@ -77,6 +76,7 @@ SInt32	OSDecrementAtomic(SInt32 * value)
 	return OSAddAtomic(-1, value);
 }
 
+#ifdef CMPXCHG8B
 void *	OSDequeueAtomic(void ** inList, SInt32 inOffset)
 {
 	void *	oldListHead;
@@ -91,7 +91,6 @@ void *	OSDequeueAtomic(void ** inList, SInt32 inOffset)
 		newListHead = *(void **) (((char *) oldListHead) + inOffset);
 	} while (! OSCompareAndSwap((UInt32)oldListHead,
 					(UInt32)newListHead, (UInt32 *)inList));
-	
 	return oldListHead;
 }
 
@@ -107,7 +106,7 @@ void	OSEnqueueAtomic(void ** inList, void * inNewLink, SInt32 inOffset)
 	} while (! OSCompareAndSwap((UInt32)oldListHead, (UInt32)newListHead,
 					(UInt32 *)inList));
 }
-
+#endif /* CMPXCHG8B */
 #endif	/* !__ppc__ */
 
 static UInt32	OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt32 * value)
diff --git a/libkern/gen/OSDebug.cpp b/libkern/gen/OSDebug.cpp
index f4c4d6760..f464ad889 100644
--- a/libkern/gen/OSDebug.cpp
+++ b/libkern/gen/OSDebug.cpp
@@ -37,6 +37,7 @@
 #include <kern/lock.h>
 
 #include <libkern/libkern.h>	// From bsd's libkern directory
+#include <mach/vm_param.h>
 
 __BEGIN_DECLS
 // From osmfk/kern/thread.h but considered to be private
@@ -45,6 +46,8 @@ extern vm_offset_t max_valid_stack_address(void);
 
 // From osfmk/kmod.c
 extern void kmod_dump_log(vm_offset_t *addr, unsigned int cnt);
+
+extern addr64_t kvtophys(vm_offset_t va);
 __END_DECLS
 
 static mutex_t *sOSReportLock = mutex_alloc(0);
@@ -77,6 +80,39 @@ OSReportWithBacktrace(const char *str, ...)
 static vm_offset_t minstackaddr = min_valid_stack_address();
 static vm_offset_t maxstackaddr = max_valid_stack_address();
 
+#if __i386__
+#define i386_RETURN_OFFSET 4
+
+static unsigned int
+i386_validate_stackptr(vm_offset_t stackptr)
+{
+	/* Existence and alignment check
+	 */
+	if (!stackptr || (stackptr & 0x3))
+		return 0;
+  
+	/* Is a virtual->physical translation present?
+	 */
+	if (!kvtophys(stackptr))
+		return 0;
+  
+	/* Check if the return address lies on the same page;
+	 * If not, verify that a translation exists.
+	 */
+	if (((PAGE_SIZE - (stackptr & PAGE_MASK)) < i386_RETURN_OFFSET) &&
+	    !kvtophys(stackptr + i386_RETURN_OFFSET))
+		return 0;
+	return 1;
+}
+
+static unsigned int
+i386_validate_raddr(vm_offset_t raddr)
+{
+	return ((raddr > VM_MIN_KERNEL_ADDRESS) &&
+	    (raddr < VM_MAX_KERNEL_ADDRESS));
+}
+#endif
+
 unsigned OSBacktrace(void **bt, unsigned maxAddrs)
 {
     unsigned frame;
@@ -110,67 +146,50 @@ unsigned OSBacktrace(void **bt, unsigned maxAddrs)
 
     for ( ; i < maxAddrs; i++)
 	    bt[i] = (void *) 0;
-#elif 0 && __i386__	// Note that this should be ported for i386
-    // This function is not safe, we should get this code ported appropriately
-    if (maxAddrs > 16) {
-	for (frame = 16; frame < maxAddrs; frame++)
-	    bt[frame] = __builtin_return_address(frame);
-	maxAddrs = 16;
-    }
+#elif __i386__
+#define SANE_i386_FRAME_SIZE 8*1024
+    vm_offset_t stackptr, stackptr_prev, raddr;
+    unsigned frame_index = 0;
+/* Obtain current frame pointer */
+    __asm__ volatile("movl %%ebp, %0" : "=m" (stackptr)); 
 
-    switch(maxAddrs) {
-    case 15+1: bt[15] = __builtin_return_address(15);
-    case 14+1: bt[14] = __builtin_return_address(14);
-    case 13+1: bt[13] = __builtin_return_address(13);
-    case 12+1: bt[12] = __builtin_return_address(12);
-    case 11+1: bt[11] = __builtin_return_address(11);
-    case 10+1: bt[10] = __builtin_return_address(10);
-    case  9+1: bt[ 9] = __builtin_return_address( 9);
-    case  8+1: bt[ 8] = __builtin_return_address( 8);
-    case  7+1: bt[ 7] = __builtin_return_address( 7);
-    case  6+1: bt[ 6] = __builtin_return_address( 6);
-    case  5+1: bt[ 5] = __builtin_return_address( 5);
-    case  4+1: bt[ 4] = __builtin_return_address( 4);
-    case  3+1: bt[ 3] = __builtin_return_address( 3);
-    case  2+1: bt[ 2] = __builtin_return_address( 2);
-    case  1+1: bt[ 1] = __builtin_return_address( 1);
-    case  0+1: bt[ 0] = __builtin_return_address( 0);
-    case 0: default: break;
-    }
+    if (!i386_validate_stackptr(stackptr))
+	    goto pad;
 
-    frame = maxAddrs;
-#else
-    // This function is not safe, we should get this code ported appropriately
-    if (maxAddrs > 16) {
-	for (frame = 16; frame < maxAddrs; frame++)
-	    bt[frame] = 0;
-	maxAddrs = 16;
-    }
+    raddr = *((vm_offset_t *) (stackptr + i386_RETURN_OFFSET));
+
+    if (!i386_validate_raddr(raddr))
+	    goto pad;
+
+    bt[frame_index++] = (void *) raddr;
+
+    for ( ; frame_index < maxAddrs; frame_index++) {
+	    stackptr_prev = stackptr;
+	    stackptr = *((vm_offset_t *) stackptr_prev);
+
+	    if (!i386_validate_stackptr(stackptr))
+		    break;
+	/* Stack grows downwards */
+	    if (stackptr < stackptr_prev)
+		    break;
 
-    switch (maxAddrs) {
-    case 15+1: bt[15] = __builtin_return_address(15);
-    case 14+1: bt[14] = __builtin_return_address(14);
-    case 13+1: bt[13] = __builtin_return_address(13);
-    case 12+1: bt[12] = __builtin_return_address(12);
-    case 11+1: bt[11] = __builtin_return_address(11);
-    case 10+1: bt[10] = __builtin_return_address(10);
-    case  9+1: bt[ 9] = __builtin_return_address( 9);
-    case  8+1: bt[ 8] = __builtin_return_address( 8);
-    case  7+1: bt[ 7] = __builtin_return_address( 7);
-    case  6+1: bt[ 6] = __builtin_return_address( 6);
-    case  5+1: bt[ 5] = __builtin_return_address( 5);
-    case  4+1: bt[ 4] = __builtin_return_address( 4);
-    case  3+1: bt[ 3] = __builtin_return_address( 3);
-    case  2+1: bt[ 2] = __builtin_return_address( 2);
-    case  1+1: bt[ 1] = __builtin_return_address( 1);
-    case  0+1: bt[ 0] = __builtin_return_address( 0);
-    case    0:
-    default  :
-	break;
+	    if ((stackptr_prev ^ stackptr) > SANE_i386_FRAME_SIZE)
+		    break;
+
+	    raddr = *((vm_offset_t *) (stackptr + i386_RETURN_OFFSET));
+
+	    if (!i386_validate_raddr(raddr))
+		    break;
+
+	    bt[frame_index] = (void *) raddr;
     }
+pad:
+    frame = frame_index;
 
-    frame = maxAddrs;
+    for ( ; frame_index < maxAddrs; frame_index++)
+	    bt[frame_index] = (void *) 0;
+#else
+#error arch
 #endif
-
     return frame;
 }
diff --git a/libkern/libkern/OSAtomic.h b/libkern/libkern/OSAtomic.h
index 429ea8fbc..a8a77c502 100644
--- a/libkern/libkern/OSAtomic.h
+++ b/libkern/libkern/OSAtomic.h
@@ -267,6 +267,7 @@ extern Boolean	OSTestAndSet(UInt32 bit, UInt8 * startAddress);
 
 extern Boolean	OSTestAndClear(UInt32 bit, UInt8 * startAddress);
 
+#ifdef __ppc__
 /*! @function OSEnqueueAtomic
     @abstract Singly linked list head insertion, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
     @discussion The OSEnqueueAtomic function places an element at the head of a single linked list, which is specified with the address of a head pointer, listHead. The element structure has a next field whose offset is specified.
@@ -290,6 +291,7 @@ extern void	OSEnqueueAtomic(void ** listHead, void * element,
 
 extern void *	OSDequeueAtomic(void ** listHead,
 				SInt32 elementNextFieldOffset);
+#endif /* __ppc__ */
 
 /*! @function OSSynchronizeIO
     @abstract The OSSynchronizeIO routine ensures orderly load and store operations to noncached memory mapped I/O devices.
diff --git a/libkern/libkern/OSAtomic.h.save b/libkern/libkern/OSAtomic.h.save
index 5409e23c1..53d5f8b99 100644
--- a/libkern/libkern/OSAtomic.h.save
+++ b/libkern/libkern/OSAtomic.h.save
@@ -261,6 +261,7 @@ extern Boolean	OSTestAndSet(UInt32 bit, UInt8 * startAddress);
 
 extern Boolean	OSTestAndClear(UInt32 bit, UInt8 * startAddress);
 
+#ifdef __ppc__
 /*! @function OSEnqueueAtomic
     @abstract Singly linked list head insertion, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
     @discussion The OSEnqueueAtomic function places an element at the head of a single linked list, which is specified with the address of a head pointer, listHead. The element structure has a next field whose offset is specified.
@@ -284,6 +285,7 @@ extern void	OSEnqueueAtomic(void ** listHead, void * element,
 
 extern void *	OSDequeueAtomic(void ** listHead,
 				SInt32 elementNextFieldOffset);
+#endif /* __ppc__ */
 
 /*! @function OSSynchronizeIO
     @abstract The OSSynchronizeIO routine ensures orderly load and store operations to noncached memory mapped I/O devices.
diff --git a/libkern/libkern/OSByteOrder.h b/libkern/libkern/OSByteOrder.h
index efde3cba8..85819b680 100644
--- a/libkern/libkern/OSByteOrder.h
+++ b/libkern/libkern/OSByteOrder.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -25,26 +25,64 @@
  * 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/*
- * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved.
- *
- * HISTORY
- *
- */
 
 #ifndef _OS_OSBYTEORDER_H
 #define _OS_OSBYTEORDER_H
 
 #include <stdint.h>
 
-#if defined(__GNUC__) && defined(__ppc__)
+/* Macros for swapping constant values in the preprocessing stage. */
+#define OSSwapConstInt16(x) \
+    ((uint16_t)((((uint16_t)(x) & 0xff00) >> 8) | \
+                (((uint16_t)(x) & 0x00ff) << 8)))
+
+#define OSSwapConstInt32(x) \
+    ((uint32_t)((((uint32_t)(x) & 0xff000000) >> 24) | \
+                (((uint32_t)(x) & 0x00ff0000) >>  8) | \
+                (((uint32_t)(x) & 0x0000ff00) <<  8) | \
+                (((uint32_t)(x) & 0x000000ff) << 24)))
+
+#define OSSwapConstInt64(x) \
+    ((uint64_t)((((uint64_t)(x) & 0xff00000000000000ULL) >> 56) | \
+                (((uint64_t)(x) & 0x00ff000000000000ULL) >> 40) | \
+                (((uint64_t)(x) & 0x0000ff0000000000ULL) >> 24) | \
+                (((uint64_t)(x) & 0x000000ff00000000ULL) >>  8) | \
+                (((uint64_t)(x) & 0x00000000ff000000ULL) <<  8) | \
+                (((uint64_t)(x) & 0x0000000000ff0000ULL) << 24) | \
+                (((uint64_t)(x) & 0x000000000000ff00ULL) << 40) | \
+                (((uint64_t)(x) & 0x00000000000000ffULL) << 56)))
+
+#if defined(__GNUC__)
+
+#if (defined(__ppc__) || defined(__ppc64__))
 #include <libkern/ppc/OSByteOrder.h>
-#elif defined(__GNUC__) && defined(__i386__)
+#elif (defined(__i386__) || defined(__x86_64__))
 #include <libkern/i386/OSByteOrder.h>
 #else
 #include <libkern/machine/OSByteOrder.h>
 #endif
 
+#define OSSwapInt16(x) \
+    (__builtin_constant_p(x) ? OSSwapConstInt16(x) : _OSSwapInt16(x))
+
+#define OSSwapInt32(x) \
+    (__builtin_constant_p(x) ? OSSwapConstInt32(x) : _OSSwapInt32(x))
+
+#define OSSwapInt64(x) \
+    (__builtin_constant_p(x) ? OSSwapConstInt64(x) : _OSSwapInt64(x))
+
+#else /* ! __GNUC__ */
+
+#include <libkern/machine/OSByteOrder.h>
+
+#define OSSwapInt16(x) OSSwapConstInt16(x)
+
+#define OSSwapInt32(x) OSSwapConstInt32(x)
+
+#define OSSwapInt64(x) OSSwapConstInt64(x)
+
+#endif /* __GNUC__ */
+
 enum {
     OSUnknownByteOrder,
     OSLittleEndian,
@@ -63,37 +101,6 @@ OSHostByteOrder(void) {
 #endif
 }
 
-/* Macros for swapping constant values in the preprocessing stage. */
-#define OSSwapConstInt16(x) ((((uint16_t)(x) & 0xff00) >> 8) | \
-                             (((uint16_t)(x) & 0x00ff) << 8))
-
-#define OSSwapConstInt32(x) ((((uint32_t)(x) & 0xff000000) >> 24) | \
-                             (((uint32_t)(x) & 0x00ff0000) >>  8) | \
-                             (((uint32_t)(x) & 0x0000ff00) <<  8) | \
-                             (((uint32_t)(x) & 0x000000ff) << 24))
-
-#define OSSwapConstInt64(x) ((((uint64_t)(x) & 0xff00000000000000ULL) >> 56) | \
-                             (((uint64_t)(x) & 0x00ff000000000000ULL) >> 40) | \
-                             (((uint64_t)(x) & 0x0000ff0000000000ULL) >> 24) | \
-                             (((uint64_t)(x) & 0x000000ff00000000ULL) >>  8) | \
-                             (((uint64_t)(x) & 0x00000000ff000000ULL) <<  8) | \
-                             (((uint64_t)(x) & 0x0000000000ff0000ULL) << 24) | \
-                             (((uint64_t)(x) & 0x000000000000ff00ULL) << 40) | \
-                             (((uint64_t)(x) & 0x00000000000000ffULL) << 56))
-
-#if !defined(__GNUC__)
-#define __builtin_constant_p(x) (0)
-#endif
-
-#define OSSwapInt16(x) \
-    (__builtin_constant_p(x) ? OSSwapConstInt16(x) : _OSSwapInt16(x))
-
-#define OSSwapInt32(x) \
-    (__builtin_constant_p(x) ? OSSwapConstInt32(x) : _OSSwapInt32(x))
-
-#define OSSwapInt64(x) \
-    (__builtin_constant_p(x) ? OSSwapConstInt64(x) : _OSSwapInt64(x))
-
 #define OSReadBigInt(x, y)		OSReadBigInt32(x, y)
 #define OSWriteBigInt(x, y, z)		OSWriteBigInt32(x, y, z)
 #define OSSwapBigToHostInt(x)		OSSwapBigToHostInt32(x)
@@ -103,141 +110,98 @@ OSHostByteOrder(void) {
 #define OSSwapHostToLittleInt(x)	OSSwapHostToLittleInt32(x)
 #define OSSwapLittleToHostInt(x)	OSSwapLittleToHostInt32(x)
 
-#if		defined(__BIG_ENDIAN__)
-
-/* Functions for loading big endian to host endianess. */
+/* Functions for loading native endian values. */
 
 OS_INLINE
 uint16_t
-OSReadBigInt16(
+_OSReadInt16(
     const volatile void               * base,
-    uintptr_t                     offset
+    uintptr_t                     byteOffset
 )
 {
-    return *(volatile uint16_t *)((uintptr_t)base + offset);
+    return *(volatile uint16_t *)((uintptr_t)base + byteOffset);
 }
 
 OS_INLINE
 uint32_t
-OSReadBigInt32(
+_OSReadInt32(
     const volatile void               * base,
-    uintptr_t                     offset
+    uintptr_t                     byteOffset
 )
 {
-    return *(volatile uint32_t *)((uintptr_t)base + offset);
+    return *(volatile uint32_t *)((uintptr_t)base + byteOffset);
 }
 
 OS_INLINE
 uint64_t
-OSReadBigInt64(
+_OSReadInt64(
     const volatile void               * base,
-    uintptr_t                     offset
+    uintptr_t                     byteOffset
 )
 {
-    return *(volatile uint64_t *)((uintptr_t)base + offset);
+    return *(volatile uint64_t *)((uintptr_t)base + byteOffset);
 }
 
-/* Functions for storing host endianess to big endian. */
+/* Functions for storing native endian values. */
 
 OS_INLINE
 void
-OSWriteBigInt16(
+_OSWriteInt16(
     volatile void               * base,
-    uintptr_t                     offset,
+    uintptr_t                     byteOffset,
     uint16_t                      data
 )
 {
-    *(volatile uint16_t *)((uintptr_t)base + offset) = data;
+    *(volatile uint16_t *)((uintptr_t)base + byteOffset) = data;
 }
 
 OS_INLINE
 void
-OSWriteBigInt32(
+_OSWriteInt32(
     volatile void               * base,
-    uintptr_t                     offset,
+    uintptr_t                     byteOffset,
     uint32_t                      data
 )
 {
-    *(volatile uint32_t *)((uintptr_t)base + offset) = data;
+    *(volatile uint32_t *)((uintptr_t)base + byteOffset) = data;
 }
 
 OS_INLINE
 void
-OSWriteBigInt64(
+_OSWriteInt64(
     volatile void               * base,
-    uintptr_t                     offset,
+    uintptr_t                     byteOffset,
     uint64_t                      data
 )
 {
-    *(volatile uint64_t *)((uintptr_t)base + offset) = data;
+    *(volatile uint64_t *)((uintptr_t)base + byteOffset) = data;
 }
 
-/* Functions for loading little endian to host endianess. */
+#if		defined(__BIG_ENDIAN__)
 
-OS_INLINE
-uint16_t
-OSReadLittleInt16(
-    volatile void               * base,
-    uintptr_t                     offset
-)
-{
-    return OSReadSwapInt16(base, offset);
-}
+/* Functions for loading big endian to host endianess. */
 
-OS_INLINE
-uint32_t
-OSReadLittleInt32(
-    volatile void               * base,
-    uintptr_t                     offset
-)
-{
-    return OSReadSwapInt32(base, offset);
-}
+#define OSReadBigInt16(base, byteOffset) _OSReadInt16(base, byteOffset)
+#define OSReadBigInt32(base, byteOffset) _OSReadInt32(base, byteOffset)
+#define OSReadBigInt64(base, byteOffset) _OSReadInt64(base, byteOffset)
 
-OS_INLINE
-uint64_t
-OSReadLittleInt64(
-    volatile void               * base,
-    uintptr_t                     offset
-)
-{
-    return OSReadSwapInt64(base, offset);
-}
+/* Functions for storing host endianess to big endian. */
 
-/* Functions for storing host endianess to little endian. */
+#define OSWriteBigInt16(base, byteOffset, data) _OSWriteInt16(base, byteOffset, data)
+#define OSWriteBigInt32(base, byteOffset, data) _OSWriteInt32(base, byteOffset, data)
+#define OSWriteBigInt64(base, byteOffset, data) _OSWriteInt64(base, byteOffset, data)
 
-OS_INLINE
-void
-OSWriteLittleInt16(
-    volatile void               * base,
-    uintptr_t                     offset,
-    uint16_t                      data
-)
-{
-    OSWriteSwapInt16(base, offset, data);
-}
+/* Functions for loading little endian to host endianess. */
 
-OS_INLINE
-void
-OSWriteLittleInt32(
-    volatile void               * base,
-    uintptr_t                     offset,
-    uint32_t                      data
-)
-{
-    OSWriteSwapInt32(base, offset, data);
-}
+#define OSReadLittleInt16(base, byteOffset) OSReadSwapInt16(base, byteOffset)
+#define OSReadLittleInt32(base, byteOffset) OSReadSwapInt32(base, byteOffset)
+#define OSReadLittleInt64(base, byteOffset) OSReadSwapInt64(base, byteOffset)
 
-OS_INLINE
-void
-OSWriteLittleInt64(
-    volatile void               * base,
-    uintptr_t                     offset,
-    uint64_t                      data
-)
-{
-    OSWriteSwapInt64(base, offset, data);
-}
+/* Functions for storing host endianess to little endian. */
+
+#define OSWriteLittleInt16(base, byteOffset, data) OSWriteSwapInt16(base, byteOffset, data)
+#define OSWriteLittleInt32(base, byteOffset, data) OSWriteSwapInt32(base, byteOffset, data)
+#define OSWriteLittleInt64(base, byteOffset, data) OSWriteSwapInt64(base, byteOffset, data)
 
 /* Host endianess to big endian byte swapping macros for constants. */
 
@@ -247,32 +211,9 @@ OSWriteLittleInt64(
 
 /* Generic host endianess to big endian byte swapping functions. */
 
-OS_INLINE
-uint16_t
-OSSwapHostToBigInt16(
-    uint16_t                        data
-)
-{
-    return data;
-}
-
-OS_INLINE
-uint32_t
-OSSwapHostToBigInt32(
-    uint32_t                        data
-)
-{
-    return data;
-}
-
-OS_INLINE
-uint64_t
-OSSwapHostToBigInt64(
-    uint64_t                        data
-)
-{
-    return data;
-}
+#define OSSwapHostToBigInt16(x) ((uint16_t)(x))
+#define OSSwapHostToBigInt32(x) ((uint32_t)(x))
+#define OSSwapHostToBigInt64(x) ((uint64_t)(x))
 
 /* Host endianess to little endian byte swapping macros for constants. */
 
@@ -294,32 +235,9 @@ OSSwapHostToBigInt64(
 
 /* Generic big endian to host endianess byte swapping functions. */
 
-OS_INLINE
-uint16_t
-OSSwapBigToHostInt16(
-    uint16_t                        data
-)
-{
-    return data;
-}
-
-OS_INLINE
-uint32_t
-OSSwapBigToHostInt32(
-    uint32_t                        data
-)
-{
-    return data;
-}
-
-OS_INLINE
-uint64_t
-OSSwapBigToHostInt64(
-    uint64_t                        data
-)
-{
-    return data;
-}
+#define OSSwapBigToHostInt16(x) ((uint16_t)(x))
+#define OSSwapBigToHostInt32(x) ((uint32_t)(x))
+#define OSSwapBigToHostInt64(x) ((uint64_t)(x))
 
 /* Little endian to host endianess byte swapping macros for constants. */
    
@@ -337,137 +255,27 @@ OSSwapBigToHostInt64(
 
 /* Functions for loading big endian to host endianess. */
 
-OS_INLINE
-uint16_t
-OSReadBigInt16(
-    const volatile void               * base,
-    uintptr_t                     offset
-)
-{
-    return OSReadSwapInt16(base, offset);
-}
-
-OS_INLINE
-uint32_t
-OSReadBigInt32(
-    const volatile void               * base,
-    uintptr_t                     offset
-)
-{
-    return OSReadSwapInt32(base, offset);
-}
-
-OS_INLINE
-uint64_t
-OSReadBigInt64(
-    const volatile void               * base,
-    uintptr_t                     offset
-)
-{
-    return OSReadSwapInt64(base, offset);
-}
+#define OSReadBigInt16(base, byteOffset) OSReadSwapInt16(base, byteOffset)
+#define OSReadBigInt32(base, byteOffset) OSReadSwapInt32(base, byteOffset)
+#define OSReadBigInt64(base, byteOffset) OSReadSwapInt64(base, byteOffset)
 
 /* Functions for storing host endianess to big endian. */
 
-OS_INLINE
-void
-OSWriteBigInt16(
-    volatile void               * base,
-    uintptr_t                     offset,
-    uint16_t                      data
-)
-{
-    OSWriteSwapInt16(base, offset, data);
-}
-
-OS_INLINE
-void
-OSWriteBigInt32(
-    volatile void               * base,
-    uintptr_t                     offset,
-    uint32_t                      data
-)
-{
-    OSWriteSwapInt32(base, offset, data);
-}
-
-OS_INLINE
-void
-OSWriteBigInt64(
-    volatile void               * base,
-    uintptr_t                     offset,
-    uint64_t                      data
-)
-{
-    OSWriteSwapInt64(base, offset, data);
-}
+#define OSWriteBigInt16(base, byteOffset, data) OSWriteSwapInt16(base, byteOffset, data)
+#define OSWriteBigInt32(base, byteOffset, data) OSWriteSwapInt32(base, byteOffset, data)
+#define OSWriteBigInt64(base, byteOffset, data) OSWriteSwapInt64(base, byteOffset, data)
 
 /* Functions for loading little endian to host endianess. */
 
-OS_INLINE
-uint16_t
-OSReadLittleInt16(
-    const volatile void               * base,
-    uintptr_t                     offset
-)
-{
-    return *(volatile uint16_t *)((uintptr_t)base + offset);
-}
-
-OS_INLINE
-uint32_t
-OSReadLittleInt32(
-    const volatile void               * base,
-    uintptr_t                     offset
-)
-{
-    return *(volatile uint32_t *)((uintptr_t)base + offset);
-}
-
-OS_INLINE
-uint64_t
-OSReadLittleInt64(
-    const volatile void               * base,
-    uintptr_t                     offset
-)
-{
-    return *(volatile uint64_t *)((uintptr_t)base + offset);
-}
+#define OSReadLittleInt16(base, byteOffset) _OSReadInt16(base, byteOffset)
+#define OSReadLittleInt32(base, byteOffset) _OSReadInt32(base, byteOffset)
+#define OSReadLittleInt64(base, byteOffset) _OSReadInt64(base, byteOffset)
 
 /* Functions for storing host endianess to little endian. */
 
-OS_INLINE
-void
-OSWriteLittleInt16(
-    volatile void               * base,
-    uintptr_t                     offset,
-    uint16_t                        data
-)
-{
-    *(volatile uint16_t *)((uintptr_t)base + offset) = data;
-}
-
-OS_INLINE
-void
-OSWriteLittleInt32(
-    volatile void               * base,
-    uintptr_t                     offset,
-    uint32_t                        data
-)
-{
-    *(volatile uint32_t *)((uintptr_t)base + offset) = data;
-}
-
-OS_INLINE
-void
-OSWriteLittleInt64(
-    volatile void               * base,
-    uintptr_t                     offset,
-    uint64_t                      data
-)
-{
-    *(volatile uint64_t *)((uintptr_t)base + offset) = data;
-}
+#define OSWriteLittleInt16(base, byteOffset, data) _OSWriteInt16(base, byteOffset, data)
+#define OSWriteLittleInt32(base, byteOffset, data) _OSWriteInt32(base, byteOffset, data)
+#define OSWriteLittleInt64(base, byteOffset, data) _OSWriteInt64(base, byteOffset, data)
 
 /* Host endianess to big endian byte swapping macros for constants. */
 
@@ -489,32 +297,9 @@ OSWriteLittleInt64(
 
 /* Generic host endianess to little endian byte swapping functions. */
 
-OS_INLINE
-uint16_t
-OSSwapHostToLittleInt16(
-    uint16_t                        data
-)
-{
-    return data;
-}
-
-OS_INLINE
-uint32_t
-OSSwapHostToLittleInt32(
-    uint32_t                        data
-)
-{
-    return data;
-}
-
-OS_INLINE
-uint64_t
-OSSwapHostToLittleInt64(
-    uint64_t                        data
-)
-{
-    return data;
-}
+#define OSSwapHostToLittleInt16(x) ((uint16_t)(x))
+#define OSSwapHostToLittleInt32(x) ((uint32_t)(x))
+#define OSSwapHostToLittleInt64(x) ((uint64_t)(x))
 
 /* Big endian to host endianess byte swapping macros for constants. */
 
@@ -536,32 +321,9 @@ OSSwapHostToLittleInt64(
 
 /* Generic little endian to host endianess byte swapping functions. */
 
-OS_INLINE
-uint16_t
-OSSwapLittleToHostInt16(
-    uint16_t                        data
-)
-{
-    return data;
-}
-
-OS_INLINE
-uint32_t
-OSSwapLittleToHostInt32(
-    uint32_t                        data
-)
-{
-    return data;
-}
-
-OS_INLINE
-uint64_t
-OSSwapLittleToHostInt64(
-    uint64_t                        data
-)
-{
-    return data;
-}
+#define OSSwapLittleToHostInt16(x) ((uint16_t)(x))
+#define OSSwapLittleToHostInt32(x) ((uint32_t)(x))
+#define OSSwapLittleToHostInt64(x) ((uint64_t)(x))
 
 #else
 #error Unknown endianess.
diff --git a/libkern/libkern/OSCrossEndian.h b/libkern/libkern/OSCrossEndian.h
index 345c2c313..0bbbf58e3 100644
--- a/libkern/libkern/OSCrossEndian.h
+++ b/libkern/libkern/OSCrossEndian.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -58,27 +58,30 @@
 #ifndef _LIBKERN_OSCROSSENDIAN_H
 #define _LIBKERN_OSCROSSENDIAN_H
 
+#include <sys/sysctl.h>
+
 #if __ppc__
 
-static __inline__ int _OSRosettaCheck(void)
+static __inline__ int
+_OSRosettaCheck(void)
 {
-    int isCrossEndian;
+	int isCrossEndian = 0;
+	int val = 0;
+	size_t size = sizeof val;
 
-    __asm__ (  "b 0f\n"
-	    "	.long 0x14400004\n"
-	    "	li %0,1\n"
-	    "0:"
-	: "=r" (isCrossEndian) : "0" (0)
-    );
+	if (sysctlbyname("sysctl.proc_native", &val, &size, NULL, 0) == -1)
+		isCrossEndian = 0;
+	else
+		isCrossEndian = val ? 0 : 1;
 
-    return isCrossEndian;
+	return isCrossEndian;
 }
 
-#else
+#else /* __ppc__ */
 
 static __inline__ int _OSRosettaCheck(void) { return 0; }
 
-#endif
+#endif /* __ppc__ */
 
 #define IF_ROSETTA() if (__builtin_expect(_OSRosettaCheck(), 0) )
 
diff --git a/libkern/libkern/c++/Makefile b/libkern/libkern/c++/Makefile
index 3c98e5543..c2660a21b 100644
--- a/libkern/libkern/c++/Makefile
+++ b/libkern/libkern/c++/Makefile
@@ -27,6 +27,7 @@ DATAFILES = \
           OSCPPDebug.h \
           OSData.h \
           OSDictionary.h \
+          OSEndianTypes.h \
           OSIterator.h \
           OSLib.h \
           OSMetaClass.h \
diff --git a/libkern/libkern/c++/OSEndianTypes.h b/libkern/libkern/c++/OSEndianTypes.h
new file mode 100644
index 000000000..8a527d460
--- /dev/null
+++ b/libkern/libkern/c++/OSEndianTypes.h
@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ *
+ * HISTORY
+ *   gvdl    20050620    Created
+ */
+
+/*!
+@header OSEndianTypes
+@abstract C++ inline types for byte swapping
+@discussion
+The OSEndianTypes consist of a number of types that are used very similarly to the traditional MacOS C scalar integers types, eg. UInt32 and SInt32.
+@copyright 2005 Apple Computer, Inc. All rights reserved.
+@updated 2005-07-25
+*/
+
+// Header doc magic trick for simple documentation
+#if 0
+/*!  @typedef BigUInt16
+@abstract A Big-endian unsigned integer scalar size 16 - UInt16 */
+typedef class BigUInt16 BigUInt16;
+#endif
+
+#if 0
+/*! @typedef BigSInt16
+@abstract A Big-endian signed integer scalar size 16 - SInt16 */
+typedef class BigSInt16 BigSInt16;
+
+/*! @typedef BigUInt32
+@abstract A Big-endian unsigned integer scalar size 32 - UInt32 */
+typedef class BigUInt32 BigUInt32;
+
+/*! @typedef BigSInt32
+@abstract A Big-endian signed integer scalar size 32 - SInt32 */
+typedef class BigSInt32 BigSInt32;
+
+/*! @typedef BigUInt64
+@abstract A Big-endian unsigned integer scalar size 64 - UInt64 */
+typedef class BigUInt64 BigUInt64;
+
+/*! @typedef BigSInt64
+@abstract A Big-endian signed integer scalar size 64 - SInt64 */
+typedef class BigSInt64 BigSInt64;
+
+/*! @typedef LittleUInt16
+@abstract A Little-endian unsigned integer scalar size 16 - UInt16 */
+typedef class LittleUInt16 LittleUInt16;
+
+/*! @typedef LittleSInt16
+@abstract A Little-endian signed integer scalar size 16 - SInt16 */
+typedef class LittleSInt16 LittleSInt16;
+
+/*! @typedef LittleUInt32
+@abstract A Little-endian unsigned integer scalar size 32 - UInt32 */
+typedef class LittleUInt32 LittleUInt32;
+
+/*! @typedef LittleSInt32
+@abstract A Little-endian signed integer scalar size 32 - SInt32 */
+typedef class LittleSInt32 LittleSInt32;
+
+/*! @typedef LittleUInt64
+@abstract A Little-endian unsigned integer scalar size 64 - UInt64 */
+typedef class LittleUInt64 LittleUInt64;
+
+/*! @typedef LittleSInt64
+@abstract A Little-endian signed integer scalar size 64 - SInt64 */
+typedef class LittleSInt64 LittleSInt64;
+
+*/
+#endif
+
+#ifndef _OS_OSENDIANHELPER_H
+#define _OS_OSENDIANHELPER_H
+
+#if __cplusplus
+
+#include <libkern/OSTypes.h>
+#include <libkern/OSByteOrder.h>
+
+// Probably should really be using templates, this is one of the few cases
+// where they do make sense.  But as the kernel is not allowed to export
+// template based C++ APIs we have to use sophisticated macros instead
+#define __OSEndianSignIntSizeDEF(argname, argend, argtype, argsize) {	\
+public:									\
+    typedef argtype ## argsize		Value;				\
+									\
+private:								\
+    typedef UInt ## argsize		UValue;				\
+    UValue mValue;							\
+									\
+    void writeValue(Value v) {						\
+	if (__builtin_constant_p(v))					\
+	    mValue = OSSwapHostTo ## argend ## ConstInt ## argsize(v);	\
+	else								\
+	    OSWrite ## argend ## Int ## argsize(&mValue, 0, (UValue) v);\
+    };									\
+									\
+    Value readValue() const {						\
+	return (Value) OSRead ## argend ## Int ## argsize(&mValue, 0);	\
+    };									\
+									\
+public:									\
+    argname() { };							\
+									\
+    argname (Value v) { writeValue(v); };				\
+    argname  &operator = (Value v) { writeValue(v); return *this; }	\
+									\
+    Value get() const { return readValue(); };				\
+    operator Value () const { return readValue(); };			\
+}
+
+class BigUInt16    __OSEndianSignIntSizeDEF(BigUInt16,    Big,    UInt, 16);
+class BigSInt16    __OSEndianSignIntSizeDEF(BigSInt16,    Big,    SInt, 16);
+class BigUInt32    __OSEndianSignIntSizeDEF(BigUInt32,    Big,    UInt, 32);
+class BigSInt32    __OSEndianSignIntSizeDEF(BigSInt32,    Big,    SInt, 32);
+class BigUInt64    __OSEndianSignIntSizeDEF(BigUInt64,    Big,    UInt, 64);
+class BigSInt64    __OSEndianSignIntSizeDEF(BigSInt64,    Big,    SInt, 64);
+class LittleUInt16 __OSEndianSignIntSizeDEF(LittleUInt16, Little, UInt, 16);
+class LittleSInt16 __OSEndianSignIntSizeDEF(LittleSInt16, Little, SInt, 16);
+class LittleUInt32 __OSEndianSignIntSizeDEF(LittleUInt32, Little, UInt, 32);
+class LittleSInt32 __OSEndianSignIntSizeDEF(LittleSInt32, Little, SInt, 32);
+class LittleUInt64 __OSEndianSignIntSizeDEF(LittleUInt64, Little, UInt, 64);
+class LittleSInt64 __OSEndianSignIntSizeDEF(LittleSInt64, Little, SInt, 64);
+
+#undef __OSEndianSignIntSizeDEF
+
+#endif /* __cplusplus */
+
+#endif /* ! _OS_OSENDIANHELPER_H */
+
+
diff --git a/libkern/libkern/c++/OSMetaClass.h b/libkern/libkern/c++/OSMetaClass.h
index fa49bd672..814fe0841 100644
--- a/libkern/libkern/c++/OSMetaClass.h
+++ b/libkern/libkern/c++/OSMetaClass.h
@@ -40,7 +40,7 @@ class OSSymbol;
 class OSDictionary;
 class OSSerialize;
 
-#if __GNUC__ < 3
+#if !defined(__ppc__) || __GNUC__ < 3
 #define APPLE_KEXT_COMPATIBILITY
 #else
 #define APPLE_KEXT_COMPATIBILITY __attribute__ ((apple_kext_compatibility))
@@ -365,7 +365,7 @@ protected:
 
     // Needs to be overriden as NULL as all OSMetaClass objects are allocated
     // statically at compile time, don't accidently try to free them.
-    void operator delete(void *mem, size_t size) { };
+    void operator delete(void *, size_t) { };
 
 public:
     static const OSMetaClass * const metaClass;
diff --git a/libkern/libkern/c++/OSObject.h b/libkern/libkern/c++/OSObject.h
index daabb9315..82fae0a6a 100644
--- a/libkern/libkern/c++/OSObject.h
+++ b/libkern/libkern/c++/OSObject.h
@@ -168,6 +168,8 @@ public:
     OSMetaClassDeclareReservedUnused(OSObject, 13);
     OSMetaClassDeclareReservedUnused(OSObject, 14);
     OSMetaClassDeclareReservedUnused(OSObject, 15);
+
+#ifdef __ppc__
     OSMetaClassDeclareReservedUnused(OSObject, 16);
     OSMetaClassDeclareReservedUnused(OSObject, 17);
     OSMetaClassDeclareReservedUnused(OSObject, 18);
@@ -184,6 +186,7 @@ public:
     OSMetaClassDeclareReservedUnused(OSObject, 29);
     OSMetaClassDeclareReservedUnused(OSObject, 30);
     OSMetaClassDeclareReservedUnused(OSObject, 31);
+#endif
 };
 
 #endif /* !_LIBKERN_OSOBJECT_H */
diff --git a/libkern/libkern/i386/OSByteOrder.h b/libkern/libkern/i386/OSByteOrder.h
index 5dde65e05..49336d991 100644
--- a/libkern/libkern/i386/OSByteOrder.h
+++ b/libkern/libkern/i386/OSByteOrder.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -32,7 +32,13 @@
 #include <stdint.h>
 
 #if !defined(OS_INLINE)
+# if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
 #        define OS_INLINE static inline
+# elif defined(__MWERKS__) || defined(__cplusplus)
+#        define OS_INLINE static inline
+# else
+#        define OS_INLINE static __inline__
+# endif
 #endif
 
 /* Generic byte swapping functions. */
@@ -43,8 +49,7 @@ _OSSwapInt16(
     uint16_t        data
 )
 {
-    __asm__ ("xchgb %b0, %h0" : "+q" (data));
-    return data;
+    return ((data << 8) | (data >> 8));
 }
 
 OS_INLINE
@@ -53,28 +58,36 @@ _OSSwapInt32(
     uint32_t        data
 )
 {
-    __asm__ ("bswap %0" : "+r" (data));
+    __asm__ ("bswap   %0" : "+r" (data));
     return data;
 }
 
+#if defined(__i386__)
 OS_INLINE
 uint64_t
 _OSSwapInt64(
     uint64_t        data
 )
 {
-    union {
-        uint64_t ull;
-        uint32_t ul[2];
-    } u;
-
-    /* This actually generates the best code */
-    u.ul[0] = data >> 32;
-    u.ul[1] = data & 0xffffffff;
-    u.ul[0] = _OSSwapInt32(u.ul[0]);
-    u.ul[1] = _OSSwapInt32(u.ul[1]);
-    return u.ull;
+    __asm__ ("bswap   %%eax\n\t"
+             "bswap   %%edx\n\t" 
+             "xchgl   %%eax, %%edx"
+             : "+A" (data));
+    return data;
 }
+#elif defined(__x86_64__)
+OS_INLINE
+uint64_t
+_OSSwapInt64(
+    uint64_t        data
+)
+{
+    __asm__ ("bswap   %0" : "+r" (data));
+    return data;
+}
+#else
+#error Unknown architecture
+#endif
 
 /* Functions for byte reversed loads. */
 
@@ -82,12 +95,12 @@ OS_INLINE
 uint16_t
 OSReadSwapInt16(
     const volatile void   * base,
-    uintptr_t       offset
+    uintptr_t       byteOffset
 )
 {
     uint16_t result;
 
-    result = *(volatile uint16_t *)((uintptr_t)base + offset);
+    result = *(volatile uint16_t *)((uintptr_t)base + byteOffset);
     return _OSSwapInt16(result);
 }
 
@@ -95,12 +108,12 @@ OS_INLINE
 uint32_t
 OSReadSwapInt32(
     const volatile void   * base,
-    uintptr_t       offset
+    uintptr_t       byteOffset
 )
 {
     uint32_t result;
 
-    result = *(volatile uint32_t *)((uintptr_t)base + offset);
+    result = *(volatile uint32_t *)((uintptr_t)base + byteOffset);
     return _OSSwapInt32(result);
 }
 
@@ -108,21 +121,13 @@ OS_INLINE
 uint64_t
 OSReadSwapInt64(
     const volatile void   * base,
-    uintptr_t       offset
+    uintptr_t       byteOffset
 )
 {
-    const volatile uint32_t * inp;
-    union ullc {
-        uint64_t     ull;
-        uint32_t     ul[2];
-    } outv;
-
-    inp = (const volatile uint32_t *)((uintptr_t)base + offset);
-    outv.ul[0] = inp[1];
-    outv.ul[1] = inp[0];
-    outv.ul[0] = _OSSwapInt32(outv.ul[0]);
-    outv.ul[1] = _OSSwapInt32(outv.ul[1]);
-    return outv.ull;
+    uint64_t result;
+
+    result = *(volatile uint64_t *)((uintptr_t)base + byteOffset);
+    return _OSSwapInt64(result);
 }
 
 /* Functions for byte reversed stores. */
@@ -131,33 +136,33 @@ OS_INLINE
 void
 OSWriteSwapInt16(
     volatile void   * base,
-    uintptr_t       offset,
+    uintptr_t       byteOffset,
     uint16_t        data
 )
 {
-    *(volatile uint16_t *)((uintptr_t)base + offset) = _OSSwapInt16(data);
+    *(volatile uint16_t *)((uintptr_t)base + byteOffset) = _OSSwapInt16(data);
 }
 
 OS_INLINE
 void
 OSWriteSwapInt32(
     volatile void   * base,
-    uintptr_t       offset,
+    uintptr_t       byteOffset,
     uint32_t        data
 )
 {
-    *(volatile uint32_t *)((uintptr_t)base + offset) = _OSSwapInt32(data);
+    *(volatile uint32_t *)((uintptr_t)base + byteOffset) = _OSSwapInt32(data);
 }
 
 OS_INLINE
 void
 OSWriteSwapInt64(
     volatile void    * base,
-    uintptr_t        offset,
+    uintptr_t        byteOffset,
     uint64_t         data
 )
 {
-    *(volatile uint64_t *)((uintptr_t)base + offset) = _OSSwapInt64(data);
+    *(volatile uint64_t *)((uintptr_t)base + byteOffset) = _OSSwapInt64(data);
 }
 
 #endif /* ! _OS_OSBYTEORDERI386_H */
diff --git a/libkern/libkern/machine/OSByteOrder.h b/libkern/libkern/machine/OSByteOrder.h
index ac25ccee8..48602154e 100644
--- a/libkern/libkern/machine/OSByteOrder.h
+++ b/libkern/libkern/machine/OSByteOrder.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -25,13 +25,6 @@
  * 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/*
- * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved. 
- *
- * HISTORY
- *
- */
-
 
 #ifndef _OS_OSBYTEORDERMACHINE_H
 #define _OS_OSBYTEORDERMACHINE_H
@@ -39,164 +32,112 @@
 #include <stdint.h>
 
 #if !defined(OS_INLINE)
+# if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+#        define OS_INLINE static inline
+# elif defined(__MWERKS__) || defined(__cplusplus)
 #        define OS_INLINE static inline
+# else
+#        define OS_INLINE static __inline__
+# endif
 #endif
 
-/* Functions for byte reversed loads. */
+/* Generic byte swapping functions. */
 
 OS_INLINE
 uint16_t
-OSReadSwapInt16(
-    volatile void               * base,
-    uintptr_t                     offset
+_OSSwapInt16(
+    uint16_t                      data
 )
 {
-    union sconv {
-	uint16_t us;
-	uint8_t  uc[2];
-    } *inp, outv;
-    inp = (union sconv *)((uint8_t *)base + offset);
-    outv.uc[0] = inp->uc[1];
-    outv.uc[1] = inp->uc[0];
-    return (outv.us);
+    return OSSwapConstInt16(data);
 }
 
 OS_INLINE
 uint32_t
-OSReadSwapInt32(
-    volatile void               * base,
-    uintptr_t                     offset
+_OSSwapInt32(
+    uint32_t                      data
 )
 {
-    union lconv {
-	uint32_t ul;
-	uint8_t  uc[4];
-    } *inp, outv;
-    inp = (union lconv *)((uint8_t *)base + offset);
-    outv.uc[0] = inp->uc[3];
-    outv.uc[1] = inp->uc[2];
-    outv.uc[2] = inp->uc[1];
-    outv.uc[3] = inp->uc[0];
-    return (outv.ul);
+    return OSSwapConstInt32(data);
 }
 
 OS_INLINE
 uint64_t
-OSReadSwapInt64(
-    volatile void               * base,
-    uintptr_t                     offset
+_OSSwapInt64(
+    uint64_t                        data
 )
 {
-    union llconv {
-	uint64_t ull;
-	uint8_t  uc[8];
-    } *inp, outv;
-    inp = (union llconv *)((uint8_t *)base + offset);
-    outv.uc[0] = inp->uc[7];
-    outv.uc[1] = inp->uc[6];
-    outv.uc[2] = inp->uc[5];
-    outv.uc[3] = inp->uc[4];
-    outv.uc[4] = inp->uc[3];
-    outv.uc[5] = inp->uc[2];
-    outv.uc[6] = inp->uc[1];
-    outv.uc[7] = inp->uc[0];
-    return (outv.ull);
+    return OSSwapConstInt64(data);
 }
 
-/* Functions for byte reversed stores. */
+/* Functions for byte reversed loads. */
 
 OS_INLINE
-void
-OSWriteSwapInt16(
-    volatile void               * base,
-    uintptr_t                     offset,
-    uint16_t                      data
+uint16_t
+OSReadSwapInt16(
+    const volatile void               * base,
+    uintptr_t                     byteOffset
 )
 {
-    union sconv {
-	uint16_t us;
-	uint8_t  uc[2];
-    } *inp, *outp;
-    inp  = (union sconv *)((uint8_t *)base + offset);
-    outp = (union sconv *)&data;
-    outp->uc[0] = inp->uc[1];
-    outp->uc[1] = inp->uc[0];
+    uint16_t data = *(volatile uint16_t *)((uintptr_t)base + byteOffset);
+    return _OSSwapInt16(data);
 }
 
 OS_INLINE
-void
-OSWriteSwapInt32(
-    volatile void               * base,
-    uintptr_t                     offset,
-    uint32_t                      data
+uint32_t
+OSReadSwapInt32(
+    const volatile void               * base,
+    uintptr_t                     byteOffset
 )
 {
-    union lconv {
-	uint32_t ul;
-	uint8_t  uc[4];
-    } *inp, *outp;
-    inp  = (union lconv *)((uint8_t *)base + offset);
-    outp = (union lconv *)&data;
-    outp->uc[0] = inp->uc[3];
-    outp->uc[1] = inp->uc[2];
-    outp->uc[2] = inp->uc[1];
-    outp->uc[3] = inp->uc[0];
+    uint32_t data = *(volatile uint32_t *)((uintptr_t)base + byteOffset);
+    return _OSSwapInt32(data);
 }
 
 OS_INLINE
-void
-OSWriteSwapInt64(
-    volatile void               * base,
-    uintptr_t                     offset,
-    uint64_t                      data
+uint64_t
+OSReadSwapInt64(
+    const volatile void               * base,
+    uintptr_t                     byteOffset
 )
 {
-    union llconv {
-	uint64_t ull;
-	uint8_t  uc[8];
-    } *inp, *outp;
-    inp = (union llconv *)((uint8_t *)base + offset);
-    outp = (union llconv *)&data;
-    outp->uc[0] = inp->uc[7];
-    outp->uc[1] = inp->uc[6];
-    outp->uc[2] = inp->uc[5];
-    outp->uc[3] = inp->uc[4];
-    outp->uc[4] = inp->uc[3];
-    outp->uc[5] = inp->uc[2];
-    outp->uc[6] = inp->uc[1];
-    outp->uc[7] = inp->uc[0];
+    uint64_t data = *(volatile uint64_t *)((uintptr_t)base + byteOffset);
+    return _OSSwapInt64(data);
 }
 
-/* Generic byte swapping functions. */
+/* Functions for byte reversed stores. */
 
 OS_INLINE
-uint16_t
-_OSSwapInt16(
+void
+OSWriteSwapInt16(
+    volatile void               * base,
+    uintptr_t                     byteOffset,
     uint16_t                      data
 )
 {
-    uint16_t temp = data;
-    return OSReadSwapInt16(&temp, 0);
+    *(volatile uint16_t *)((uintptr_t)base + byteOffset) = _OSSwapInt16(data);
 }
 
 OS_INLINE
-uint32_t
-_OSSwapInt32(
+void
+OSWriteSwapInt32(
+    volatile void               * base,
+    uintptr_t                     byteOffset,
     uint32_t                      data
 )
 {
-    uint32_t temp = data;
-    return OSReadSwapInt32(&temp, 0);
+    *(volatile uint32_t *)((uintptr_t)base + byteOffset) = _OSSwapInt32(data);
 }
 
 OS_INLINE
-uint64_t
-_OSSwapInt64(
-    uint64_t                        data
+void
+OSWriteSwapInt64(
+    volatile void               * base,
+    uintptr_t                     byteOffset,
+    uint64_t                      data
 )
 {
-    uint64_t temp = data;
-    return OSReadSwapInt64(&temp, 0);
+    *(volatile uint64_t *)((uintptr_t)base + byteOffset) = _OSSwapInt64(data);
 }
 
 #endif /* ! _OS_OSBYTEORDERMACHINE_H */
diff --git a/libkern/libkern/ppc/OSByteOrder.h b/libkern/libkern/ppc/OSByteOrder.h
index 0d0dad59f..60b3edc20 100644
--- a/libkern/libkern/ppc/OSByteOrder.h
+++ b/libkern/libkern/ppc/OSByteOrder.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -25,13 +25,6 @@
  * 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/*
- * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved. 
- *
- * HISTORY
- *
- */
-
 
 #ifndef _OS_OSBYTEORDERPPC_H
 #define _OS_OSBYTEORDERPPC_H
@@ -39,7 +32,13 @@
 #include <stdint.h>
 
 #if !defined(OS_INLINE)
+# if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
 #        define OS_INLINE static inline
+# elif defined(__MWERKS__) || defined(__cplusplus)
+#        define OS_INLINE static inline
+# else
+#        define OS_INLINE static __inline__
+# endif
 #endif
 
 /* Functions for byte reversed loads. */
@@ -47,50 +46,53 @@
 OS_INLINE
 uint16_t
 OSReadSwapInt16(
-    const volatile void               * base,
-    uintptr_t                          offset
+    const volatile void * base,
+    uintptr_t             byteOffset
 )
 {
     uint16_t result;
-    __asm__ volatile("lhbrx %0, %1, %2"
-                     : "=r" (result)
-                     : "b%"  (base), "r" (offset)
-                     : "memory");
+    volatile uint16_t *addr = (volatile uint16_t *)((uintptr_t)base + byteOffset);
+
+    __asm__ ("lhbrx %0, %2, %1"
+             : "=r" (result)
+             : "r"  (base), "bO" (byteOffset), "m" (*addr));
     return result;
 }
 
 OS_INLINE
 uint32_t
 OSReadSwapInt32(
-    const volatile void               * base,
-    uintptr_t                          offset
+    const volatile void * base,
+    uintptr_t             byteOffset
 )
 {
     uint32_t result;
-    __asm__ volatile("lwbrx %0, %1, %2"
-                     : "=r" (result)
-                     : "b%"  (base), "r" (offset)
-                     : "memory");
+    volatile uint32_t *addr = (volatile uint32_t *)((uintptr_t)base + byteOffset);
+
+    __asm__ ("lwbrx %0, %2, %1"
+             : "=r" (result)
+             : "r"  (base), "bO" (byteOffset), "m" (*addr));
     return result;
 }
 
 OS_INLINE
 uint64_t
 OSReadSwapInt64(
-    const volatile void               * base,
-    uintptr_t                          offset
+    const volatile void * base,
+    uintptr_t             byteOffset
 )
 {
-    const volatile uint64_t * inp;
-    union ullc {
-        uint64_t     ull;
-        uint32_t     ul[2];
-    } outv;
-
-    inp = (const volatile uint64_t *)base;
-    outv.ul[0] = OSReadSwapInt32(inp, offset + 4);
-    outv.ul[1] = OSReadSwapInt32(inp, offset);
-    return outv.ull;
+    volatile uint64_t *addr = (volatile uint64_t *)((uintptr_t)base + byteOffset);
+    union {
+        uint64_t u64;
+        uint32_t u32[2];
+    } u;
+
+    __asm__ ("lwbrx %0, %3, %2\n\t"
+             "lwbrx %1, %4, %2"
+             : "=&r" (u.u32[1]), "=r" (u.u32[0])
+             : "r"  (base), "bO" (byteOffset), "b" (byteOffset + 4), "m" (*addr));
+    return u.u64;
 }
 
 /* Functions for byte reversed stores. */
@@ -98,49 +100,49 @@ OSReadSwapInt64(
 OS_INLINE
 void
 OSWriteSwapInt16(
-    volatile void               * base,
-    uintptr_t                          offset,
-    uint16_t                        data
+    volatile void * base,
+    uintptr_t       byteOffset,
+    uint16_t        data
 )
 {
-    __asm__ volatile("sthbrx %0, %1, %2"
-                     :
-                     : "r" (data), "b%" (base), "r" (offset)
-                     : "memory");
+    volatile uint16_t *addr = (volatile uint16_t *)((uintptr_t)base + byteOffset);
+
+    __asm__ ("sthbrx %1, %3, %2"
+             : "=m" (*addr)
+             : "r" (data), "r" (base), "bO" (byteOffset));
 }
 
 OS_INLINE
 void
 OSWriteSwapInt32(
-    volatile void               * base,
-    uintptr_t                          offset,
-    uint32_t                        data
+    volatile void * base,
+    uintptr_t       byteOffset,
+    uint32_t        data
 )
 {
-    __asm__ volatile("stwbrx %0, %1, %2"
-                     :
-                     : "r" (data), "b%" (base), "r" (offset)
-                     : "memory" );
+    volatile uint32_t *addr = (volatile uint32_t *)((uintptr_t)base + byteOffset);
+
+    __asm__ ("stwbrx %1, %3, %2"
+             : "=m" (*addr)
+             : "r" (data), "r" (base), "bO" (byteOffset));
 }
 
 OS_INLINE
 void
 OSWriteSwapInt64(
-    volatile void               * base,
-    uintptr_t                          offset,
-    uint64_t                        data
+    volatile void * base,
+    uintptr_t       byteOffset,
+    uint64_t        data
 )
 {
-    volatile uint64_t * outp;
-    volatile union ullc {
-        uint64_t     ull;
-        uint32_t     ul[2];
-    } *inp;
-
-    outp = (volatile uint64_t *)base;
-    inp  = (volatile union ullc *)&data;
-    OSWriteSwapInt32(outp, offset, inp->ul[1]);
-    OSWriteSwapInt32(outp, offset + 4, inp->ul[0]);
+    volatile uint64_t *addr = (volatile uint64_t *)((uintptr_t)base + byteOffset);
+    uint32_t hi = data >> 32;
+    uint32_t lo = data & 0xffffffff;
+
+    __asm__ ("stwbrx %1, %4, %3\n\t"
+             "stwbrx %2, %5, %3"
+             : "=m" (*addr)
+             : "r" (lo), "r" (hi), "r" (base), "bO" (byteOffset), "b" (byteOffset + 4));
 }
 
 /* Generic byte swapping functions. */
@@ -148,31 +150,28 @@ OSWriteSwapInt64(
 OS_INLINE
 uint16_t
 _OSSwapInt16(
-    uint16_t                        data
+    uint16_t data
 )
 {
-    uint16_t temp = data;
-    return OSReadSwapInt16(&temp, 0);
+    return OSReadSwapInt16(&data, 0);
 }
 
 OS_INLINE
 uint32_t
 _OSSwapInt32(
-    uint32_t                        data
+    uint32_t data
 )
 {
-    uint32_t temp = data;
-    return OSReadSwapInt32(&temp, 0);
+    return OSReadSwapInt32(&data, 0);
 }
 
 OS_INLINE
 uint64_t
 _OSSwapInt64(
-    uint64_t                        data
+    uint64_t data
 )
 {
-    uint64_t temp = data;
-    return OSReadSwapInt64(&temp, 0);
+    return OSReadSwapInt64(&data, 0);
 }
 
 #endif /* ! _OS_OSBYTEORDERPPC_H */
diff --git a/libkern/mach-o/loader.h b/libkern/mach-o/loader.h
index 741038d20..2e4e8e9f6 100644
--- a/libkern/mach-o/loader.h
+++ b/libkern/mach-o/loader.h
@@ -50,12 +50,6 @@
  */
 #include <mach/machine/thread_status.h>
 
-/*
- * XXX historically, we have not included this header.  Continue to not do so.
- *
- * #include <architecture/byte_order.h>
- */
-
 /*
  * The mach header appears at the very beginning of the object file; it
  * is the same for both 32-bit and 64-bit architectures.
@@ -72,11 +66,11 @@ struct mach_header {
 
 /* Constant for the magic field of the mach_header (32-bit architectures) */
 #define	MH_MAGIC	0xfeedface	/* the mach magic number */
-#define MH_CIGAM	NXSwapInt(MH_MAGIC)
+#define MH_CIGAM	0xcefaedfe	
 
 /* Constant for the magic field of the mach_header_64 (64-bit architectures) */
 #define MH_MAGIC_64	0xfeedfacf	/* the 64-bit mach magic number */
-#define MH_CIGAM_64	NXSwapInt(MH_MAGIC_64)
+#define MH_CIGAM_64	0xcffaedfe	
 
 /* Constants for the cmd field of new load commands, the type */
 #define LC_SEGMENT_64	0x19	/* 64-bit segment of this file to be mapped */
diff --git a/libsa/catalogue.cpp b/libsa/catalogue.cpp
index 1c4c563bf..8d8b019db 100644
--- a/libsa/catalogue.cpp
+++ b/libsa/catalogue.cpp
@@ -29,6 +29,7 @@
 #include <IOKit/IODeviceTreeSupport.h>
 #include <IOKit/IORegistryEntry.h>
 #include <IOKit/IOCatalogue.h>
+#include <IOKit/IOKitKeysPrivate.h>
 #include <libkern/c++/OSUnserialize.h>
 #include <libkern/OSByteOrder.h>
 #include <libsa/catalogue.h>
@@ -117,6 +118,7 @@ bool validateExtensionDict(OSDictionary * extension, int index) {
     bool id_missing = false;
     bool is_kernel_resource = false;
     bool has_executable = false;
+    bool ineligible_for_safe_boot = false;
     OSString * bundleIdentifier = NULL;    // do not release
     OSObject * rawValue = NULL;            // do not release
     OSString * stringValue = NULL;         // do not release
@@ -127,6 +129,7 @@ bool validateExtensionDict(OSDictionary * extension, int index) {
     OSString * key = NULL;                 // do not release
     VERS_version vers;
     VERS_version compatible_vers;
+    char namep[16];      // unused but needed for PE_parse_boot_arg()
 
     // Info dict is a dictionary
     if (!OSDynamicCast(OSDictionary, extension)) {
@@ -349,9 +352,10 @@ bool validateExtensionDict(OSDictionary * extension, int index) {
         keyIterator = NULL;
     }
 
-    // OSBundleRequired is a legal value - *not* required at boot time
-    // so we can do install CDs and the like with mkext files containing
-    // all normally-used drivers.
+    // OSBundleRequired, if present, must have a legal value.
+    // If it is not present and if we are safe-booting,
+    // then the kext is not eligible.
+    //
     rawValue = extension->getObject("OSBundleRequired");
     if (rawValue) {
         stringValue = OSDynamicCast(OSString, rawValue);
@@ -369,6 +373,10 @@ bool validateExtensionDict(OSDictionary * extension, int index) {
             goto finish;
         }
 
+    } else if (PE_parse_boot_arg("-x", namep)) { /* safe boot */
+        ineligible_for_safe_boot = true;
+        result = false;
+        goto finish;
     }
 
 
@@ -376,19 +384,24 @@ finish:
     if (keyIterator)   keyIterator->release();
 
     if (!result) {
-        if (not_a_dict) {
+        if (ineligible_for_safe_boot) {
+            IOLog(VTYELLOW "Skipping extension \"%s\" during safe boot "
+                "(no OSBundleRequired property)\n"
+                VTRESET,
+                bundleIdentifier->getCStringNoCopy());
+        } else if (not_a_dict) {
             if (index > -1) {
-                IOLog(VTYELLOW "mkext entry %d:." VTRESET, index);
+                IOLog(VTYELLOW "mkext entry %d: " VTRESET, index);
             } else {
-                IOLog(VTYELLOW "kernel extension" VTRESET);
+                IOLog(VTYELLOW "kernel extension " VTRESET);
             }
             IOLog(VTYELLOW "info dictionary isn't a dictionary\n"
                 VTRESET);
         } else if (id_missing) {
             if (index > -1) {
-                IOLog(VTYELLOW "mkext entry %d:." VTRESET, index);
+                IOLog(VTYELLOW "mkext entry %d: " VTRESET, index);
             } else {
-                IOLog(VTYELLOW "kernel extension" VTRESET);
+                IOLog(VTYELLOW "kernel extension " VTRESET);
             }
             IOLog(VTYELLOW "\"CFBundleIdentifier\" property is "
                 "missing or not a string\n"
@@ -712,13 +725,12 @@ OSDictionary * readExtension(OSDictionary * propertyDict,
         bootxDriverDataObject->getBytesNoCopy(0,
         sizeof(MemoryMapFileInfo));
 #if defined (__ppc__)
-    dataBuffer = (BootxDriverInfo *)ml_static_ptovirt(
-      driverInfo->paddr);
+    dataBuffer = (BootxDriverInfo *)ml_static_ptovirt(driverInfo->paddr);
 #elif defined (__i386__)
-    dataBuffer = (BootxDriverInfo *)driverInfo->paddr;
-    dataBuffer->plistAddr = ml_static_ptovirt(dataBuffer->plistAddr);
+    dataBuffer = (BootxDriverInfo *)ml_boot_ptovirt(driverInfo->paddr);
+    dataBuffer->plistAddr = (char *)ml_boot_ptovirt((vm_address_t)dataBuffer->plistAddr);
     if (dataBuffer->moduleAddr)
-      dataBuffer->moduleAddr = ml_static_ptovirt(dataBuffer->moduleAddr);
+      dataBuffer->moduleAddr = (void *)ml_boot_ptovirt((vm_address_t)dataBuffer->moduleAddr);
 #else
 #error unsupported architecture
 #endif
@@ -807,7 +819,7 @@ OSDictionary * readExtension(OSDictionary * propertyDict,
 finish:
 
     if (loaded_kmod) {
-        kfree((unsigned int)loaded_kmod, sizeof(kmod_info_t));
+        kfree(loaded_kmod, sizeof(kmod_info_t));
     }
 
     // do not release bootxDriverDataObject
@@ -957,7 +969,7 @@ bool extractExtensionsFromArchive(MemoryMapFileInfo * mkext_file_info,
 #if defined (__ppc__)
     mkext_data = (mkext_header *)mkext_file_info->paddr;
 #elif defined (__i386__)
-    mkext_data = (mkext_header *)ml_static_ptovirt(mkext_file_info->paddr);
+    mkext_data = (mkext_header *)ml_boot_ptovirt(mkext_file_info->paddr);
 #else
 #error unsupported architecture
 #endif
@@ -989,6 +1001,16 @@ bool extractExtensionsFromArchive(MemoryMapFileInfo * mkext_file_info,
         goto finish;
     }
 
+    IORegistryEntry * root = IORegistryEntry::getRegistryRoot();
+    assert(root);
+    OSData * checksumObj = OSData::withBytes((void *)&checksum,
+        sizeof(checksum));
+    assert(checksumObj);
+    if (checksumObj) {
+        root->setProperty(kIOStartupMkextCRC, checksumObj);
+        checksumObj->release();
+    }
+
    /* If the MKEXT archive isn't fat, check that the CPU type & subtype
     * match that of the running kernel.
     */
@@ -1030,7 +1052,7 @@ bool extractExtensionsFromArchive(MemoryMapFileInfo * mkext_file_info,
          i++) {
 
         if (loaded_kmod) {
-            kfree((unsigned int)loaded_kmod, sizeof(kmod_info_t));
+            kfree(loaded_kmod, sizeof(kmod_info_t));
             loaded_kmod = 0;
         }
 
@@ -1208,7 +1230,7 @@ bool extractExtensionsFromArchive(MemoryMapFileInfo * mkext_file_info,
 
 finish:
 
-    if (loaded_kmod) kfree((unsigned int)loaded_kmod, sizeof(kmod_info_t));
+    if (loaded_kmod) kfree(loaded_kmod, sizeof(kmod_info_t));
     if (driverPlistDataObject) {
         kmem_free(kernel_map,
             (unsigned int)driverPlistDataObject->getBytesNoCopy(),
diff --git a/libsa/kext.cpp b/libsa/kext.cpp
index 3a4556454..f3fed593c 100644
--- a/libsa/kext.cpp
+++ b/libsa/kext.cpp
@@ -172,7 +172,6 @@ bool getKext(
                     if (!uncompressModule(compressedCode, &driverCode)) {
                         IOLog("extension \"%s\": couldn't uncompress code\n",
                             bundleid);
-                        LOG_DELAY(1);
                         result = false;
                         goto finish;
                     }
@@ -323,7 +322,7 @@ bool kextIsDependency(const char * kext_name, char * is_kernel) {
         extDict->getObject("compressedCode"));
 
     if ((driverCode || compressedCode) && is_kernel && *is_kernel) {
-	*is_kernel = 2;
+        *is_kernel = 2;
     }
 
     if (!driverCode && !compressedCode && !isKernelResourceObj) {
@@ -339,8 +338,8 @@ finish:
 /*********************************************************************
 *********************************************************************/
 static bool
-figureDependenciesForKext(OSDictionary * kextPlist,
-    OSDictionary * dependencies,
+addDependenciesForKext(OSDictionary * kextPlist,
+    OSArray   * dependencyList,
     OSString * trueParent,
     Boolean    skipKernelDependencies)
 {
@@ -350,6 +349,7 @@ figureDependenciesForKext(OSDictionary * kextPlist,
     OSDictionary * libraries = 0;  // don't release
     OSCollectionIterator * keyIterator = 0; // must release
     OSString * libraryName = 0; // don't release
+    OSString * dependentName = 0; // don't release
 
     kextName = OSDynamicCast(OSString,
         kextPlist->getObject("CFBundleIdentifier"));
@@ -373,6 +373,8 @@ figureDependenciesForKext(OSDictionary * kextPlist,
         goto finish;
     }
 
+    dependentName = trueParent ? trueParent : kextName;
+
     while ( (libraryName = OSDynamicCast(OSString,
         keyIterator->getNextObject())) ) {
 
@@ -389,12 +391,15 @@ figureDependenciesForKext(OSDictionary * kextPlist,
         } else {
             char is_kernel_component;
 
-            if (!kextIsDependency(libraryName->getCStringNoCopy(), &is_kernel_component))
+            if (!kextIsDependency(libraryName->getCStringNoCopy(),
+                &is_kernel_component)) {
+
                 is_kernel_component = false;
+            }
 
             if (!skipKernelDependencies || !is_kernel_component) {
-                dependencies->setObject(libraryName,
-                    trueParent ? trueParent : kextName);
+                dependencyList->setObject(dependentName);
+                dependencyList->setObject(libraryName);
             }
             if (!hasDirectKernelDependency && is_kernel_component) {
                 hasDirectKernelDependency = true;
@@ -402,11 +407,22 @@ figureDependenciesForKext(OSDictionary * kextPlist,
         }
     }
     if (!hasDirectKernelDependency) {
+        const OSSymbol * kernelName = 0;
+
         /* a kext without any kernel dependency is assumed dependent on 6.0 */
-        dependencies->setObject("com.apple.kernel.libkern",
-                trueParent ? trueParent : kextName);
+        dependencyList->setObject(dependentName);
+
+        kernelName = OSSymbol::withCString("com.apple.kernel.libkern");
+        if (!kernelName) {
+            // XXX: Add log message
+            result = false;
+            goto finish;
+        }
+        dependencyList->setObject(kernelName);
+        kernelName->release();
+
         IOLog("Extension \"%s\" has no kernel dependency.\n",
-        	kextName->getCStringNoCopy());
+            kextName->getCStringNoCopy());
     }
 
 finish:
@@ -452,14 +468,8 @@ bool add_dependencies_for_kmod(const char * kmod_name, dgraph_t * dgraph)
 {
     bool result = true;
     OSDictionary * kextPlist = 0; // don't release
-    OSDictionary * workingDependencies = 0; // must release
-    OSDictionary * pendingDependencies = 0; // must release
-    OSDictionary * swapDict = 0; // don't release
-    OSString * dependentName = 0; // don't release
-    const char * dependent_name = 0;  // don't free
-    OSString * libraryName = 0; // don't release
-    const char * library_name = 0;  // don't free
-    OSCollectionIterator * dependencyIterator = 0; // must release
+    unsigned int index = 0;
+    OSArray * dependencyList = 0;  // must release
     unsigned char * code = 0;
     unsigned long code_length = 0;
     bool code_is_kmem = false;
@@ -467,9 +477,11 @@ bool add_dependencies_for_kmod(const char * kmod_name, dgraph_t * dgraph)
     char is_kernel_component = false;
     dgraph_entry_t * dgraph_entry = 0; // don't free
     dgraph_entry_t * dgraph_dependency = 0; // don't free
-    unsigned int graph_depth = 0;
     bool kext_is_dependency = true;
 
+   /*****
+    * Set up the root kmod.
+    */
     if (!getKext(kmod_name, &kextPlist, &code, &code_length,
         &code_is_kmem)) {
         IOLog("can't find extension %s\n", kmod_name);
@@ -501,8 +513,7 @@ bool add_dependencies_for_kmod(const char * kmod_name, dgraph_t * dgraph)
     }
 
     // pass ownership of code to kld patcher
-    if (code)
-    {
+    if (code) {
         if (kload_map_entry(dgraph_entry) != kload_error_none) {
             IOLog("can't map %s in preparation for loading\n", kmod_name);
             result = false;
@@ -515,95 +526,78 @@ bool add_dependencies_for_kmod(const char * kmod_name, dgraph_t * dgraph)
     code_length = 0;
     code_is_kmem = false;
 
-    workingDependencies = OSDictionary::withCapacity(5);
-    if (!workingDependencies) {
-        IOLog("memory allocation failure\n");
-        result = false;
-        goto finish;
-    }
-
-    pendingDependencies = OSDictionary::withCapacity(5);
-    if (!pendingDependencies) {
+   /*****
+    * Now handle all the dependencies.
+    */
+    dependencyList = OSArray::withCapacity(5);
+    if (!dependencyList) {
         IOLog("memory allocation failure\n");
         result = false;
         goto finish;
     }
 
-    if (!figureDependenciesForKext(kextPlist, workingDependencies, NULL, false)) {
+    index = 0;
+    if (!addDependenciesForKext(kextPlist, dependencyList, NULL, false)) {
         IOLog("can't determine immediate dependencies for extension %s\n",
             kmod_name);
         result = false;
         goto finish;
     }
 
-    graph_depth = 0;
-    while (workingDependencies->getCount()) {
-        if (graph_depth > 255) {
+   /* IMPORTANT: loop condition gets list count every time through, as the
+    * array CAN change each iteration.
+    */
+    for (index = 0; index < dependencyList->getCount(); index += 2) {
+        OSString * dependentName = 0;
+        OSString * libraryName = 0;
+        const char * dependent_name = 0;
+        const char * library_name = 0;
+
+       /* 255 is an arbitrary limit. Multiplied  by 2 because the dependency
+        * list is stocked with pairs (dependent -> dependency).
+        */
+        if (index > (2 * 255)) {
             IOLog("extension dependency graph ridiculously long, indicating a loop\n");
             result = false;
             goto finish;
         }
 
-        if (dependencyIterator) {
-            dependencyIterator->release();
-            dependencyIterator = 0;
-        }
+        dependentName = OSDynamicCast(OSString,
+            dependencyList->getObject(index));
+        libraryName = OSDynamicCast(OSString,
+            dependencyList->getObject(index + 1));
 
-        dependencyIterator = OSCollectionIterator::withCollection(
-            workingDependencies);
-        if (!dependencyIterator) {
-            IOLog("memory allocation failure\n");
+        if (!dependentName || !libraryName) {
+            IOLog("malformed dependency list\n");
             result = false;
             goto finish;
         }
 
-        while ( (libraryName =
-                 OSDynamicCast(OSString, dependencyIterator->getNextObject())) ) {
+        dependent_name = dependentName->getCStringNoCopy();
+        library_name = libraryName->getCStringNoCopy();
 
-            library_name = libraryName->getCStringNoCopy();
+        if (!getKext(library_name, &kextPlist, NULL, NULL, NULL)) {
 
-            dependentName = OSDynamicCast(OSString,
-                workingDependencies->getObject(libraryName));
-
-            dependent_name = dependentName->getCStringNoCopy();
+            IOLog("can't find extension %s\n", library_name);
+            result = false;
+            goto finish;
+        }
 
+        OSString * string = OSDynamicCast(OSString,
+            kextPlist->getObject("OSBundleSharedExecutableIdentifier"));
+        if (string) {
+            library_name = string->getCStringNoCopy();
             if (!getKext(library_name, &kextPlist, NULL, NULL, NULL)) {
                 IOLog("can't find extension %s\n", library_name);
                 result = false;
                 goto finish;
             }
+        }
 
-	    OSString * string;
-	    if ((string = OSDynamicCast(OSString,
-			    kextPlist->getObject("OSBundleSharedExecutableIdentifier"))))
-	    {
-		library_name = string->getCStringNoCopy();
-		if (!getKext(library_name, &kextPlist, NULL, NULL, NULL)) {
-		    IOLog("can't find extension %s\n", library_name);
-		    result = false;
-		    goto finish;
-		}
-	    }
-
-            kext_is_dependency = kextIsDependency(library_name,
-                &is_kernel_component);
-
-            if (!kext_is_dependency) {
-
-               /* For binaryless kexts, add a new pending dependency from the
-                * original dependent onto the dependencies of the current,
-                * binaryless, dependency.
-                */
-                if (!figureDependenciesForKext(kextPlist, pendingDependencies,
-                    dependentName, true)) {
+        kext_is_dependency = kextIsDependency(library_name,
+            &is_kernel_component);
 
-                    IOLog("can't determine immediate dependencies for extension %s\n",
-                        library_name);
-                    result = false;
-                    goto finish;
-                }
-                continue;
-            } else {
+        if (kext_is_dependency) {
                 dgraph_entry = dgraph_find_dependent(dgraph, dependent_name);
                 if (!dgraph_entry) {
                     IOLog("internal error with dependency graph\n");
@@ -658,8 +652,8 @@ bool add_dependencies_for_kmod(const char * kmod_name, dgraph_t * dgraph)
 
            /* Now put the library's dependencies onto the pending set.
             */
-            if (!figureDependenciesForKext(kextPlist, pendingDependencies,
-                NULL, false)) {
+            if (!addDependenciesForKext(kextPlist, dependencyList,
+                kext_is_dependency ? NULL : dependentName, !kext_is_dependency)) {
 
                 IOLog("can't determine immediate dependencies for extension %s\n",
                     library_name);
@@ -668,23 +662,12 @@ bool add_dependencies_for_kmod(const char * kmod_name, dgraph_t * dgraph)
             }
         }
 
-        dependencyIterator->release();
-        dependencyIterator = 0;
-
-        workingDependencies->flushCollection();
-        swapDict = workingDependencies;
-        workingDependencies = pendingDependencies;
-        pendingDependencies = swapDict;
-        graph_depth++;
-    }
-
 finish:
     if (code && code_is_kmem) {
         kmem_free(kernel_map, (unsigned int)code, code_length);
     }
-    if (workingDependencies)  workingDependencies->release();
-    if (pendingDependencies)  pendingDependencies->release();
-    if (dependencyIterator)   dependencyIterator->release();
+    if (dependencyList)  dependencyList->release();
+
     return result;
 }
 
@@ -712,7 +695,7 @@ kern_return_t load_kernel_extension(char * kmod_name)
    /* See if the kmod is already loaded.
     */
     if ((kmod_info = kmod_lookupbyname_locked(kmod_name))) {
-	kfree((vm_offset_t) kmod_info, sizeof(kmod_info_t));
+        kfree(kmod_info, sizeof(kmod_info_t));
         return KERN_SUCCESS;
     }
 
diff --git a/libsa/kld_patch.c b/libsa/kld_patch.c
index 136580bd1..0e3acc86c 100644
--- a/libsa/kld_patch.c
+++ b/libsa/kld_patch.c
@@ -37,6 +37,7 @@
 #include <mach-o/reloc.h>
 #if !KERNEL
 #include <mach-o/swap.h>
+#include <libkern/OSByteOrder.h>
 #endif
 
 #if KERNEL
@@ -630,13 +631,13 @@ kld_macho_swap(struct mach_header * mh)
             cmd < ncmds;
             cmd++, seg = (struct segment_command *)(((vm_offset_t)seg) + seg->cmdsize))
     {
-        if (NXSwapLong(LC_SYMTAB) == seg->cmd) {
+        if (OSSwapConstInt32(LC_SYMTAB) == seg->cmd) {
 	    swap_symtab_command((struct symtab_command *) seg, hostOrder);
 	    swap_nlist((struct nlist *) (((vm_offset_t) mh) + ((struct symtab_command *) seg)->symoff),
 		       ((struct symtab_command *) seg)->nsyms, hostOrder);
 	    continue;
 	}
-        if (NXSwapLong(LC_SEGMENT) != seg->cmd) {
+        if (OSSwapConstInt32(LC_SEGMENT) != seg->cmd) {
 	    swap_load_command((struct load_command *) seg, hostOrder);
             continue;
 	}
@@ -750,21 +751,21 @@ static Boolean findBestArch(struct fileRecord *file, const char *pathName)
 	unsigned long i;
 	struct fat_arch *arch;
 
-	fat->nfat_arch = NXSwapBigLongToHost(fat->nfat_arch);
+	fat->nfat_arch = OSSwapBigToHostInt32(fat->nfat_arch);
 	return_if(file->fMapSize < sizeof(struct fat_header)
 				    + fat->nfat_arch * sizeof(struct fat_arch),
 	    false, ("%s is too fat\n", file->fPath));
 
 	arch = (struct fat_arch *) &fat[1];
 	for (i = 0; i < fat->nfat_arch; i++) {
-	    arch[i].cputype    = NXSwapBigLongToHost(arch[i].cputype);
-	    arch[i].cpusubtype = NXSwapBigLongToHost(arch[i].cpusubtype);
-	    arch[i].offset     = NXSwapBigLongToHost(arch[i].offset);
-	    arch[i].size       = NXSwapBigLongToHost(arch[i].size);
-	    arch[i].align      = NXSwapBigLongToHost(arch[i].align);
+	    arch[i].cputype    = OSSwapBigToHostInt32(arch[i].cputype);
+	    arch[i].cpusubtype = OSSwapBigToHostInt32(arch[i].cpusubtype);
+	    arch[i].offset     = OSSwapBigToHostInt32(arch[i].offset);
+	    arch[i].size       = OSSwapBigToHostInt32(arch[i].size);
+	    arch[i].align      = OSSwapBigToHostInt32(arch[i].align);
 	}
 
-	magic = NXSwapBigLongToHost(fat->magic);
+	magic = OSSwapBigToHostInt32(fat->magic);
     }
 
     // Now see if we can find any valid architectures
@@ -1301,7 +1302,7 @@ relocateSection(const struct fileRecord *file, struct sectionRecord *sectionRec)
 	    void * addr = *entry;
 #if !KERNEL
 	    if (file->fSwapped)
-		addr = (void *) NXSwapLong((long) addr);
+		addr = (void *) OSSwapInt32((uint32_t) addr);
 #endif
 	    symbol = findSymbolByAddress(file, addr);
 	}
@@ -1332,7 +1333,7 @@ findSymbolRefAtLocation(const struct fileRecord *file,
 	    void * addr = *loc;
 #if !KERNEL
 	    if (file->fSwapped)
-		addr = (void *) NXSwapLong((long) addr);
+		addr = (void *) OSSwapInt32((uint32_t) addr);
 #endif
 	    result = findSymbolByAddress(file, addr);
 	    if (!result)
@@ -1732,7 +1733,7 @@ static Boolean resolveKernelVTable(struct metaClassRecord *metaClass)
 	void * addr = *curEntry;
 #if !KERNEL
 	if (file->fSwapped)
-	    addr = (void *) NXSwapLong((long) addr);
+	    addr = (void *) OSSwapInt32((uint32_t) addr);
 #endif
 	curPatch->fSymbol = (struct nlist *) 
 	    findSymbolByAddress(file, addr);
diff --git a/makedefs/MakeInc.cmd b/makedefs/MakeInc.cmd
index b9aa84920..52ee80340 100644
--- a/makedefs/MakeInc.cmd
+++ b/makedefs/MakeInc.cmd
@@ -7,7 +7,7 @@ MD=     /usr/bin/md
 
 RM = /bin/rm -f
 CP = /bin/cp
-LN = /bin/ln -s
+LN = /bin/ln -fs
 CAT = /bin/cat
 MKDIR = /bin/mkdir -p
 
diff --git a/makedefs/MakeInc.def b/makedefs/MakeInc.def
index 73842031c..937a465c3 100644
--- a/makedefs/MakeInc.def
+++ b/makedefs/MakeInc.def
@@ -21,7 +21,7 @@ endif
 #
 ifndef COMPONENT_LIST 
 export COMPONENT_LIST 	= osfmk bsd iokit pexpert libkern libsa
-export COMPONENT_LIST_UC := $(shell echo -n $(COMPONENT_LIST) | $(TR) a-z A-Z)
+export COMPONENT_LIST_UC := $(shell printf "%s" "$(COMPONENT_LIST)" | $(TR) a-z A-Z)
 endif
 ifndef	COMPONENT
 export COMPONENT 	:= $(firstword $(subst /, ,$(shell $(RELPATH) $(SRCROOT) $(SOURCE))))
@@ -39,14 +39,14 @@ endif
 #
 ifndef ARCH_CONFIGS
 ifdef RC_ARCHS
-export ARCH_CONFIGS 	:= $(shell echo -n $(RC_ARCHS) | $(TR) a-z A-Z)
+export ARCH_CONFIGS 	:= $(shell printf "%s" "$(RC_ARCHS)" | $(TR) a-z A-Z)
 else
 export ARCH_CONFIGS 	:= $(shell arch | $(TR) a-z A-Z)
 endif
 endif
 ifdef	ARCH_CONFIG
 ifndef ARCH_CONFIG_LC
-export ARCH_CONFIG_LC 	:= $(shell echo -n $(ARCH_CONFIG) | $(TR) A-Z a-z)
+export ARCH_CONFIG_LC 	:= $(shell printf "%s" "$(ARCH_CONFIG)" | $(TR) A-Z a-z)
 endif
 endif
 
@@ -54,10 +54,19 @@ endif
 # Kernel Configuration options  
 #
 # supported configurations : RELEASE DEBUG PROFILE
-#
+# By default, make wll build RELEASE, otherwise the value of KERNEL_CONFIG
+# will be used as kernel configuration. If KERNEL_CONFIGS (plural) is set
+# it will override KERNEL_CONFIG. Make sure to set KERNEL_CONFIGS because
+# build_all rule loops over it when building.
+
 ifndef KERNEL_CONFIGS
+ifndef KERNEL_CONFIG
 export KERNEL_CONFIGS 	= RELEASE
+else
+export KERNEL_CONFIGS = $(KERNEL_CONFIG)
 endif
+endif
+
 ifndef KERNEL_CONFIG
 export KERNEL_CONFIG 	= $(firstword $(KERNEL_CONFIGS))
 endif
@@ -74,7 +83,7 @@ export INSTALL_ARCHS 	= $(ARCH_CONFIGS)
 else
 export INSTALL_ARCHS 	= $(ARCH_CONFIGS)
 endif
-export INSTALL_ARCHS_LC := $(shell echo -n $(ARCH_CONFIGS) | $(TR) A-Z a-z)
+export INSTALL_ARCHS_LC := $(shell printf "%s" "$(ARCH_CONFIGS)" | $(TR) A-Z a-z)
 endif
 
 export INSTALL_ARCH_DEFAULT	= PPC
@@ -87,9 +96,8 @@ export DEFINES = -DAPPLE -DNeXT -DKERNEL -DKERNEL_PRIVATE -DXNU_KERNEL_PRIVATE -
 #
 # Compiler command
 #
-KCC  = /usr/bin/cc
-KC++ = /usr/bin/c++
-CC   = $(KCC)
+KCC  := $(CC)
+KC++ := $(CXX)
 
 #
 # Compiler warning flags
@@ -112,10 +120,9 @@ MWARNFLAGS_STD = \
 export MWARNFLAGS ?= $(MWARNFLAGS_STD)
 
 CXXWARNFLAGS_STD = \
-	-Wall -Wno-format-y2k -W -Wstrict-prototypes -Wmissing-prototypes \
+	-Wall -Wno-format-y2k -W \
 	-Wpointer-arith -Wreturn-type -Wcast-qual -Wwrite-strings -Wswitch \
-	-Wshadow -Wcast-align -Wchar-subscripts -Winline -Wredundant-decls \
-	-fpermissive
+	-Wshadow -Wcast-align -Wchar-subscripts -Winline -Wredundant-decls
 
 export CXXWARNFLAGS ?= $(CXXWARNFLAGS_STD)
 
@@ -138,15 +145,14 @@ endif
 
 export CFLAGS_GEN = -static -g -nostdinc -nostdlib -no-cpp-precomp \
 	-fno-builtin -finline -fno-keep-inline-functions -msoft-float \
-	-fsigned-bitfields $(OTHER_CFLAGS)
+	-fsigned-bitfields $(OTHER_CFLAGS) -force_cpusubtype_ALL
 
 export CFLAGS_RELEASE 	= 
 export CFLAGS_DEBUG 	= 
 export CFLAGS_PROFILE 	=  -pg
 
 export CFLAGS_PPC 	= -arch ppc -Dppc -DPPC -D__PPC__ -DPAGE_SIZE_FIXED
-export CFLAGS_I386 	= -arch i386 -Di386 -DI386 -D__I386__ -DPAGE_SIZE_FIXED \
-	-march=i686 -mpreferred-stack-boundary=2 -falign-functions=4 -mcpu=pentium4 -force_cpusubtype_ALL
+export CFLAGS_I386 	= -arch i386 -Di386 -DI386 -D__I386__ -DPAGE_SIZE_FIXED
 
 export CFLAGS_RELEASEPPC = -O2 -mcpu=750 -mmultiple -fschedule-insns
 export CFLAGS_RELEASE_TRACEPPC = -O2 -mcpu=750 -mmultiple -fschedule-insns
@@ -162,9 +168,11 @@ export CFLAGS 	= $(CFLAGS_GEN) \
 		  $($(addsuffix $(ARCH_CONFIG), $(addsuffix $(KERNEL_CONFIG),CFLAGS_))) \
 		  $(DEFINES)
 
+export MIGCC = $(CC)
+
 # Default C++ flags
 #
-CXXFLAGS_GEN  = -fno-rtti -fno-exceptions -fcheck-new -fapple-kext -fpermissive
+CXXFLAGS_GEN  = -fno-rtti -fno-exceptions -fcheck-new -fapple-kext
 
 CXXFLAGS      = $(CXXFLAGS_GEN) \
 		  $($(addsuffix $(ARCH_CONFIG),CXXFLAGS_)) \
@@ -173,8 +181,8 @@ CXXFLAGS      = $(CXXFLAGS_GEN) \
 #
 # Assembler command
 #
-AS	= /usr/bin/cc
-S_KCC	= /usr/bin/cc
+AS	= $(CC)
+S_KCC	= $(CC)
 
 #
 # Default SFLAGS
@@ -196,7 +204,7 @@ export SFLAGS 	= $(SFLAGS_GEN) \
 #
 # Linker command
 #
-LD	= /usr/bin/ld
+LD	= $(KC++) -nostdlib
 
 #
 # Default LDFLAGS
@@ -216,15 +224,16 @@ export LDFLAGS_COMPONENT	= $(LDFLAGS_COMPONENT_GEN) \
 
 export LDFLAGS_KERNEL_GEN = \
 	-static \
+	-fapple-kext \
 	-force_cpusubtype_ALL \
-	-e __start \
-	-segalign 0x1000 \
-	-sectalign __TEXT __text 0x1000 \
-	-sectalign __DATA __common 0x1000 \
-	-sectalign __DATA __bss 0x1000 \
-	-sectcreate __PRELINK __text /dev/null \
-	-sectcreate __PRELINK __symtab /dev/null \
-	-sectcreate __PRELINK __info /dev/null
+	-Wl,-e,__start \
+	-Wl,-segalign,0x1000 \
+	-Wl,-sectalign,__TEXT,__text,0x1000 \
+	-Wl,-sectalign,__DATA,__common,0x1000 \
+	-Wl,-sectalign,__DATA,__bss,0x1000 \
+	-Wl,-sectcreate,__PRELINK,__text,/dev/null \
+	-Wl,-sectcreate,__PRELINK,__symtab,/dev/null \
+	-Wl,-sectcreate,__PRELINK,__info,/dev/null
 
 export LDFLAGS_KERNEL_RELEASE 	=
 #  -noseglinkedit
@@ -233,14 +242,14 @@ export LDFLAGS_KERNEL_PROFILE 	=
 
 export LDFLAGS_KERNEL_PPC      = \
 	-arch ppc \
-	-segaddr __VECTORS 0x0 \
-	-segaddr __HIB 0x7000  \
-	-segaddr __TEXT 0xe000
+	-Wl,-segaddr,__VECTORS,0x0 \
+	-Wl,-segaddr,__HIB,0x7000  \
+	-Wl,-segaddr,__TEXT,0xe000
  
 export LDFLAGS_KERNEL_I386     = \
 	-arch i386 \
-	-segaddr __HIB  0xC0100000 \
-	-segaddr __TEXT 0xC0111000
+	-Wl,-segaddr,__HIB,0x100000 \
+	-Wl,-segaddr,__TEXT,0x111000
 
 export LDFLAGS_KERNEL	= $(LDFLAGS_KERNEL_GEN) \
 		  $($(addsuffix $(ARCH_CONFIG),LDFLAGS_KERNEL_)) \
diff --git a/makedefs/MakeInc.dir b/makedefs/MakeInc.dir
index fd5b753b6..06f191b1f 100644
--- a/makedefs/MakeInc.dir
+++ b/makedefs/MakeInc.dir
@@ -384,24 +384,22 @@ clean:
 #
 cscope.files:
 	@echo "Building file list for cscope and tags"
-	@find . -name '*.h' -type f > _cscope.files 2> /dev/null
-	@echo bsd/ufs/ufs/ufs_readwrite.c >> _cscope.files 2> /dev/null
-	@for i in `echo ${ALL_SUBDIRS}`; \
-	do \
-		cat ${SRCROOT}/$${i}/conf/files ${SRCROOT}/$${i}/conf/files.ppc; \
-		cat ${SRCROOT}/$${i}/conf/files.i386; \
-	done | \
-	sed -e '/^#/d' -e '/^[ 	]*$$/d' -e '/^OPTIONS\//d' | \
-	sed -e '1,$$s/^\([^ 	]*\)[ 	].*$$/\1/' >> _cscope.files 2> /dev/null
-	@sort < _cscope.files > cscope.files 2> /dev/null
-	@rm -f _cscope.files 2> /dev/null
+	@find . -name '*.h' -type f | grep -v ^..BUILD > _cscope.files 2> /dev/null
+	@find . -name '*.defs' -type f | grep -v ^..BUILD >> _cscope.files 2> /dev/null
+	@find . -name '*.c' -type f | grep -v ^..BUILD >> _cscope.files 2> /dev/null
+	@find . -name '*.cpp' -type f | grep -v ^..BUILD >> _cscope.files 2> /dev/null
+	@find . -name '*.s' -type f | grep -v ^..BUILD >> _cscope.files 2> /dev/null
+	@find . -name '*.h.template' -type f | grep -v ^..BUILD >> _cscope.files 2> /dev/null
+	@echo -k -q -c > cscope.files 2> /dev/null
+	@sort -u < _cscope.files >> cscope.files 2> /dev/null
+	@rm -f _cscope.files _cscope.files2 2> /dev/null
 
 #
 # Build cscope database
 #
 cscope: cscope.files
 	@echo "Building cscope database"
-	@cscope -b 2> /dev/null
+	@cscope -bvU 2> /dev/null
 
 #
 # Build tags
diff --git a/makedefs/MakeInc.rule b/makedefs/MakeInc.rule
index bfd896d29..d24d1fb75 100644
--- a/makedefs/MakeInc.rule
+++ b/makedefs/MakeInc.rule
@@ -604,7 +604,7 @@ do_build_mach_kernel: $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/kgmacros
 	@install $(DATA_INSTALL_FLAGS) $(SRCROOT)/config/version.c $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/version.c;
 	@$(SRCROOT)/config/newvers.pl $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/version.c;
 	${KCC} -c ${filter-out ${${join $@,_CFLAGS_RM}}, ${CFLAGS}} ${${join $@,_CFLAGS_ADD}} ${INCFLAGS} ${${join $@,_INCFLAGS}} $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/version.c -o $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/version.o
-	$(LD) $(LDFLAGS_KERNEL) $(addprefix $(TARGET)/,$(foreach component,$(COMPONENT_LIST), $(addprefix $(component)/$(firstword $($(addsuffix _KERNEL_CONFIG, $(shell echo -n $(component) | tr a-z A-Z))) $(KERNEL_CONFIG))/, $(addsuffix .o, $(component))))) $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/version.o -o $(TARGET)/mach_kernel.sys $(LD_KERNEL_LIBS);	\
+	$(LD) $(LDFLAGS_KERNEL) $(addprefix $(TARGET)/,$(foreach component,$(COMPONENT_LIST), $(addprefix $(component)/$(firstword $($(addsuffix _KERNEL_CONFIG, $(shell printf "%s" "$(component)" | tr a-z A-Z))) $(KERNEL_CONFIG))/, $(addsuffix .o, $(component))))) $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/version.o -o $(TARGET)/mach_kernel.sys $(LD_KERNEL_LIBS);	\
 	$(STRIP) $(STRIP_FLAGS) $(TARGET)/mach_kernel.sys -o $(TARGET)/mach_kernel;
 
 $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/kgmacros: $(SRCROOT)/kgmacros
diff --git a/osfmk/chud/chud_cpu.c b/osfmk/chud/chud_cpu.c
new file mode 100644
index 000000000..4b7f8bdb7
--- /dev/null
+++ b/osfmk/chud/chud_cpu.c
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/mach_types.h>
+#include <mach/mach_host.h>
+
+#include <kern/host.h>
+#include <kern/processor.h>
+#include <kern/cpu_data.h>
+
+#include <machine/machine_routines.h>
+
+#include <chud/chud_xnu.h>
+
+#pragma mark **** cpu count ****
+
+__private_extern__ int
+chudxnu_avail_cpu_count(void)
+{
+    host_basic_info_data_t hinfo;
+    kern_return_t kr;
+    mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
+
+    kr = host_info(host_self(), HOST_BASIC_INFO, (integer_t *)&hinfo, &count);
+    if(kr == KERN_SUCCESS) {
+        return hinfo.avail_cpus;
+    } else {
+        return 0;
+    }
+}
+
+__private_extern__ int
+chudxnu_phys_cpu_count(void)
+{
+    host_basic_info_data_t hinfo;
+    kern_return_t kr;
+    mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
+
+    kr = host_info(host_self(), HOST_BASIC_INFO, (integer_t *)&hinfo, &count);
+    if(kr == KERN_SUCCESS) {
+        return hinfo.max_cpus;
+    } else {
+        return 0;
+    }
+}
+
+__private_extern__
+int chudxnu_cpu_number(void)
+{
+    return cpu_number();
+}
+
+#pragma mark **** branch trace buffer ****
+
+extern int pc_trace_buf[1024];
+
+__private_extern__ uint32_t *
+chudxnu_get_branch_trace_buffer(uint32_t *entries)
+{
+    if(entries) {
+        *entries = sizeof(pc_trace_buf)/sizeof(int);
+    }
+    return pc_trace_buf;
+}
+
+#pragma mark **** interrupts enable/disable ****
+
+__private_extern__ boolean_t
+chudxnu_get_interrupts_enabled(void)
+{
+    return ml_get_interrupts_enabled();
+}
+
+__private_extern__ boolean_t
+chudxnu_set_interrupts_enabled(boolean_t enable)
+{
+    return ml_set_interrupts_enabled(enable);
+}
+
+__private_extern__ boolean_t
+chudxnu_at_interrupt_context(void)
+{
+    return ml_at_interrupt_context();
+}
+
+__private_extern__ void
+chudxnu_cause_interrupt(void)
+{
+    ml_cause_interrupt();
+}
+
+#pragma mark **** preemption enable/disable ****
+
+__private_extern__ void
+chudxnu_enable_preemption(void)
+{
+	enable_preemption();
+}
+
+__private_extern__ void
+chudxnu_disable_preemption(void)
+{
+	disable_preemption();
+}
+
+__private_extern__ int
+chudxnu_get_preemption_level(void)
+{
+	return get_preemption_level();
+}
diff --git a/osfmk/ppc/chud/chud_glue.c b/osfmk/chud/chud_glue.c
similarity index 100%
rename from osfmk/ppc/chud/chud_glue.c
rename to osfmk/chud/chud_glue.c
diff --git a/osfmk/ppc/chud/chud_memory.c b/osfmk/chud/chud_memory.c
similarity index 79%
rename from osfmk/ppc/chud/chud_memory.c
rename to osfmk/chud/chud_memory.c
index 80d7e251f..01d431ff9 100644
--- a/osfmk/ppc/chud/chud_memory.c
+++ b/osfmk/chud/chud_memory.c
@@ -27,13 +27,16 @@
  */
 
 #include <mach/vm_param.h>
-#include <ppc/chud/chud_xnu.h>
-#include <ppc/machine_routines.h>
+#include <chud/chud_xnu.h>
+#include <machine/machine_routines.h>
+
+extern unsigned int IODefaultCacheBits(addr64_t pa);
+extern unsigned int vm_page_free_count;
 
 __private_extern__
 uint64_t chudxnu_avail_memory_size(void)
 {
-    return mem_size;
+    return max_mem;
 }
 
 __private_extern__
@@ -42,6 +45,16 @@ uint64_t chudxnu_phys_memory_size(void)
     return mem_actual;
 }
 
+/* 
+ * This function is not intended to be valid for any amount of time,
+ * it is just an instantaneous snapshot of the current free memory size.
+ */
+__private_extern__
+uint64_t chudxnu_free_memory_size(void)
+{
+    return (uint64_t)vm_page_free_count * (uint64_t)page_size;
+}
+
 __private_extern__
 vm_offset_t chudxnu_io_map(uint64_t phys_addr, vm_size_t size)
 {
diff --git a/osfmk/chud/chud_osfmk_callback.c b/osfmk/chud/chud_osfmk_callback.c
new file mode 100644
index 000000000..3c4f21099
--- /dev/null
+++ b/osfmk/chud/chud_osfmk_callback.c
@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <stdint.h>
+#include <mach/boolean.h>
+#include <mach/mach_types.h>
+
+#include <kern/kern_types.h>
+#include <kern/processor.h>
+#include <kern/timer_call.h>
+#include <kern/thread_call.h>
+#include <kern/kalloc.h>
+#include <kern/thread.h>
+
+#include <machine/machine_routines.h>
+#include <machine/cpu_data.h>
+
+#include <chud/chud_xnu.h>
+#include <chud/chud_xnu_private.h>
+
+#pragma mark **** timer ****
+__private_extern__ chud_timer_t
+chudxnu_timer_alloc(chudxnu_timer_callback_func_t func, uint32_t param0)
+{
+    return (chud_timer_t)thread_call_allocate((thread_call_func_t)func, (thread_call_param_t)param0);
+}
+
+__private_extern__ kern_return_t
+chudxnu_timer_callback_enter(
+	chud_timer_t timer,
+	uint32_t param1,
+	uint32_t time,
+	uint32_t units)
+{
+    uint64_t t_delay;
+    clock_interval_to_deadline(time, units, &t_delay);
+    thread_call_enter1_delayed((thread_call_t)timer, (thread_call_param_t)param1, t_delay);
+    return KERN_SUCCESS;
+}
+
+__private_extern__ kern_return_t
+chudxnu_timer_callback_cancel(chud_timer_t timer)
+{
+    thread_call_cancel((thread_call_t)timer);
+    return KERN_SUCCESS;
+}
+
+__private_extern__ kern_return_t
+chudxnu_timer_free(chud_timer_t timer)
+{
+    thread_call_cancel((thread_call_t)timer);
+    thread_call_free((thread_call_t)timer);
+    return KERN_SUCCESS;
+}
+
+#pragma mark **** thread timer - DEPRECATED ****
+
+static thread_call_t thread_timer_call = NULL;
+static chudxnu_thread_timer_callback_func_t thread_timer_callback_fn = NULL;
+
+static void chudxnu_private_thread_timer_callback(
+		thread_call_param_t param0,
+		thread_call_param_t param1)
+{
+#pragma unused (param1)
+    chudxnu_thread_timer_callback_func_t fn = thread_timer_callback_fn;
+    
+    if(thread_timer_call) {
+        thread_call_free(thread_timer_call);
+        thread_timer_call = NULL;
+
+        if(fn) {
+            (fn)((uint32_t)param0);
+        }
+    }
+}
+
+// DEPRECATED
+__private_extern__
+kern_return_t chudxnu_thread_timer_callback_enter(
+		chudxnu_thread_timer_callback_func_t func,
+		uint32_t param,
+		uint32_t time,
+		uint32_t units)
+{
+    if(!thread_timer_call) {
+        uint64_t t_delay;
+        thread_timer_callback_fn = func;
+
+        thread_timer_call = thread_call_allocate(
+				(thread_call_func_t)
+					chudxnu_private_thread_timer_callback,
+				(thread_call_param_t)
+					param);
+        clock_interval_to_deadline(time, units, &t_delay);
+        thread_call_enter_delayed(thread_timer_call, t_delay);
+        return KERN_SUCCESS;
+    } else {
+        return KERN_FAILURE; // thread timer call already pending
+    }
+}
+
+// DEPRECATED
+__private_extern__
+kern_return_t chudxnu_thread_timer_callback_cancel(void)
+{
+    if(thread_timer_call) {
+    	thread_call_cancel(thread_timer_call);
+        thread_call_free(thread_timer_call);
+        thread_timer_call = NULL;
+    }
+    thread_timer_callback_fn = NULL;
+    return KERN_SUCCESS;
+}
diff --git a/osfmk/chud/chud_thread.c b/osfmk/chud/chud_thread.c
new file mode 100644
index 000000000..1f67149b6
--- /dev/null
+++ b/osfmk/chud/chud_thread.c
@@ -0,0 +1,448 @@
+/*
+ * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <mach/mach_types.h>
+#include <mach/task.h>
+#include <mach/thread_act.h>
+
+#include <kern/kern_types.h>
+#include <kern/processor.h>
+#include <kern/thread.h>
+#include <kern/kalloc.h>
+
+#include <chud/chud_xnu.h>
+#include <chud/chud_xnu_private.h>
+
+#include <machine/machine_routines.h>
+
+// include the correct file to find real_ncpus
+#if defined(__i386__) || defined(__x86_64__)
+#	include <i386/mp.h>	
+#endif // i386 or x86_64
+
+#if defined(__ppc__) || defined(__ppc64__)
+#	include <ppc/cpu_internal.h>
+#endif // ppc or ppc64
+
+#pragma mark **** thread binding ****
+
+__private_extern__ kern_return_t
+chudxnu_bind_thread(thread_t thread, int cpu)
+{
+    processor_t proc = NULL;
+	
+	if(cpu >= real_ncpus) // sanity check
+		return KERN_FAILURE;
+	
+	proc = cpu_to_processor(cpu);
+
+	if(proc && !(proc->state == PROCESSOR_OFF_LINE) &&
+			   !(proc->state == PROCESSOR_SHUTDOWN)) {
+		/* disallow bind to shutdown processor */
+		thread_bind(thread, proc);
+		if(thread==current_thread()) {
+			(void)thread_block(THREAD_CONTINUE_NULL);
+		}
+		return KERN_SUCCESS;
+	}
+    return KERN_FAILURE;
+}
+
+__private_extern__ kern_return_t
+chudxnu_unbind_thread(thread_t thread)
+{
+    thread_bind(thread, PROCESSOR_NULL);
+    return KERN_SUCCESS;
+}
+
+#pragma mark **** task and thread info ****
+
+__private_extern__
+boolean_t chudxnu_is_64bit_task(task_t task)
+{
+	return (task_has_64BitAddr(task));
+}
+
+#define THING_TASK		0
+#define THING_THREAD	1
+
+// an exact copy of processor_set_things() except no mig conversion at the end!
+static kern_return_t
+chudxnu_private_processor_set_things(
+	processor_set_t		pset,
+	mach_port_t		**thing_list,
+	mach_msg_type_number_t	*count,
+	int			type)
+{
+	unsigned int actual;	/* this many things */
+	unsigned int maxthings;
+	unsigned int i;
+
+	vm_size_t size, size_needed;
+	void  *addr;
+
+	if (pset == PROCESSOR_SET_NULL)
+		return (KERN_INVALID_ARGUMENT);
+
+	size = 0; addr = 0;
+
+	for (;;) {
+		pset_lock(pset);
+		if (!pset->active) {
+			pset_unlock(pset);
+
+			return (KERN_FAILURE);
+		}
+
+		if (type == THING_TASK)
+			maxthings = pset->task_count;
+		else
+			maxthings = pset->thread_count;
+
+		/* do we have the memory we need? */
+
+		size_needed = maxthings * sizeof (mach_port_t);
+		if (size_needed <= size)
+			break;
+
+		/* unlock the pset and allocate more memory */
+		pset_unlock(pset);
+
+		if (size != 0)
+			kfree(addr, size);
+
+		assert(size_needed > 0);
+		size = size_needed;
+
+		addr = kalloc(size);
+		if (addr == 0)
+			return (KERN_RESOURCE_SHORTAGE);
+	}
+
+	/* OK, have memory and the processor_set is locked & active */
+
+	actual = 0;
+	switch (type) {
+
+	case THING_TASK:
+	{
+		task_t		task, *tasks = (task_t *)addr;
+
+		for (task = (task_t)queue_first(&pset->tasks);
+				!queue_end(&pset->tasks, (queue_entry_t)task);
+					task = (task_t)queue_next(&task->pset_tasks)) {
+			task_reference_internal(task);
+			tasks[actual++] = task;
+		}
+
+		break;
+	}
+
+	case THING_THREAD:
+	{
+		thread_t	thread, *threads = (thread_t *)addr;
+
+		for (i = 0, thread = (thread_t)queue_first(&pset->threads);
+				!queue_end(&pset->threads, (queue_entry_t)thread);
+					thread = (thread_t)queue_next(&thread->pset_threads)) {
+			thread_reference_internal(thread);
+			threads[actual++] = thread;
+		}
+
+		break;
+	}
+	}
+		
+	pset_unlock(pset);
+
+	if (actual < maxthings)
+		size_needed = actual * sizeof (mach_port_t);
+
+	if (actual == 0) {
+		/* no things, so return null pointer and deallocate memory */
+		*thing_list = 0;
+		*count = 0;
+
+		if (size != 0)
+			kfree(addr, size);
+	}
+	else {
+		/* if we allocated too much, must copy */
+
+		if (size_needed < size) {
+			void *newaddr;
+
+			newaddr = kalloc(size_needed);
+			if (newaddr == 0) {
+				switch (type) {
+
+				case THING_TASK:
+				{
+					task_t		*tasks = (task_t *)addr;
+
+					for (i = 0; i < actual; i++)
+						task_deallocate(tasks[i]);
+					break;
+				}
+
+				case THING_THREAD:
+				{
+					thread_t	*threads = (thread_t *)addr;
+
+					for (i = 0; i < actual; i++)
+						thread_deallocate(threads[i]);
+					break;
+				}
+				}
+
+				kfree(addr, size);
+				return (KERN_RESOURCE_SHORTAGE);
+			}
+
+			bcopy((void *) addr, (void *) newaddr, size_needed);
+			kfree(addr, size);
+			addr = newaddr;
+		}
+
+		*thing_list = (mach_port_t *)addr;
+		*count = actual;
+	}
+
+	return (KERN_SUCCESS);
+}
+
+// an exact copy of task_threads() except no mig conversion at the end!
+static kern_return_t
+chudxnu_private_task_threads(
+	task_t			task,
+	thread_act_array_t      *threads_out,
+    	mach_msg_type_number_t  *count)
+{
+	mach_msg_type_number_t	actual;
+	thread_t				*threads;
+	thread_t				thread;
+	vm_size_t				size, size_needed;
+	void					*addr;
+	unsigned int			i, j;
+
+	if (task == TASK_NULL)
+		return (KERN_INVALID_ARGUMENT);
+
+	size = 0; addr = 0;
+
+	for (;;) {
+		task_lock(task);
+		if (!task->active) {
+			task_unlock(task);
+
+			if (size != 0)
+				kfree(addr, size);
+
+			return (KERN_FAILURE);
+		}
+
+		actual = task->thread_count;
+
+		/* do we have the memory we need? */
+		size_needed = actual * sizeof (mach_port_t);
+		if (size_needed <= size)
+			break;
+
+		/* unlock the task and allocate more memory */
+		task_unlock(task);
+
+		if (size != 0)
+			kfree(addr, size);
+
+		assert(size_needed > 0);
+		size = size_needed;
+
+		addr = kalloc(size);
+		if (addr == 0)
+			return (KERN_RESOURCE_SHORTAGE);
+	}
+
+	/* OK, have memory and the task is locked & active */
+	threads = (thread_t *)addr;
+
+	i = j = 0;
+
+	for (thread = (thread_t)queue_first(&task->threads); i < actual;
+				++i, thread = (thread_t)queue_next(&thread->task_threads)) {
+		thread_reference_internal(thread);
+		threads[j++] = thread;
+	}
+
+	assert(queue_end(&task->threads, (queue_entry_t)thread));
+
+	actual = j;
+	size_needed = actual * sizeof (mach_port_t);
+
+	/* can unlock task now that we've got the thread refs */
+	task_unlock(task);
+
+	if (actual == 0) {
+		/* no threads, so return null pointer and deallocate memory */
+
+		*threads_out = 0;
+		*count = 0;
+
+		if (size != 0)
+			kfree(addr, size);
+	}
+	else {
+		/* if we allocated too much, must copy */
+
+		if (size_needed < size) {
+			void *newaddr;
+
+			newaddr = kalloc(size_needed);
+			if (newaddr == 0) {
+				for (i = 0; i < actual; ++i)
+					thread_deallocate(threads[i]);
+				kfree(addr, size);
+				return (KERN_RESOURCE_SHORTAGE);
+			}
+
+			bcopy(addr, newaddr, size_needed);
+			kfree(addr, size);
+			threads = (thread_t *)newaddr;
+		}
+
+		*threads_out = threads;
+		*count = actual;
+	}
+
+	return (KERN_SUCCESS);
+}
+
+
+__private_extern__ kern_return_t
+chudxnu_all_tasks(
+	task_array_t		*task_list,
+	mach_msg_type_number_t	*count)
+{
+	return chudxnu_private_processor_set_things(&default_pset, (mach_port_t **)task_list, count, THING_TASK);	
+}
+
+__private_extern__ kern_return_t
+chudxnu_free_task_list(
+	task_array_t		*task_list,
+	mach_msg_type_number_t	*count)
+{
+	vm_size_t size = (*count)*sizeof(mach_port_t);
+	void *addr = *task_list;
+
+	if(addr) {
+		int i, maxCount = *count;
+		for(i=0; i<maxCount; i++) {
+			task_deallocate((*task_list)[i]);
+		}		
+		kfree(addr, size);
+		*task_list = NULL;
+		*count = 0;
+		return KERN_SUCCESS;
+	} else {
+		return KERN_FAILURE;
+	}
+}
+
+__private_extern__ kern_return_t
+chudxnu_all_threads(
+	thread_array_t		*thread_list,
+	mach_msg_type_number_t	*count)
+{
+	return chudxnu_private_processor_set_things(&default_pset, (mach_port_t **)thread_list, count, THING_THREAD);
+}
+
+__private_extern__ kern_return_t
+chudxnu_task_threads(
+	task_t task,
+	thread_array_t *thread_list,
+	mach_msg_type_number_t *count)
+{
+	return chudxnu_private_task_threads(task, thread_list, count);
+}
+
+__private_extern__ kern_return_t
+chudxnu_free_thread_list(
+	thread_array_t	*thread_list,
+	mach_msg_type_number_t	*count)
+{
+	vm_size_t size = (*count)*sizeof(mach_port_t);
+	void *addr = *thread_list;
+
+	if(addr) {
+		int i, maxCount = *count;
+		for(i=0; i<maxCount; i++) {
+			thread_deallocate((*thread_list)[i]);
+		}		
+		kfree(addr, size);
+		*thread_list = NULL;
+		*count = 0;
+		return KERN_SUCCESS;
+	} else {
+		return KERN_FAILURE;
+	}
+}
+
+__private_extern__ task_t
+chudxnu_current_task(void)
+{
+	return current_task();
+}
+
+__private_extern__ thread_t
+chudxnu_current_thread(void)
+{
+	return current_thread();
+}
+
+__private_extern__ task_t
+chudxnu_task_for_thread(thread_t thread)
+{
+    return get_threadtask(thread);
+}
+
+__private_extern__ kern_return_t
+chudxnu_thread_info(
+	thread_t thread,
+	thread_flavor_t flavor,
+	thread_info_t thread_info_out,
+	mach_msg_type_number_t *thread_info_count)
+{
+	return thread_info(thread, flavor, thread_info_out, thread_info_count);
+}
+
+__private_extern__ kern_return_t
+chudxnu_thread_last_context_switch(thread_t thread, uint64_t *timestamp)
+{
+    *timestamp = thread->last_switch;
+    return KERN_SUCCESS;
+}
+
diff --git a/osfmk/chud/chud_xnu.h b/osfmk/chud/chud_xnu.h
new file mode 100644
index 000000000..7f5c2884b
--- /dev/null
+++ b/osfmk/chud/chud_xnu.h
@@ -0,0 +1,263 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _CHUD_XNU_H_
+#define _CHUD_XNU_H_
+
+
+#include <stdint.h>
+#include <mach/boolean.h>
+#include <mach/mach_types.h>
+
+#pragma mark **** version ****
+extern uint32_t chudxnu_version(void);
+
+#pragma mark **** task ****
+// ********************************************************************************
+// task
+// ********************************************************************************
+extern int chudxnu_pid_for_task(task_t task);
+extern task_t chudxnu_task_for_pid(int pid);
+extern int chudxnu_current_pid(void);
+
+extern kern_return_t chudxnu_task_read(task_t task, void *kernaddr, uint64_t usraddr, vm_size_t size);
+extern kern_return_t chudxnu_task_write(task_t task, uint64_t useraddr, void *kernaddr, vm_size_t size);
+extern kern_return_t chudxnu_kern_read(void *destaddr, vm_offset_t srcaddr, vm_size_t size);
+extern kern_return_t chudxnu_kern_write(vm_offset_t destaddr, void *srcaddr, vm_size_t size);
+
+extern boolean_t chudxnu_is_64bit_task(task_t task);
+
+#pragma mark **** thread ****
+// ********************************************************************************
+// thread
+// ********************************************************************************
+extern kern_return_t chudxnu_bind_thread(thread_t thread, int cpu);
+extern kern_return_t chudxnu_unbind_thread(thread_t thread);
+
+extern kern_return_t chudxnu_thread_get_state(thread_t thread, thread_flavor_t flavor, thread_state_t tstate, mach_msg_type_number_t *count, boolean_t user_only);
+extern kern_return_t chudxnu_thread_set_state(thread_t thread, thread_flavor_t flavor, thread_state_t tstate, mach_msg_type_number_t count, boolean_t user_only);
+extern kern_return_t chudxnu_thread_user_state_available(thread_t thread);
+
+extern kern_return_t chudxnu_thread_get_callstack(thread_t thread, uint32_t *callStack, mach_msg_type_number_t *count, boolean_t user_only);
+extern kern_return_t chudxnu_thread_get_callstack64(thread_t thread, uint64_t *callStack, mach_msg_type_number_t *count, boolean_t user_only);
+
+extern task_t chudxnu_current_task(void);
+extern thread_t chudxnu_current_thread(void);
+				    
+extern task_t chudxnu_task_for_thread(thread_t thread);
+
+extern kern_return_t chudxnu_all_tasks(task_array_t *task_list, mach_msg_type_number_t *count);
+extern kern_return_t chudxnu_free_task_list(task_array_t *task_list, mach_msg_type_number_t	*count);
+									 
+extern kern_return_t chudxnu_all_threads(thread_array_t *thread_list, mach_msg_type_number_t *count);
+extern kern_return_t chudxnu_task_threads(task_t task, thread_array_t *thread_list, mach_msg_type_number_t *count);
+extern kern_return_t chudxnu_free_thread_list(thread_array_t *thread_list, mach_msg_type_number_t *count);
+
+extern kern_return_t chudxnu_thread_info(  thread_t thread, thread_flavor_t flavor, thread_info_t thread_info_out, mach_msg_type_number_t *thread_info_count);
+
+extern kern_return_t chudxnu_thread_last_context_switch(thread_t thread, uint64_t *timestamp);
+
+#pragma mark **** memory ****
+// ********************************************************************************
+// memory
+// ********************************************************************************
+
+extern uint64_t chudxnu_avail_memory_size(void);
+extern uint64_t chudxnu_phys_memory_size(void);
+extern uint64_t chudxnu_free_memory_size(void);
+
+extern vm_offset_t chudxnu_io_map(uint64_t phys_addr, vm_size_t size);
+
+extern uint32_t chudxnu_phys_addr_wimg(uint64_t phys_addr);
+
+#pragma mark **** cpu ****
+// ********************************************************************************
+// cpu
+// ********************************************************************************
+extern int chudxnu_avail_cpu_count(void);
+extern int chudxnu_phys_cpu_count(void);
+extern int chudxnu_cpu_number(void);
+
+extern kern_return_t chudxnu_enable_cpu(int cpu, boolean_t enable);
+
+extern kern_return_t chudxnu_enable_cpu_nap(int cpu, boolean_t enable);
+extern boolean_t chudxnu_cpu_nap_enabled(int cpu);
+
+extern boolean_t chudxnu_get_interrupts_enabled(void);
+extern boolean_t chudxnu_set_interrupts_enabled(boolean_t enable);
+extern boolean_t chudxnu_at_interrupt_context(void);
+extern void chudxnu_cause_interrupt(void);
+
+extern void chudxnu_enable_preemption(void);
+extern void chudxnu_disable_preemption(void);
+extern int chudxnu_get_preemption_level(void);
+
+extern kern_return_t chudxnu_set_shadowed_spr(int cpu, int spr, uint32_t val);
+extern kern_return_t chudxnu_set_shadowed_spr64(int cpu, int spr, uint64_t val);
+
+extern uint32_t chudxnu_get_orig_cpu_l2cr(int cpu);
+extern uint32_t chudxnu_get_orig_cpu_l3cr(int cpu);
+
+extern kern_return_t chudxnu_read_spr(int cpu, int spr, uint32_t *val_p);
+extern kern_return_t chudxnu_read_spr64(int cpu, int spr, uint64_t *val_p);
+extern kern_return_t chudxnu_write_spr(int cpu, int spr, uint32_t val);
+extern kern_return_t chudxnu_write_spr64(int cpu, int spr, uint64_t val);
+
+extern void chudxnu_flush_caches(void);
+extern void chudxnu_enable_caches(boolean_t enable);
+
+extern kern_return_t chudxnu_perfmon_acquire_facility(task_t);
+extern kern_return_t chudxnu_perfmon_release_facility(task_t);
+
+extern uint32_t * chudxnu_get_branch_trace_buffer(uint32_t *entries);
+
+typedef struct {
+    uint32_t hwResets;
+    uint32_t hwMachineChecks;
+    uint32_t hwDSIs;
+    uint32_t hwISIs;
+    uint32_t hwExternals;
+    uint32_t hwAlignments;
+    uint32_t hwPrograms;
+    uint32_t hwFloatPointUnavailable;
+    uint32_t hwDecrementers;
+    uint32_t hwIOErrors;
+    uint32_t hwSystemCalls;
+    uint32_t hwTraces;
+    uint32_t hwFloatingPointAssists;
+    uint32_t hwPerformanceMonitors;
+    uint32_t hwAltivecs;
+    uint32_t hwInstBreakpoints;
+    uint32_t hwSystemManagements;
+    uint32_t hwAltivecAssists;
+    uint32_t hwThermal;
+    uint32_t hwSoftPatches;
+    uint32_t hwMaintenances;
+    uint32_t hwInstrumentations;
+} rupt_counters_t;
+
+extern kern_return_t chudxnu_get_cpu_rupt_counters(int cpu, rupt_counters_t *rupts);
+extern kern_return_t chudxnu_clear_cpu_rupt_counters(int cpu);
+
+extern kern_return_t chudxnu_passup_alignment_exceptions(boolean_t enable);
+
+extern kern_return_t chudxnu_scom_read(uint32_t reg, uint64_t *data);
+extern kern_return_t chudxnu_scom_write(uint32_t reg, uint64_t data);
+
+#pragma mark **** callbacks ****
+// ********************************************************************************
+// callbacks
+// ********************************************************************************
+
+extern void chudxnu_cancel_all_callbacks(void);
+
+// cpu timer - each cpu has its own callback 
+typedef kern_return_t (*chudxnu_cpu_timer_callback_func_t)(thread_flavor_t flavor, thread_state_t tstate,  mach_msg_type_number_t count);
+extern kern_return_t chudxnu_cpu_timer_callback_enter(chudxnu_cpu_timer_callback_func_t func, uint32_t time, uint32_t units); // callback is entered on current cpu
+extern kern_return_t chudxnu_cpu_timer_callback_cancel(void); // callback is cleared on current cpu
+extern kern_return_t chudxnu_cpu_timer_callback_cancel_all(void); // callback is cleared on all cpus
+
+enum {
+    PPC_TRAP_PROGRAM		= 0x700,
+    PPC_TRAP_TRACE		= 0xD00,
+    PPC_TRAP_PERFMON		= 0xF00,
+};
+
+enum {
+	X86_TRAP_DEBUG			= 0x1,
+};
+
+// trap callback - one callback for system
+typedef kern_return_t (*chudxnu_trap_callback_func_t)(uint32_t trapentry, thread_flavor_t flavor, thread_state_t tstate,  mach_msg_type_number_t count);
+extern kern_return_t chudxnu_trap_callback_enter(chudxnu_trap_callback_func_t func);
+extern kern_return_t chudxnu_trap_callback_cancel(void);
+
+enum {
+    PPC_INTERRUPT_DECREMENTER	= 0x900,
+    PPC_INTERRUPT_INTERRUPT	= 0x500,
+    PPC_INTERRUPT_CPU_SIGNAL	= 0x2200,
+};
+
+enum {
+    X86_INTERRUPT_PERFMON	= 0xB,
+};
+
+// interrupt callback - one callback for system
+typedef kern_return_t (*chudxnu_interrupt_callback_func_t)(uint32_t trapentry, thread_flavor_t flavor, thread_state_t tstate,  mach_msg_type_number_t count);
+extern kern_return_t chudxnu_interrupt_callback_enter(chudxnu_interrupt_callback_func_t func);
+extern kern_return_t chudxnu_interrupt_callback_cancel(void);
+
+// ast callback - one callback for system
+typedef kern_return_t (*chudxnu_perfmon_ast_callback_func_t)(thread_flavor_t flavor, thread_state_t tstate,  mach_msg_type_number_t count);
+extern kern_return_t chudxnu_perfmon_ast_callback_enter(chudxnu_perfmon_ast_callback_func_t func);
+extern kern_return_t chudxnu_perfmon_ast_callback_cancel(void);
+extern kern_return_t chudxnu_perfmon_ast_send(void);
+extern kern_return_t chudxnu_perfmon_ast_send_urgent(boolean_t urgent);
+
+// cpusig callback - one callback for system
+typedef kern_return_t (*chudxnu_cpusig_callback_func_t)(int request, thread_flavor_t flavor, thread_state_t tstate, mach_msg_type_number_t count);
+extern kern_return_t chudxnu_cpusig_callback_enter(chudxnu_cpusig_callback_func_t func);
+extern kern_return_t chudxnu_cpusig_callback_cancel(void);
+extern kern_return_t chudxnu_cpusig_send(int otherCPU, uint32_t request);
+
+// kdebug callback - one callback for system
+typedef kern_return_t (*chudxnu_kdebug_callback_func_t)(uint32_t debugid, uint32_t arg0, uint32_t arg1, uint32_t arg2, uint32_t arg3, uint32_t arg4);
+extern kern_return_t chudxnu_kdebug_callback_enter(chudxnu_kdebug_callback_func_t func);
+extern kern_return_t chudxnu_kdebug_callback_cancel(void);
+
+// timer callback - multiple callbacks
+typedef kern_return_t (*chudxnu_timer_callback_func_t)(uint32_t param0, uint32_t param1);
+typedef void *	chud_timer_t;
+extern chud_timer_t chudxnu_timer_alloc(chudxnu_timer_callback_func_t func, uint32_t param0);
+extern kern_return_t chudxnu_timer_callback_enter(chud_timer_t timer, uint32_t param1, uint32_t time, uint32_t units);
+extern kern_return_t chudxnu_timer_callback_cancel(chud_timer_t timer);
+extern kern_return_t chudxnu_timer_free(chud_timer_t timer);
+
+// CHUD systemcall callback - one callback for system
+//typedef kern_return_t (*chudxnu_syscall_callback_func_t)(thread_flavor_t flavor, thread_state_t tstate, mach_msg_type_number_t count); // v2
+typedef kern_return_t (*chudxnu_syscall_callback_func_t)(uint32_t code, uint32_t arg0, uint32_t arg1, uint32_t arg2, uint32_t arg3, uint32_t arg4);
+extern kern_return_t chudxnu_syscall_callback_enter(chudxnu_syscall_callback_func_t func);
+extern kern_return_t chudxnu_syscall_callback_cancel(void);
+
+// ********************************************************************************
+// DEPRECATED
+// ********************************************************************************
+extern kern_return_t chudxnu_bind_current_thread(int cpu);
+
+extern kern_return_t chudxnu_unbind_current_thread(void);
+
+extern kern_return_t chudxnu_current_thread_get_callstack(uint32_t *callStack, mach_msg_type_number_t *count, boolean_t user_only);
+                        
+extern thread_t chudxnu_current_act(void);
+
+// thread timer callback - one callback for system
+typedef kern_return_t (*chudxnu_thread_timer_callback_func_t)(uint32_t param);
+extern kern_return_t chudxnu_thread_timer_callback_enter(chudxnu_thread_timer_callback_func_t func, uint32_t param, uint32_t time, uint32_t units);
+extern kern_return_t chudxnu_thread_timer_callback_cancel(void);
+
+#endif /* _CHUD_XNU_H_ */
diff --git a/osfmk/i386/rtclock_entries.h b/osfmk/chud/chud_xnu_glue.h
similarity index 66%
rename from osfmk/i386/rtclock_entries.h
rename to osfmk/chud/chud_xnu_glue.h
index 12c0bdc9c..bc7d67d8b 100644
--- a/osfmk/i386/rtclock_entries.h
+++ b/osfmk/chud/chud_xnu_glue.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -25,24 +25,11 @@
  * 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/*
- * @OSF_COPYRIGHT@
- */
 
-extern int		rtc_config(void);
-extern int		rtc_init(void);
-extern kern_return_t	rtc_gettime(
-				mach_timespec_t		* curtime);
-extern void		rtc_gettime_interrupts_disabled(
-				mach_timespec_t		* curtime);
-extern kern_return_t	rtc_settime(
-				mach_timespec_t		* curtime);
-extern kern_return_t	rtc_getattr(
-				clock_flavor_t		flavor,
-				clock_attr_t		ttr,
-				mach_msg_type_number_t	* count);
-extern void		rtc_setalrm(
-				mach_timespec_t		* alarmtime);
-extern void		rtclock_intr(
-				struct i386_interrupt_state	*regs);
-extern void		rtc_sleep_wakeup(void);
+#if defined (__ppc__)
+#include "ppc/chud_xnu_glue.h"
+#elif defined (__i386__)
+#include "i386/chud_xnu_glue.h"
+#else
+#error architecture not supported
+#endif
diff --git a/osfmk/chud/chud_xnu_private.h b/osfmk/chud/chud_xnu_private.h
new file mode 100644
index 000000000..8c6a24574
--- /dev/null
+++ b/osfmk/chud/chud_xnu_private.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _CHUD_XNU_PRIVATE_H_
+#define _CHUD_XNU_PRIVATE_H_
+
+#include <stdint.h>
+#include <mach/boolean.h>
+#include <mach/mach_types.h>
+
+#if defined (__ppc__)
+#include "chud/ppc/chud_xnu_private.h"
+#elif defined (__i386__)
+#include "chud/i386/chud_xnu_private.h"
+#else
+#error architecture not supported
+#endif
+
+#endif /* _CHUD_XNU_PRIVATE_H_ */
diff --git a/osfmk/i386/AT386/bbclock_entries.h b/osfmk/chud/i386/chud_cpu_asm.h
similarity index 80%
rename from osfmk/i386/AT386/bbclock_entries.h
rename to osfmk/chud/i386/chud_cpu_asm.h
index b4200929d..153f059e4 100644
--- a/osfmk/i386/AT386/bbclock_entries.h
+++ b/osfmk/chud/i386/chud_cpu_asm.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -25,14 +25,8 @@
  * 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/*
- * @OSF_COPYRIGHT@
- */
 
-extern kern_return_t	bbc_gettime(
-				mach_timespec_t		* curtime);
-extern kern_return_t	bbc_settime(
-				mach_timespec_t		* curtime);
-extern int		bbc_config(void);
+#ifndef _CHUD_CPU_ASM_H_
+#define _CHUD_CPU_ASM_H_
 
-#define	NO_SETALRM	(void (*) (mach_timespec_t * alarm_time))0
+#endif // _CHUD_CPU_ASM_H_
diff --git a/osfmk/chud/i386/chud_cpu_asm.s b/osfmk/chud/i386/chud_cpu_asm.s
new file mode 100644
index 000000000..aed4310e5
--- /dev/null
+++ b/osfmk/chud/i386/chud_cpu_asm.s
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+ 
+#define ASSEMBLER
+#include <i386/asm.h>
+#include <mach/kern_return.h>
+
diff --git a/osfmk/chud/i386/chud_cpu_i386.c b/osfmk/chud/i386/chud_cpu_i386.c
new file mode 100644
index 000000000..25ac19b74
--- /dev/null
+++ b/osfmk/chud/i386/chud_cpu_i386.c
@@ -0,0 +1,166 @@
+/*
+ * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/mach_types.h>
+#include <mach/mach_host.h>
+
+#include <kern/host.h>
+#include <kern/processor.h>
+
+#include <i386/cpu_data.h>
+#include <i386/machine_routines.h>
+#include <i386/perfmon.h>
+#include <i386/mp.h>
+
+#include <chud/chud_xnu.h>
+
+#pragma mark **** cpu enable/disable ****
+
+extern kern_return_t processor_start(processor_t     processor); // osfmk/kern/processor.c
+extern kern_return_t processor_exit(processor_t     processor); // osfmk/kern/processor.c
+
+__private_extern__
+kern_return_t chudxnu_enable_cpu(int cpu, boolean_t enable)
+{
+    chudxnu_unbind_thread(current_thread());
+	
+    if(cpu < 0 || (unsigned int)cpu >= real_ncpus) // sanity check
+        return KERN_FAILURE;
+	
+	if((cpu_data_ptr[cpu] != NULL) && cpu != master_cpu) {
+		processor_t processor = cpu_to_processor(cpu);
+		
+		if(processor == master_processor) // don't mess with the boot processor
+			return KERN_FAILURE;
+
+        if(enable) {
+			// make sure it isn't already running
+			if(processor->state == PROCESSOR_OFF_LINE || 
+				processor->state == PROCESSOR_SHUTDOWN) {
+				return processor_start(processor);
+			}
+			return KERN_SUCCESS;	// it's already running
+        } else {
+			// make sure it hasn't already exited
+			if(processor->state != PROCESSOR_OFF_LINE &&
+				processor->state != PROCESSOR_SHUTDOWN) {
+				return processor_exit(processor);
+			}
+			return KERN_SUCCESS;
+        }
+    }
+    return KERN_FAILURE;
+}
+
+#pragma mark **** cache flush ****
+
+__private_extern__
+void
+chudxnu_flush_caches(void)
+{
+/* XXX */
+}
+
+__private_extern__
+void
+chudxnu_enable_caches(boolean_t enable)
+{
+#pragma unused (enable)
+/* XXX */
+}
+
+#pragma mark **** perfmon facility ****
+
+__private_extern__ kern_return_t
+chudxnu_perfmon_acquire_facility(task_t task)
+{
+    return pmc_acquire(task);
+}
+
+__private_extern__ kern_return_t
+chudxnu_perfmon_release_facility(task_t task)
+{
+    return pmc_release(task);
+}
+
+#pragma mark **** rupt counters ****
+
+__private_extern__ kern_return_t
+chudxnu_get_cpu_rupt_counters(int cpu, rupt_counters_t *rupts)
+{
+    if(cpu < 0 || (unsigned int)cpu >= real_ncpus) { // sanity check
+        return KERN_FAILURE;
+    }
+
+    if(rupts) {
+        boolean_t oldlevel = ml_set_interrupts_enabled(FALSE);
+        cpu_data_t	*per_proc;
+
+        per_proc = cpu_data_ptr[cpu];
+        rupts->hwResets = 0;
+        rupts->hwMachineChecks = 0;
+        rupts->hwDSIs = 0;
+        rupts->hwISIs = 0;
+        rupts->hwExternals = 0;
+        rupts->hwAlignments = 0;
+        rupts->hwPrograms = 0;
+        rupts->hwFloatPointUnavailable = 0;
+        rupts->hwDecrementers = 0;
+        rupts->hwIOErrors = 0;
+        rupts->hwSystemCalls = 0;
+        rupts->hwTraces = 0;
+        rupts->hwFloatingPointAssists = 0;
+        rupts->hwPerformanceMonitors = 0;
+        rupts->hwAltivecs = 0;
+        rupts->hwInstBreakpoints = 0;
+        rupts->hwSystemManagements = 0;
+        rupts->hwAltivecAssists = 0;
+        rupts->hwThermal = 0;
+        rupts->hwSoftPatches = 0;
+        rupts->hwMaintenances = 0;
+        rupts->hwInstrumentations = 0;
+
+        ml_set_interrupts_enabled(oldlevel);
+        return KERN_SUCCESS;
+    } else {
+        return KERN_FAILURE;
+    }
+}
+
+__private_extern__ kern_return_t
+chudxnu_clear_cpu_rupt_counters(int cpu)
+{
+    if(cpu < 0 || (unsigned int)cpu >= real_ncpus) { // sanity check
+        return KERN_FAILURE;
+    }
+
+/*
+ * XXX
+ *    bzero((char *)&(cpu_data_ptr[cpu]->hwCtrs), sizeof(struct hwCtrs));
+ */
+    return KERN_SUCCESS;
+}
diff --git a/osfmk/chud/i386/chud_osfmk_callback_i386.c b/osfmk/chud/i386/chud_osfmk_callback_i386.c
new file mode 100644
index 000000000..e2695db69
--- /dev/null
+++ b/osfmk/chud/i386/chud_osfmk_callback_i386.c
@@ -0,0 +1,596 @@
+/*
+ * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <stdint.h>
+#include <mach/boolean.h>
+#include <mach/mach_types.h>
+
+#include <kern/kern_types.h>
+#include <kern/processor.h>
+#include <kern/timer_call.h>
+#include <kern/thread_call.h>
+#include <kern/kalloc.h>
+#include <kern/thread.h>
+
+#include <machine/machine_routines.h>
+#include <machine/cpu_data.h>
+#include <machine/trap.h>
+
+#include <chud/chud_xnu.h>
+#include <chud/chud_xnu_private.h>
+
+#include <i386/misc_protos.h>
+#include <i386/mp.h>
+#include <i386/machine_cpu.h>
+
+#include <sys/kdebug.h>
+#define CHUD_TIMER_CALLBACK_CANCEL	0
+#define CHUD_TIMER_CALLBACK_ENTER	1
+#define CHUD_TIMER_CALLBACK		2
+#define CHUD_AST_SEND			3
+#define CHUD_AST_CALLBACK		4
+#define CHUD_CPUSIG_SEND		5
+#define CHUD_CPUSIG_CALLBACK		6
+
+__private_extern__
+void chudxnu_cancel_all_callbacks(void)
+{
+    chudxnu_cpusig_callback_cancel();
+    chudxnu_cpu_timer_callback_cancel_all();
+    chudxnu_interrupt_callback_cancel();
+    chudxnu_perfmon_ast_callback_cancel();
+    chudxnu_kdebug_callback_cancel();
+    chudxnu_thread_timer_callback_cancel();
+    chudxnu_trap_callback_cancel();
+#if XXX
+	chudxnu_syscall_callback_cancel();
+#endif
+}
+
+static chudcpu_data_t chudcpu_boot_cpu;
+void *
+chudxnu_cpu_alloc(boolean_t boot_processor)
+{
+	chudcpu_data_t	*chud_proc_info;
+
+
+	if (boot_processor) {
+		chud_proc_info = &chudcpu_boot_cpu;
+	} else {
+		chud_proc_info = (chudcpu_data_t *)
+					kalloc(sizeof(chudcpu_data_t));
+		if (chud_proc_info == (chudcpu_data_t *)NULL) {
+			return (void *)NULL;
+		}
+	}
+	bzero((char *)chud_proc_info, sizeof(chudcpu_data_t));
+	chud_proc_info->t_deadline = 0xFFFFFFFFFFFFFFFFULL;
+	mpqueue_init(&chud_proc_info->cpu_request_queue);
+
+
+	return (void *)chud_proc_info;
+}
+
+void
+chudxnu_cpu_free(void *cp)
+{
+	if (cp == NULL || cp == (void *)&chudcpu_boot_cpu) {
+		return;
+	} else {
+		kfree(cp,sizeof(chudcpu_data_t));
+	}
+}
+
+static void
+chudxnu_private_cpu_timer_callback(
+	timer_call_param_t param0,
+	timer_call_param_t param1)
+{
+#pragma unused (param0)
+#pragma unused (param1)
+	chudcpu_data_t			*chud_proc_info;
+	boolean_t			oldlevel;
+	x86_thread_state_t 		state;
+	mach_msg_type_number_t		count;
+	chudxnu_cpu_timer_callback_func_t fn;
+
+	oldlevel = ml_set_interrupts_enabled(FALSE);
+	chud_proc_info = (chudcpu_data_t *)(current_cpu_datap()->cpu_chud);
+
+	count = x86_THREAD_STATE_COUNT;
+	if (chudxnu_thread_get_state(current_thread(),
+				     x86_THREAD_STATE,
+				     (thread_state_t)&state,
+				     &count,
+				     FALSE) == KERN_SUCCESS) {
+			fn = chud_proc_info->cpu_timer_callback_fn;
+       		if (fn) {
+			KERNEL_DEBUG_CONSTANT(
+				MACHDBG_CODE(DBG_MACH_CHUD,
+					CHUD_TIMER_CALLBACK) | DBG_FUNC_NONE,
+				(uint32_t)fn, 0,0,0,0);
+				//state.eip, state.cs, 0, 0);
+       			(fn)(
+				x86_THREAD_STATE,
+				(thread_state_t)&state,
+				count);
+       		} 
+	} 
+	
+	ml_set_interrupts_enabled(oldlevel);
+}
+
+__private_extern__ kern_return_t
+chudxnu_cpu_timer_callback_enter(
+	chudxnu_cpu_timer_callback_func_t	func,
+	uint32_t				time,
+	uint32_t				units)
+{
+	chudcpu_data_t	*chud_proc_info;
+	boolean_t	oldlevel;
+
+	oldlevel = ml_set_interrupts_enabled(FALSE);
+	chud_proc_info = (chudcpu_data_t *)(current_cpu_datap()->cpu_chud);
+
+	// cancel any existing callback for this cpu
+	timer_call_cancel(&(chud_proc_info->cpu_timer_call));
+
+	chud_proc_info->cpu_timer_callback_fn = func;
+
+	clock_interval_to_deadline(time, units, &(chud_proc_info->t_deadline));
+	timer_call_setup(&(chud_proc_info->cpu_timer_call),
+			 chudxnu_private_cpu_timer_callback, NULL);
+	timer_call_enter(&(chud_proc_info->cpu_timer_call),
+			 chud_proc_info->t_deadline);
+
+	KERNEL_DEBUG_CONSTANT(
+		MACHDBG_CODE(DBG_MACH_CHUD,
+			     CHUD_TIMER_CALLBACK_ENTER) | DBG_FUNC_NONE,
+		(uint32_t) func, time, units, 0, 0);
+
+	ml_set_interrupts_enabled(oldlevel);
+	return KERN_SUCCESS;
+}
+
+__private_extern__ kern_return_t
+chudxnu_cpu_timer_callback_cancel(void)
+{
+	chudcpu_data_t	*chud_proc_info;
+	boolean_t	oldlevel;
+
+	oldlevel = ml_set_interrupts_enabled(FALSE);
+	chud_proc_info = (chudcpu_data_t *)(current_cpu_datap()->cpu_chud);
+
+	timer_call_cancel(&(chud_proc_info->cpu_timer_call));
+
+	KERNEL_DEBUG_CONSTANT(
+		MACHDBG_CODE(DBG_MACH_CHUD,
+			     CHUD_TIMER_CALLBACK_CANCEL) | DBG_FUNC_NONE,
+		0, 0, 0, 0, 0);
+
+	// set to max value:
+	chud_proc_info->t_deadline |= ~(chud_proc_info->t_deadline);
+	chud_proc_info->cpu_timer_callback_fn = NULL;
+
+	ml_set_interrupts_enabled(oldlevel);
+ 	return KERN_SUCCESS;
+}
+
+__private_extern__ kern_return_t
+chudxnu_cpu_timer_callback_cancel_all(void)
+{
+	unsigned int	cpu;
+	chudcpu_data_t	*chud_proc_info;
+
+	for(cpu=0; cpu < real_ncpus; cpu++) {
+		chud_proc_info = (chudcpu_data_t *) cpu_data_ptr[cpu]->cpu_chud;
+		if (chud_proc_info == NULL)
+			continue;
+		timer_call_cancel(&(chud_proc_info->cpu_timer_call));
+		chud_proc_info->t_deadline |= ~(chud_proc_info->t_deadline);
+		chud_proc_info->cpu_timer_callback_fn = NULL;
+	}
+	return KERN_SUCCESS;
+}
+
+#pragma mark **** trap ****
+static chudxnu_trap_callback_func_t trap_callback_fn = NULL;
+
+static kern_return_t
+chudxnu_private_trap_callback(
+	int			trapno,
+	void			*regs,
+	int			unused1,
+	int			unused2)
+{
+#pragma unused (regs)
+#pragma unused (unused1)
+#pragma unused (unused2)
+    kern_return_t retval = KERN_FAILURE;
+	chudxnu_trap_callback_func_t fn = trap_callback_fn;
+
+        if(fn) {
+		boolean_t			oldlevel;
+		x86_thread_state_t	state;	// once we have an 64bit- independent way to determine if a thread is
+									// running kernel code, we'll switch to x86_thread_state_t.
+		mach_msg_type_number_t		count;	
+
+		oldlevel = ml_set_interrupts_enabled(FALSE);
+
+		count = x86_THREAD_STATE_COUNT;
+		if(chudxnu_thread_get_state(current_thread(),
+				x86_THREAD_STATE,
+				(thread_state_t)&state,
+				&count,
+				FALSE) == KERN_SUCCESS) {
+	  
+		  retval = (fn)(
+				  trapno,
+				  x86_THREAD_STATE,
+				  (thread_state_t)&state,
+				  count);
+		}
+    ml_set_interrupts_enabled(oldlevel);
+	}
+
+    return retval;
+}
+
+__private_extern__ kern_return_t
+chudxnu_trap_callback_enter(chudxnu_trap_callback_func_t func)
+{
+    trap_callback_fn = func;
+    perfTrapHook = chudxnu_private_trap_callback;
+    return KERN_SUCCESS;
+}
+
+__private_extern__ kern_return_t
+chudxnu_trap_callback_cancel(void)
+{
+    trap_callback_fn = NULL;
+        perfTrapHook = NULL;
+    return KERN_SUCCESS;
+}
+
+#pragma mark **** ast ****
+static
+chudxnu_perfmon_ast_callback_func_t perfmon_ast_callback_fn = NULL;
+
+static kern_return_t
+chudxnu_private_chud_ast_callback(
+	int			trapno,
+	void			*regs,
+	int			unused1,
+	int			unused2)
+{
+#pragma unused (trapno)
+#pragma unused (regs)
+#pragma unused (unused1)
+#pragma unused (unused2)
+	boolean_t	oldlevel = ml_set_interrupts_enabled(FALSE);
+	ast_t		*myast = ast_pending();
+	kern_return_t	retval = KERN_FAILURE;
+	chudxnu_perfmon_ast_callback_func_t fn = perfmon_ast_callback_fn;
+    
+	if (*myast & AST_CHUD_URGENT) {
+		*myast &= ~(AST_CHUD_URGENT | AST_CHUD);
+		if ((*myast & AST_PREEMPTION) != AST_PREEMPTION)
+			*myast &= ~(AST_URGENT);
+		retval = KERN_SUCCESS;
+	} else if (*myast & AST_CHUD) {
+		*myast &= ~(AST_CHUD);
+		retval = KERN_SUCCESS;
+	}
+
+	if (fn) {
+		x86_thread_state_t state;
+		mach_msg_type_number_t count;
+		count = x86_THREAD_STATE_COUNT;
+		
+		if (chudxnu_thread_get_state(
+			current_thread(),
+			x86_THREAD_STATE,
+			(thread_state_t) &state, &count,
+			TRUE) == KERN_SUCCESS) {
+
+			KERNEL_DEBUG_CONSTANT(
+				MACHDBG_CODE(DBG_MACH_CHUD,
+				    CHUD_AST_CALLBACK) | DBG_FUNC_NONE,
+				(uint32_t) fn, 0, 0, 0, 0);
+
+			(fn)(
+				x86_THREAD_STATE,
+				(thread_state_t) &state,
+				count);
+		}
+	}
+    
+	ml_set_interrupts_enabled(oldlevel);
+	return retval;
+}
+
+__private_extern__ kern_return_t
+chudxnu_perfmon_ast_callback_enter(chudxnu_perfmon_ast_callback_func_t func)
+{
+	perfmon_ast_callback_fn = func;
+	perfASTHook = chudxnu_private_chud_ast_callback;
+	return KERN_SUCCESS;
+}
+
+__private_extern__ kern_return_t
+chudxnu_perfmon_ast_callback_cancel(void)
+{
+    perfmon_ast_callback_fn = NULL;
+    perfASTHook = NULL;
+    return KERN_SUCCESS;
+}
+
+__private_extern__ kern_return_t
+chudxnu_perfmon_ast_send_urgent(boolean_t urgent)
+{
+    boolean_t oldlevel = ml_set_interrupts_enabled(FALSE);
+	ast_t *myast = ast_pending();
+
+    if(urgent) {
+        *myast |= (AST_CHUD_URGENT | AST_URGENT);
+    } else {
+        *myast |= (AST_CHUD);
+    }
+
+    KERNEL_DEBUG_CONSTANT(
+	MACHDBG_CODE(DBG_MACH_CHUD, CHUD_AST_SEND) | DBG_FUNC_NONE,
+	urgent, 0, 0, 0, 0);
+
+    ml_set_interrupts_enabled(oldlevel);
+    return KERN_SUCCESS;
+}
+
+__private_extern__ kern_return_t
+chudxnu_perfmon_ast_send(void)
+{
+    return chudxnu_perfmon_ast_send_urgent(TRUE);
+}
+
+#pragma mark **** interrupt ****
+static chudxnu_interrupt_callback_func_t interrupt_callback_fn = NULL;
+
+static void
+chudxnu_private_interrupt_callback(void *foo)
+{
+#pragma unused (foo)
+	chudxnu_interrupt_callback_func_t fn = interrupt_callback_fn;
+
+	if(fn) {
+		boolean_t			oldlevel;
+		x86_thread_state_t		state;
+		mach_msg_type_number_t		count;
+
+		oldlevel = ml_set_interrupts_enabled(FALSE);
+
+		count = x86_THREAD_STATE_COUNT;
+		if(chudxnu_thread_get_state(current_thread(),
+					    x86_THREAD_STATE,
+					    (thread_state_t)&state,
+					    &count,
+					    FALSE) == KERN_SUCCESS) {
+			(fn)(
+				X86_INTERRUPT_PERFMON,
+				x86_THREAD_STATE,
+				(thread_state_t)&state,
+				count);
+		}
+		ml_set_interrupts_enabled(oldlevel);
+	}
+}
+
+__private_extern__ kern_return_t
+chudxnu_interrupt_callback_enter(chudxnu_interrupt_callback_func_t func)
+{
+    interrupt_callback_fn = func;
+    lapic_set_pmi_func((i386_intr_func_t)chudxnu_private_interrupt_callback);
+    return KERN_SUCCESS;
+}
+
+__private_extern__ kern_return_t
+chudxnu_interrupt_callback_cancel(void)
+{
+    interrupt_callback_fn = NULL;
+    lapic_set_pmi_func(NULL);
+    return KERN_SUCCESS;
+}
+
+#pragma mark **** cpu signal ****
+static chudxnu_cpusig_callback_func_t cpusig_callback_fn = NULL;
+
+static          kern_return_t
+chudxnu_private_cpu_signal_handler(int request)
+{
+	chudxnu_cpusig_callback_func_t fn = cpusig_callback_fn;
+	
+	if (fn) {
+	x86_thread_state_t  state;
+		mach_msg_type_number_t count = x86_THREAD_STATE_COUNT;
+
+		if (chudxnu_thread_get_state(current_thread(),
+					     x86_THREAD_STATE,
+					     (thread_state_t) &state, &count,
+					     FALSE) == KERN_SUCCESS) {
+			KERNEL_DEBUG_CONSTANT(
+				MACHDBG_CODE(DBG_MACH_CHUD,
+					CHUD_CPUSIG_CALLBACK) | DBG_FUNC_NONE,
+				(uint32_t)fn, request, 0, 0, 0);
+			return (fn)(
+					request, x86_THREAD_STATE,
+					(thread_state_t) &state, count);
+		} else {
+			return KERN_FAILURE;
+		}
+	}
+	return KERN_SUCCESS; //ignored
+}
+/*
+ * chudxnu_cpu_signal_handler() is called from the IPI handler
+ * when a CHUD signal arrives from another processor.
+ */
+__private_extern__ void
+chudxnu_cpu_signal_handler(void)
+{
+	chudcpu_signal_request_t	*reqp;
+	chudcpu_data_t			*chudinfop;
+
+	chudinfop = (chudcpu_data_t *) current_cpu_datap()->cpu_chud;
+
+	mpdequeue_head(&(chudinfop->cpu_request_queue),
+		       (queue_entry_t *) &reqp);
+	while (reqp != NULL) {
+		chudxnu_private_cpu_signal_handler(reqp->req_code);
+		reqp->req_sync = 0;
+		mpdequeue_head(&(chudinfop->cpu_request_queue),
+			       (queue_entry_t *) &reqp);
+	}
+}
+
+__private_extern__ kern_return_t
+chudxnu_cpusig_callback_enter(chudxnu_cpusig_callback_func_t func)
+{
+	cpusig_callback_fn = func;
+	return KERN_SUCCESS;
+}
+
+__private_extern__ kern_return_t
+chudxnu_cpusig_callback_cancel(void)
+{
+	cpusig_callback_fn = NULL;
+	return KERN_SUCCESS;
+}
+
+__private_extern__ kern_return_t
+chudxnu_cpusig_send(int otherCPU, uint32_t request_code)
+{
+	int				thisCPU;
+	kern_return_t			retval = KERN_FAILURE;
+	chudcpu_signal_request_t	request;
+	uint64_t			deadline;
+	chudcpu_data_t			*target_chudp;
+	boolean_t old_level;
+
+	disable_preemption();
+	// force interrupts on for a cross CPU signal.
+	old_level = chudxnu_set_interrupts_enabled(TRUE);
+	thisCPU = cpu_number();
+
+	if ((unsigned) otherCPU < real_ncpus &&
+	    thisCPU != otherCPU &&
+	    cpu_data_ptr[otherCPU]->cpu_running) {
+
+		target_chudp = (chudcpu_data_t *)
+					cpu_data_ptr[otherCPU]->cpu_chud;
+
+		/* Fill out request */
+		request.req_sync = 0xFFFFFFFF;		/* set sync flag */
+		//request.req_type = CPRQchud;		/* set request type */
+		request.req_code = request_code;	/* set request */
+
+		KERNEL_DEBUG_CONSTANT(
+			MACHDBG_CODE(DBG_MACH_CHUD,
+				     CHUD_CPUSIG_SEND) | DBG_FUNC_NONE,
+			otherCPU, request_code, 0, 0, 0);
+
+		/*
+		 * Insert the new request in the target cpu's request queue
+		 * and signal target cpu.
+		 */
+		mpenqueue_tail(&target_chudp->cpu_request_queue,
+			       &request.req_entry);
+		i386_signal_cpu(otherCPU, MP_CHUD, ASYNC);
+
+		/* Wait for response or timeout */
+		deadline = mach_absolute_time() + LockTimeOut;
+		while (request.req_sync != 0) {
+			if (mach_absolute_time() > deadline) {
+				panic("chudxnu_cpusig_send(%d,%d) timed out\n",
+					otherCPU, request_code);
+			}
+			cpu_pause();
+		}
+		retval = KERN_SUCCESS;
+	} else {
+		retval = KERN_INVALID_ARGUMENT;
+	}
+
+	chudxnu_set_interrupts_enabled(old_level);
+	enable_preemption();
+	return retval;
+}
+
+#ifdef XXX
+#pragma mark **** CHUD syscall (PPC) ****
+
+typedef int (*PPCcallEnt)(struct savearea *save);
+extern PPCcallEnt      PPCcalls[];
+
+static chudxnu_syscall_callback_func_t syscall_callback_fn = NULL;
+
+static int
+chudxnu_private_syscall_callback(struct savearea *ssp)
+{
+	if(ssp) {
+		if(syscall_callback_fn) {
+			struct ppc_thread_state64 state;
+			kern_return_t retval;
+			mach_msg_type_number_t count = PPC_THREAD_STATE64_COUNT;
+			chudxnu_copy_savearea_to_threadstate(PPC_THREAD_STATE64, (thread_state_t)&state, &count, ssp);
+			ssp->save_r3 = (syscall_callback_fn)(PPC_THREAD_STATE64, (thread_state_t)&state, count);
+		} else {
+			ssp->save_r3 = KERN_FAILURE;
+		}
+	}
+	
+    return 1; // check for ASTs (always)
+}
+
+__private_extern__ kern_return_t
+chudxnu_syscall_callback_enter(chudxnu_syscall_callback_func_t func)
+{
+	syscall_callback_fn = func;
+	PPCcalls[9] = chudxnu_private_syscall_callback;
+    __asm__ volatile("eieio");	/* force order */
+    __asm__ volatile("sync");	/* force to memory */
+    return KERN_SUCCESS;
+}
+
+__private_extern__ kern_return_t
+chudxnu_syscall_callback_cancel(void)
+{
+	syscall_callback_fn = NULL;
+	PPCcalls[9] = NULL;
+    __asm__ volatile("eieio");	/* force order */
+    __asm__ volatile("sync");	/* force to memory */
+    return KERN_SUCCESS;
+}
+#endif
diff --git a/osfmk/chud/i386/chud_thread_i386.c b/osfmk/chud/i386/chud_thread_i386.c
new file mode 100644
index 000000000..5b057025b
--- /dev/null
+++ b/osfmk/chud/i386/chud_thread_i386.c
@@ -0,0 +1,657 @@
+/*
+ * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <mach/mach_types.h>
+#include <mach/task.h>
+#include <mach/thread_act.h>
+
+#include <kern/kern_types.h>
+#include <kern/processor.h>
+#include <kern/thread.h>
+
+#include <vm/vm_map.h>
+#include <vm/pmap.h>
+
+#include <chud/chud_xnu.h>
+#include <chud/chud_xnu_private.h>
+
+#include <i386/misc_protos.h>
+#include <i386/proc_reg.h>
+#include <i386/mp_desc.h>
+
+#pragma mark **** thread state ****
+
+__private_extern__ kern_return_t
+chudxnu_thread_user_state_available(thread_t thread)
+{
+#pragma unused (thread)
+	return KERN_SUCCESS;
+}
+
+__private_extern__ kern_return_t
+chudxnu_thread_get_state(
+	thread_t	 	thread, 
+	thread_flavor_t	 	flavor,
+	thread_state_t	 	tstate,
+	mach_msg_type_number_t	*count,
+	boolean_t	 	user_only)
+{
+	if (user_only) {
+		/* We can't get user state for kernel threads */
+		if (thread->task == kernel_task)
+			return KERN_FAILURE;
+		/* this properly handles deciding whether or not the thread is 64 bit or not */
+		return machine_thread_get_state(thread, flavor, tstate, count);
+	} else {
+		// i386 machine_thread_get_kern_state() is different from the PPC version which returns
+		// the previous save area - user or kernel - rather than kernel or NULL if no kernel
+		// interrupt state available
+	
+		// the real purpose of this branch is the following:
+		// the user doesn't care if the thread states are user or kernel, he
+		// just wants the thread state, so we need to determine the proper one
+		// to return, kernel or user, for the given thread.
+		if(thread == current_thread() && current_cpu_datap()->cpu_int_state) {
+			// the above are conditions where we possibly can read the kernel
+			// state. we still need to determine if this interrupt happened in
+			// kernel or user context
+			if(USER_STATE(thread) == current_cpu_datap()->cpu_int_state &&
+				current_cpu_datap()->cpu_interrupt_level == 1) {
+				// interrupt happened in user land
+				return machine_thread_get_state(thread, flavor, tstate, count);
+			} else {
+				// kernel interrupt.
+				return machine_thread_get_kern_state(thread, flavor, tstate, count);
+			}
+		} else {
+            // get the user-mode thread state
+			return machine_thread_get_state(thread, flavor, tstate, count);
+		}
+	}
+}
+
+__private_extern__ kern_return_t
+chudxnu_thread_set_state(
+	thread_t		thread, 
+	thread_flavor_t		flavor,
+	thread_state_t		tstate,
+	mach_msg_type_number_t	count,
+	boolean_t 		user_only)
+{
+#pragma unused (user_only)
+	return machine_thread_set_state(thread, flavor, tstate, count);
+}
+
+#pragma mark **** task memory read/write ****
+    
+__private_extern__ kern_return_t
+chudxnu_task_read(
+	task_t		task,
+	void		*kernaddr,
+	uint64_t	usraddr,
+	vm_size_t	size)
+{
+	kern_return_t ret = KERN_SUCCESS;
+    
+	if(current_task()==task) {
+		if(ml_at_interrupt_context()) {
+			return KERN_FAILURE; // can't do copyin on interrupt stack
+		}
+	
+		if(copyin(usraddr, kernaddr, size)) {
+			ret = KERN_FAILURE;
+		}
+	} else {
+		vm_map_t map = get_task_map(task);
+		ret = vm_map_read_user(map, usraddr, kernaddr, size);
+	}
+    
+	return ret;
+}
+			
+__private_extern__ kern_return_t
+chudxnu_task_write(
+	task_t		task,
+	uint64_t	useraddr,
+	void		*kernaddr,
+	vm_size_t	size)
+{
+	kern_return_t ret = KERN_SUCCESS;
+    
+	if(current_task()==task) {    
+		if(ml_at_interrupt_context()) {
+			return KERN_FAILURE; // can't do copyout on interrupt stack
+		}
+	
+		if(copyout(kernaddr, useraddr, size)) {
+			ret = KERN_FAILURE;
+		}
+	} else {
+		vm_map_t map = get_task_map(task);
+		ret = vm_map_write_user(map, kernaddr, useraddr, size);
+	}		
+    
+	return ret;
+}
+
+__private_extern__ kern_return_t
+chudxnu_kern_read(void *dstaddr, vm_offset_t srcaddr, vm_size_t size)
+{
+    while(size>0) {
+		ppnum_t pp;
+		addr64_t phys_addr;    
+		
+		/* Get the page number */
+		pp = pmap_find_phys(kernel_pmap, srcaddr);
+		if(!pp) {
+			return KERN_FAILURE;	/* Not mapped... */
+		}
+		
+		/* Shove in the page offset */
+		phys_addr = ((addr64_t)pp << 12) |
+				(srcaddr & 0x0000000000000FFFULL);
+		if(phys_addr >= mem_actual) {
+			return KERN_FAILURE;	/* out of range */
+		}
+		
+		if((phys_addr&0x1) || size==1) {
+			*((uint8_t *)dstaddr) =
+				ml_phys_read_byte_64(phys_addr);
+			dstaddr = ((uint8_t *)dstaddr) + 1;
+			srcaddr += sizeof(uint8_t);
+			size -= sizeof(uint8_t);
+		} else if((phys_addr&0x3) || size<=2) {
+			*((uint16_t *)dstaddr) =
+				ml_phys_read_half_64(phys_addr);
+			dstaddr = ((uint16_t *)dstaddr) + 1;
+			srcaddr += sizeof(uint16_t);
+			size -= sizeof(uint16_t);
+		} else {
+			*((uint32_t *)dstaddr) =
+				ml_phys_read_word_64(phys_addr);
+			dstaddr = ((uint32_t *)dstaddr) + 1;
+			srcaddr += sizeof(uint32_t);
+			size -= sizeof(uint32_t);
+		}
+    }
+    return KERN_SUCCESS;
+}
+
+__private_extern__ kern_return_t
+chudxnu_kern_write(
+	vm_offset_t	dstaddr,
+	void		*srcaddr,
+	vm_size_t	size)
+{
+	while(size>0) {
+		ppnum_t pp;
+		addr64_t phys_addr;    
+		
+		/* Get the page number */
+		pp = pmap_find_phys(kernel_pmap, dstaddr);
+		if(!pp) {
+			return KERN_FAILURE;	/* Not mapped... */
+		}
+		
+		/* Shove in the page offset */
+		phys_addr = ((addr64_t)pp << 12) |
+				(dstaddr & 0x0000000000000FFFULL);
+		if(phys_addr > mem_actual) {
+			return KERN_FAILURE;	/* out of range */
+		}
+		
+		if((phys_addr&0x1) || size==1) {
+			ml_phys_write_byte_64(phys_addr, *((uint8_t *)srcaddr));
+			srcaddr = ((uint8_t *)srcaddr) + 1;
+			dstaddr += sizeof(uint8_t);
+			size -= sizeof(uint8_t);
+		} else if((phys_addr&0x3) || size<=2) {
+			ml_phys_write_half_64(phys_addr, *((uint16_t *)srcaddr));
+			srcaddr = ((uint16_t *)srcaddr) + 1;
+			dstaddr += sizeof(uint16_t);
+			size -= sizeof(uint16_t);
+		} else {
+			ml_phys_write_word_64(phys_addr, *((uint32_t *)srcaddr));
+			srcaddr = ((uint32_t *)srcaddr) + 1;
+			dstaddr += sizeof(uint32_t);
+			size -= sizeof(uint32_t);
+		}
+    }
+    
+    return KERN_SUCCESS;
+}
+
+#define VALID_STACK_ADDRESS(supervisor, addr, minKernAddr, maxKernAddr)   (supervisor ? (addr>=minKernAddr && addr<=maxKernAddr) : TRUE)
+// don't try to read in the hole
+#define VALID_STACK_ADDRESS64(supervisor, addr, minKernAddr, maxKernAddr) \
+	(supervisor ? (addr >= minKernAddr && addr <= maxKernAddr) : \
+		(addr != 0 && (addr <= 0x00007FFFFFFFFFFFULL || addr >= 0xFFFF800000000000ULL)))
+
+typedef struct _cframe64_t {
+	uint64_t	prevFP;		// can't use a real pointer here until we're a 64 bit kernel
+	uint64_t	caller;
+	uint64_t	args[0];
+}cframe64_t;
+
+
+typedef struct _cframe_t {
+	struct _cframe_t	*prev;	// when we go 64 bits, this needs to be capped at 32 bits
+	uint32_t		caller;
+	uint32_t		args[0];
+} cframe_t;
+
+__private_extern__
+kern_return_t chudxnu_thread_get_callstack64(
+	thread_t		thread,
+	uint64_t		*callstack,
+	mach_msg_type_number_t	*count,
+	boolean_t		user_only)
+{
+    kern_return_t kr = KERN_FAILURE;
+	kern_return_t ret = KERN_SUCCESS;
+    task_t task = thread->task;
+    uint64_t currPC = 0;
+	uint64_t prevPC = 0;
+    uint64_t currFP = 0;
+    uint64_t prevFP = 0;
+    uint64_t rsp = 0;
+    uint64_t kernStackMin = min_valid_stack_address();
+    uint64_t kernStackMax = max_valid_stack_address();
+    uint64_t *buffer = callstack;
+    int bufferIndex = 0;
+    int bufferMaxIndex = *count;
+    boolean_t supervisor = FALSE;
+	boolean_t is64bit = FALSE;
+	void * t_regs;
+	
+	if (user_only) {
+		/* We can't get user state for kernel threads */
+		if (task == kernel_task) {
+			return KERN_FAILURE;
+		}
+        t_regs = USER_STATE(thread);
+		
+		if(is_saved_state64(t_regs)) {
+			void *int_state = current_cpu_datap()->cpu_int_state;
+			x86_saved_state64_t *s64 = saved_state64(t_regs);
+			
+			if(int_state) {	// are we on an interrupt that happened in user land
+				supervisor = !(t_regs == int_state && current_cpu_datap()->cpu_interrupt_level == 1);
+			} else {
+				if(s64) {
+					supervisor = ((s64->isf.cs & SEL_PL) != SEL_PL_U);
+				} else {
+					// assume 32 bit kernel
+					supervisor = FALSE;	
+				}
+			}
+			is64bit = TRUE;
+		} else {
+			x86_saved_state32_t *regs;
+
+			regs = saved_state32(t_regs);
+			
+			// find out if we're in supervisor mode
+			supervisor = ((regs->cs & SEL_PL) != SEL_PL_U);
+			is64bit = FALSE;
+		}
+    } else {
+    	t_regs = current_cpu_datap()->cpu_int_state;
+		x86_saved_state32_t *regs;
+
+        regs = saved_state32(t_regs);
+		
+		// find out if we're in supervisor mode
+        supervisor = ((regs->cs & SEL_PL) != SEL_PL_U);
+		is64bit = FALSE;
+    }
+	
+	if(is64bit) {
+		x86_saved_state64_t *regs = saved_state64(t_regs);
+		
+		if(user_only) {
+			/* cant get user state for kernel threads */
+			if(task == kernel_task) {
+				return KERN_FAILURE;
+			}
+			regs = USER_REGS64(thread);
+		} 
+		
+		currPC = regs->isf.rip;
+		currFP = regs->rbp;
+		
+		if(!currPC)
+		{
+			*count = 0;
+			return KERN_FAILURE;
+		}
+		
+		bufferIndex = 0;
+		
+		//allot space for saving %rsp on the 
+		//bottom of the stack for user callstacks
+		if(!supervisor)
+			bufferMaxIndex = bufferMaxIndex - 1;    
+			
+		if(bufferMaxIndex < 1) {
+			*count = 0;
+			return KERN_RESOURCE_SHORTAGE;
+		}
+		buffer[bufferIndex++] = currPC; // save RIP on the top of the stack
+
+		// now make a 64bit back trace
+		while (VALID_STACK_ADDRESS64(supervisor, currFP, kernStackMin, kernStackMax))
+		{
+			// this is the address where caller lives in the user thread
+			uint64_t caller = currFP + sizeof(uint64_t);
+			if(!currFP) {
+				currPC = 0;
+				break;
+			}
+			
+			if(bufferIndex >= bufferMaxIndex) {
+				*count = bufferMaxIndex;
+				return KERN_RESOURCE_SHORTAGE;
+			}
+
+			/* read our caller */
+			kr = chudxnu_task_read(task, &currPC, caller, sizeof(uint64_t));    
+			
+			if(kr != KERN_SUCCESS) {
+				currPC = 0;
+				break;
+			}
+			
+			/* 
+			 * retrive contents of the frame pointer and advance to the next stack
+			 * frame if it's valid 
+			 */
+			prevFP = 0;
+			kr = chudxnu_task_read(task, &prevFP, currFP, sizeof(uint64_t));
+			
+			if(kr != KERN_SUCCESS) {
+				currPC = 0;
+				break;
+			}
+	
+			if(VALID_STACK_ADDRESS64(supervisor, prevFP, kernStackMin, kernStackMax)) {
+				buffer[bufferIndex++] = currPC;
+				prevPC = currPC;
+			}
+			if(prevFP < currFP) {
+				break;
+			} else {
+				currFP = prevFP;
+			}
+		}
+
+		// append (rsp) on the bottom of the callstack
+		kr = chudxnu_task_read(task, &rsp, (addr64_t) regs->isf.rsp, sizeof(uint64_t));
+		if(kr == KERN_SUCCESS) {
+			buffer[bufferIndex++] = rsp;
+		}
+    } else {
+		/* !thread_is_64bit() */
+		/* we grab 32 bit frames and silently promote them to 64 bits */
+		uint32_t tmpWord = 0;
+		x86_saved_state32_t *regs = NULL;
+		
+		if(user_only) {
+			/* cant get user state for kernel threads */
+			if(task == kernel_task || supervisor) {
+				return 0x11;
+			}
+			regs = USER_REGS32(thread);
+		} else {
+			regs = saved_state32(current_cpu_datap()->cpu_int_state);
+		}
+		
+		if(regs == NULL) {
+			*count = 0;
+			return 0x12;
+		}
+
+		currPC = (uint64_t) regs->eip;
+		currFP = (uint64_t) regs->ebp;
+		
+		bufferIndex = 0;
+		//if(!supervisor)
+		//	bufferMaxIndex = bufferMaxIndex - 1;    //allot space for saving %rsp on the stack for user callstacks
+		if(bufferMaxIndex < 1) {
+			*count = 0;
+			return KERN_RESOURCE_SHORTAGE;
+		}
+		buffer[bufferIndex++] = currPC; // save EIP on the top of the stack
+
+		// now make a 64bit back trace from 32 bit stack frames
+		while (VALID_STACK_ADDRESS(supervisor, currFP, kernStackMin, kernStackMax))
+		{
+			cframe_t *fp = (cframe_t *) (uint32_t) currFP;
+
+			if(bufferIndex >= bufferMaxIndex) {
+				*count = bufferMaxIndex;
+				return KERN_RESOURCE_SHORTAGE;
+			}
+
+			/* read the next frame */
+			if(supervisor) {
+				kr = chudxnu_kern_read(&tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
+			} else {
+				kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));    
+			}
+			
+			if(kr != KERN_SUCCESS) {
+				currPC = 0;
+				break;
+			}
+			
+			currPC = (uint64_t) tmpWord;	// promote 32 bit address
+			
+			/* 
+			 * retrive contents of the frame pointer and advance to the next stack
+			 * frame if it's valid 
+			 */
+			prevFP = 0;
+			if(supervisor) {
+				kr = chudxnu_kern_read(&tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
+			} else {
+				kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
+			}
+			prevFP = (uint64_t) tmpWord;	// promote 32 bit address
+			
+			if(prevFP) {
+				buffer[bufferIndex++] = currPC;
+				prevPC = currPC;
+			}
+			if(prevFP < currFP) {
+				break;
+			} else {
+				currFP = prevFP;
+			}
+		}
+
+		// append (esp) on the bottom of the callstack
+		if(!supervisor) {
+			kr = chudxnu_task_read(task, &tmpWord, regs->uesp, sizeof(uint32_t));
+			if(kr == KERN_SUCCESS) {
+				rsp = (uint64_t) tmpWord;	// promote 32 bit address
+				buffer[bufferIndex++] = rsp;
+			}
+		}
+    }
+    
+    *count = bufferIndex;
+    return ret;
+}
+
+__private_extern__ kern_return_t
+chudxnu_thread_get_callstack(
+	thread_t		thread, 
+	uint32_t		*callStack,
+	mach_msg_type_number_t	*count,
+	boolean_t		user_only)
+{
+	kern_return_t   kr;
+	task_t          task = thread->task;
+	uint32_t        currPC;
+	uint32_t        currFP;
+	uint32_t        prevFP = 0;
+	uint32_t        prevPC = 0;
+    uint32_t        esp = 0;
+	uint32_t        kernStackMin = min_valid_stack_address();
+	uint32_t        kernStackMax = max_valid_stack_address();
+	uint32_t       *buffer = callStack;
+	int             bufferIndex = 0;
+	int             bufferMaxIndex = *count;
+	boolean_t       supervisor;
+	x86_saved_state32_t *regs = NULL;
+
+    if (user_only) {
+		/* We can't get user state for kernel threads */
+		if (task == kernel_task) {
+			return KERN_FAILURE;
+		}
+        regs = USER_REGS32(thread);
+    } else {
+    	regs = saved_state32(current_cpu_datap()->cpu_int_state);
+    }
+
+    if (regs == NULL) {
+        *count = 0;
+		return KERN_FAILURE;
+    }
+	
+	supervisor = ((regs->cs & SEL_PL) != SEL_PL_U);
+	
+	currPC = regs->eip;
+	currFP = regs->ebp;
+	
+	bufferIndex = 0;
+    if(!supervisor)
+        bufferMaxIndex -= 1;    // allot space for saving userland %esp on stack
+	if (bufferMaxIndex < 1) {
+		*count = 0;
+		return KERN_RESOURCE_SHORTAGE;
+	}
+	buffer[bufferIndex++] = currPC; //save PC in position 0.
+
+	// Now, fill buffer with stack backtraces.
+	while (VALID_STACK_ADDRESS(supervisor, currFP, kernStackMin, kernStackMax)) {
+		cframe_t	*fp = (cframe_t *) currFP;
+
+		if (bufferIndex >= bufferMaxIndex) {
+			*count = bufferMaxIndex;
+			return KERN_RESOURCE_SHORTAGE;
+		}
+
+		if (supervisor) {
+			kr = chudxnu_kern_read(
+					&currPC,
+					(vm_offset_t) &fp->caller,
+					sizeof(currPC));
+		} else {
+			kr = chudxnu_task_read(
+					task,
+					&currPC,
+					(vm_offset_t) &fp->caller,
+					sizeof(currPC));
+		}
+		if (kr != KERN_SUCCESS)
+			break;
+
+		//retrieve the contents of the frame pointer
+		// and advance to the prev stack frame if it's valid
+		prevFP = 0;
+		if (supervisor) {
+			kr = chudxnu_kern_read(
+					&prevFP,
+					(vm_offset_t) &fp->prev,
+					sizeof(prevFP));
+		} else {
+			kr = chudxnu_task_read(
+					task,
+					&prevFP,
+					(vm_offset_t) &fp->prev,
+					sizeof(prevFP));
+		}
+		if (prevFP) {
+			buffer[bufferIndex++] = currPC;
+			prevPC = currPC;
+		}
+		if (prevFP < currFP) {
+			break;
+		} else {
+			currFP = prevFP;
+		}
+	}
+
+	// put the stack pointer on the bottom of the backtrace
+    if(!supervisor) {
+        kr = chudxnu_task_read(task, &esp, regs->uesp, sizeof(uint32_t));
+        if(kr == KERN_SUCCESS) {
+            buffer[bufferIndex++] = esp;
+        }
+    }
+
+	*count = bufferIndex;
+	return KERN_SUCCESS;
+}
+
+
+#pragma mark **** DEPRECATED ****
+
+// DEPRECATED
+__private_extern__
+kern_return_t chudxnu_bind_current_thread(int cpu)
+{
+	return chudxnu_bind_thread(current_thread(), cpu);
+}
+
+// DEPRECATED
+kern_return_t chudxnu_unbind_current_thread(void)
+{
+	return chudxnu_unbind_thread(current_thread());
+}
+
+// DEPRECATED
+__private_extern__
+kern_return_t chudxnu_current_thread_get_callstack(
+	uint32_t		*callStack,
+	mach_msg_type_number_t	*count,
+	boolean_t		user_only)
+{
+	return chudxnu_thread_get_callstack(
+			current_thread(), callStack, count, user_only);
+}
+
+// DEPRECATED
+__private_extern__
+thread_t chudxnu_current_act(void)
+{
+	return chudxnu_current_thread();
+}
diff --git a/osfmk/ppc/chud/chud_xnu_glue.h b/osfmk/chud/i386/chud_xnu_glue.h
similarity index 100%
rename from osfmk/ppc/chud/chud_xnu_glue.h
rename to osfmk/chud/i386/chud_xnu_glue.h
diff --git a/osfmk/chud/i386/chud_xnu_private.h b/osfmk/chud/i386/chud_xnu_private.h
new file mode 100644
index 000000000..d594b7fdf
--- /dev/null
+++ b/osfmk/chud/i386/chud_xnu_private.h
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _I386_CHUD_XNU_PRIVATE_H_
+#define _I386_CHUD_XNU_PRIVATE_H_
+
+#include <kern/queue.h>
+
+#pragma mark **** cpu timer ****
+
+/*
+ * Cross-cpu signal request entries are queued on the target cpu's
+ * chudcpu_data_t struct. This differs from PPC because i386 doesn't
+ * support sending arguments with cross-cpu signals. Hence we have
+ * to do it ourselves.
+ */ 
+typedef struct {
+	struct queue_entry	req_entry;	/* Must be first */
+	uint32_t		req_type;
+	uint32_t		req_code;
+	volatile uint32_t	req_sync;
+} chudcpu_signal_request_t;
+
+typedef struct {
+	void					*cpu_chud_fn_tablep;
+	timer_call_data_t			cpu_timer_call;
+	uint64_t				t_deadline;
+	chudxnu_cpu_timer_callback_func_t	cpu_timer_callback_fn;
+	mpqueue_head_t				cpu_request_queue;
+} chudcpu_data_t;
+/* NB: cpu_chud_fn_tablep is expected to be the first member, at offset 0 */
+
+extern void chudxnu_cpu_signal_handler(void);
+
+#endif /* _I386_CHUD_XNU_PRIVATE_H_ */
diff --git a/osfmk/ppc/chud/chud_cpu_asm.h b/osfmk/chud/ppc/chud_cpu_asm.h
similarity index 100%
rename from osfmk/ppc/chud/chud_cpu_asm.h
rename to osfmk/chud/ppc/chud_cpu_asm.h
diff --git a/osfmk/ppc/chud/chud_cpu_asm.s b/osfmk/chud/ppc/chud_cpu_asm.s
similarity index 99%
rename from osfmk/ppc/chud/chud_cpu_asm.s
rename to osfmk/chud/ppc/chud_cpu_asm.s
index 44789e322..88a4b3308 100644
--- a/osfmk/ppc/chud/chud_cpu_asm.s
+++ b/osfmk/chud/ppc/chud_cpu_asm.s
@@ -27,7 +27,7 @@
  */
  
 #define ASSEMBLER
-#include <ppc/chud/chud_spr.h>
+#include <chud/ppc/chud_spr.h>
 #include <ppc/asm.h>
 #include <mach/kern_return.h>
 
diff --git a/osfmk/ppc/chud/chud_cpu.c b/osfmk/chud/ppc/chud_cpu_ppc.c
similarity index 96%
rename from osfmk/ppc/chud/chud_cpu.c
rename to osfmk/chud/ppc/chud_cpu_ppc.c
index 8bb5bfbd9..22927ecf4 100644
--- a/osfmk/ppc/chud/chud_cpu.c
+++ b/osfmk/chud/ppc/chud_cpu_ppc.c
@@ -31,9 +31,9 @@
 #include <kern/host.h>
 #include <kern/processor.h>
 
-#include <ppc/chud/chud_spr.h>
-#include <ppc/chud/chud_xnu.h>
-#include <ppc/chud/chud_cpu_asm.h>
+#include <chud/chud_xnu.h>
+#include <chud/ppc/chud_spr.h>
+#include <chud/ppc/chud_cpu_asm.h>
 #include <ppc/machine_routines.h>
 #include <ppc/exception.h>
 #include <ppc/hw_perfmon.h>
@@ -56,44 +56,6 @@
 #define mtsr(sr, reg) __asm__ volatile("sync" "@" "mtsr sr%0, %1 " "@" "isync" : : "i" (sr), "r" (reg)); 
 #define mfsr(reg, sr) __asm__ volatile("mfsr %0, sr%1" : "=r" (reg) : "i" (sr));
 
-#pragma mark **** cpu count ****
-
-__private_extern__
-int chudxnu_avail_cpu_count(void)
-{
-    host_basic_info_data_t hinfo;
-    kern_return_t kr;
-    mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
-
-    kr = host_info(host_self(), HOST_BASIC_INFO, (integer_t *)&hinfo, &count);
-    if(kr == KERN_SUCCESS) {
-        return hinfo.avail_cpus;
-    } else {
-        return 0;
-    }
-}
-
-__private_extern__
-int chudxnu_phys_cpu_count(void)
-{
-    host_basic_info_data_t hinfo;
-    kern_return_t kr;
-    mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
-
-    kr = host_info(host_self(), HOST_BASIC_INFO, (integer_t *)&hinfo, &count);
-    if(kr == KERN_SUCCESS) {
-        return hinfo.max_cpus;
-    } else {
-        return 0;
-    }
-}
-
-__private_extern__
-int chudxnu_cpu_number(void)
-{
-    return cpu_number();
-}
-
 #pragma mark **** cpu enable/disable ****
 
 extern kern_return_t processor_start(processor_t     processor); // osfmk/kern/processor.c
@@ -1147,45 +1109,6 @@ kern_return_t chudxnu_perfmon_release_facility(task_t task)
     return perfmon_release_facility(task);
 }
 
-#pragma mark **** branch trace buffer ****
-
-extern int pc_trace_buf[1024];
-
-__private_extern__
-uint32_t * chudxnu_get_branch_trace_buffer(uint32_t *entries)
-{
-    if(entries) {
-        *entries = sizeof(pc_trace_buf)/sizeof(int);
-    }
-    return pc_trace_buf;
-}
-
-#pragma mark **** interrupts enable/disable ****
-
-__private_extern__
-boolean_t chudxnu_get_interrupts_enabled(void)
-{
-    return ml_get_interrupts_enabled();
-}
-
-__private_extern__
-boolean_t chudxnu_set_interrupts_enabled(boolean_t enable)
-{
-    return ml_set_interrupts_enabled(enable);
-}
-
-__private_extern__
-boolean_t chudxnu_at_interrupt_context(void)
-{
-    return ml_at_interrupt_context();
-}
-
-__private_extern__
-void chudxnu_cause_interrupt(void)
-{
-    ml_cause_interrupt();
-}
-
 #pragma mark **** rupt counters ****
 
 __private_extern__
diff --git a/osfmk/ppc/chud/chud_osfmk_callback.c b/osfmk/chud/ppc/chud_osfmk_callback_ppc.c
similarity index 72%
rename from osfmk/ppc/chud/chud_osfmk_callback.c
rename to osfmk/chud/ppc/chud_osfmk_callback_ppc.c
index 788082088..763420508 100644
--- a/osfmk/ppc/chud/chud_osfmk_callback.c
+++ b/osfmk/chud/ppc/chud_osfmk_callback_ppc.c
@@ -32,7 +32,6 @@
 
 #include <kern/kern_types.h>
 #include <kern/processor.h>
-#include <kern/timer_call.h>
 #include <kern/thread_call.h>
 #include <kern/kalloc.h>
 #include <kern/thread.h>
@@ -44,8 +43,8 @@
 #include <ppc/thread.h>
 #include <ppc/trap.h>
 
-#include <ppc/chud/chud_xnu.h>
-#include <ppc/chud/chud_xnu_private.h>
+#include <chud/chud_xnu.h>
+#include <chud/chud_xnu_private.h>
 
 __private_extern__
 void chudxnu_cancel_all_callbacks(void)
@@ -60,13 +59,6 @@ void chudxnu_cancel_all_callbacks(void)
 	chudxnu_syscall_callback_cancel();
 }
 
-#pragma mark **** cpu timer ****
-typedef struct {
-	timer_call_data_t	cpu_timer_call;
-	uint64_t		t_deadline;
-	chudxnu_cpu_timer_callback_func_t	cpu_timer_callback_fn;
-} chudcpu_data_t;
-
 static chudcpu_data_t chudcpu_boot_cpu;
 
 void *chudxnu_per_proc_alloc(boolean_t boot_processor)
@@ -101,14 +93,16 @@ static void chudxnu_private_cpu_timer_callback(timer_call_param_t param0, timer_
     boolean_t oldlevel;
     struct ppc_thread_state64 state;
     mach_msg_type_number_t count;
+    chudxnu_cpu_timer_callback_func_t fn = NULL;
 
     oldlevel = ml_set_interrupts_enabled(FALSE);
     chud_proc_info = (chudcpu_data_t *)(getPerProc()->pp_chud);
 
     count = PPC_THREAD_STATE64_COUNT;
     if(chudxnu_thread_get_state(current_thread(), PPC_THREAD_STATE64, (thread_state_t)&state, &count, FALSE)==KERN_SUCCESS) {
-        if(chud_proc_info->cpu_timer_callback_fn) {
-            (chud_proc_info->cpu_timer_callback_fn)(PPC_THREAD_STATE64, (thread_state_t)&state, count);
+        fn = chud_proc_info->cpu_timer_callback_fn;
+        if(fn) {
+            (fn)(PPC_THREAD_STATE64, (thread_state_t)&state, count);
         }
     }
 
@@ -209,13 +203,14 @@ static kern_return_t chudxnu_private_trap_callback(int trapno, struct savearea *
     boolean_t oldlevel = ml_set_interrupts_enabled(FALSE);
     kern_return_t retval = KERN_FAILURE;
     uint32_t trapentry = TRAP_ENTRY_POINT(trapno);
+    chudxnu_trap_callback_func_t fn = trap_callback_fn;
 
     if(trapentry!=0x0) {
-        if(trap_callback_fn) {
+        if(fn) {
             struct ppc_thread_state64 state;
             mach_msg_type_number_t count = PPC_THREAD_STATE64_COUNT;
             chudxnu_copy_savearea_to_threadstate(PPC_THREAD_STATE64, (thread_state_t)&state, &count, ssp);
-            retval = (trap_callback_fn)(trapentry, PPC_THREAD_STATE64, (thread_state_t)&state, count);
+            retval = (fn)(trapentry, PPC_THREAD_STATE64, (thread_state_t)&state, count);
         }
     }
 
@@ -252,40 +247,41 @@ static kern_return_t chudxnu_private_chud_ast_callback(int trapno, struct savear
     boolean_t oldlevel = ml_set_interrupts_enabled(FALSE);
     ast_t *myast = ast_pending();
     kern_return_t retval = KERN_FAILURE;
+    chudxnu_perfmon_ast_callback_func_t fn = perfmon_ast_callback_fn;
     
-	if(*myast & AST_PPC_CHUD_URGENT) {
-		*myast &= ~(AST_PPC_CHUD_URGENT | AST_PPC_CHUD);
+	if(*myast & AST_CHUD_URGENT) {
+		*myast &= ~(AST_CHUD_URGENT | AST_CHUD);
 		if((*myast & AST_PREEMPTION) != AST_PREEMPTION) *myast &= ~(AST_URGENT);
 		retval = KERN_SUCCESS;
-	} else if(*myast & AST_PPC_CHUD) {
-		*myast &= ~(AST_PPC_CHUD);
+	} else if(*myast & AST_CHUD) {
+		*myast &= ~(AST_CHUD);
 		retval = KERN_SUCCESS;
 	}
 
-    if(perfmon_ast_callback_fn) {
+    if(fn) {
 		struct ppc_thread_state64 state;
 		mach_msg_type_number_t count;
 		count = PPC_THREAD_STATE64_COUNT;
 		
 		if(chudxnu_thread_get_state(current_thread(), PPC_THREAD_STATE64, (thread_state_t)&state, &count, FALSE)==KERN_SUCCESS) {
-			(perfmon_ast_callback_fn)(PPC_THREAD_STATE64, (thread_state_t)&state, count);
+			(fn)(PPC_THREAD_STATE64, (thread_state_t)&state, count);
 		}
     }
     
 #if 0
     // ASTs from ihandler go through thandler and are made to look like traps
-    // always handle AST_PPC_CHUD_URGENT if there's a callback
-    // only handle AST_PPC_CHUD if it's the only AST pending
-    if(perfmon_ast_callback_fn && ((*myast & AST_PPC_CHUD_URGENT) || ((*myast & AST_PPC_CHUD) && !(*myast & AST_URGENT)))) {
+    // always handle AST_CHUD_URGENT if there's a callback
+    // only handle AST_CHUD if it's the only AST pending
+    if(perfmon_ast_callback_fn && ((*myast & AST_CHUD_URGENT) || ((*myast & AST_CHUD) && !(*myast & AST_URGENT)))) {
         struct ppc_thread_state64 state;
         mach_msg_type_number_t count = PPC_THREAD_STATE64_COUNT;
         chudxnu_copy_savearea_to_threadstate(PPC_THREAD_STATE64, (thread_state_t)&state, &count, ssp);
-        if(*myast & AST_PPC_CHUD_URGENT) {
-            *myast &= ~(AST_PPC_CHUD_URGENT | AST_PPC_CHUD);
+        if(*myast & AST_CHUD_URGENT) {
+            *myast &= ~(AST_CHUD_URGENT | AST_CHUD);
             if((*myast & AST_PREEMPTION) != AST_PREEMPTION) *myast &= ~(AST_URGENT);
 			retval = KERN_SUCCESS;
-        } else if(*myast & AST_PPC_CHUD) {
-            *myast &= ~(AST_PPC_CHUD);
+        } else if(*myast & AST_CHUD) {
+            *myast &= ~(AST_CHUD);
 			retval = KERN_SUCCESS;
         }
         (perfmon_ast_callback_fn)(PPC_THREAD_STATE64, (thread_state_t)&state, count);
@@ -323,9 +319,9 @@ kern_return_t chudxnu_perfmon_ast_send_urgent(boolean_t urgent)
 	ast_t *myast = ast_pending();
 
     if(urgent) {
-        *myast |= (AST_PPC_CHUD_URGENT | AST_URGENT);
+        *myast |= (AST_CHUD_URGENT | AST_URGENT);
     } else {
-        *myast |= (AST_PPC_CHUD);
+        *myast |= (AST_CHUD);
     }
 
     ml_set_interrupts_enabled(oldlevel);
@@ -344,11 +340,13 @@ static chudxnu_interrupt_callback_func_t interrupt_callback_fn = NULL;
 
 static kern_return_t chudxnu_private_interrupt_callback(int trapno, struct savearea *ssp, unsigned int dsisr, unsigned int dar)
 {
-    if(interrupt_callback_fn) {
+    chudxnu_interrupt_callback_func_t fn = interrupt_callback_fn;
+    
+    if(fn) {
         struct ppc_thread_state64 state;
         mach_msg_type_number_t count = PPC_THREAD_STATE64_COUNT;
         chudxnu_copy_savearea_to_threadstate(PPC_THREAD_STATE64, (thread_state_t)&state, &count, ssp);
-        return (interrupt_callback_fn)(TRAP_ENTRY_POINT(trapno), PPC_THREAD_STATE64, (thread_state_t)&state, count);
+        return (fn)(TRAP_ENTRY_POINT(trapno), PPC_THREAD_STATE64, (thread_state_t)&state, count);
     } else {
         return KERN_FAILURE;
     }
@@ -380,11 +378,13 @@ extern perfCallback perfCpuSigHook; /* function hook into cpu_signal_handler() *
 
 static kern_return_t chudxnu_private_cpu_signal_handler(int request, struct savearea *ssp, unsigned int arg0, unsigned int arg1)
 {
-    if(cpusig_callback_fn) {
+    chudxnu_cpusig_callback_func_t fn = cpusig_callback_fn;
+    
+    if(fn) {
         struct ppc_thread_state64 state;
         mach_msg_type_number_t count = PPC_THREAD_STATE64_COUNT;
         chudxnu_copy_savearea_to_threadstate(PPC_THREAD_STATE64, (thread_state_t)&state, &count, ssp);
-        (cpusig_callback_fn)(request, PPC_THREAD_STATE64, (thread_state_t)&state, count);
+        (fn)(request, PPC_THREAD_STATE64, (thread_state_t)&state, count);
     }
     return KERN_SUCCESS; // ignored
 }
@@ -448,124 +448,3 @@ kern_return_t chudxnu_cpusig_send(int otherCPU, uint32_t request)
     ml_set_interrupts_enabled(oldlevel);
     return retval;
 }
-
-#pragma mark **** timer ****
-__private_extern__
-chud_timer_t chudxnu_timer_alloc(chudxnu_timer_callback_func_t func, uint32_t param0)
-{
-    return (chud_timer_t)thread_call_allocate((thread_call_func_t)func, (thread_call_param_t)param0);
-}
-
-__private_extern__
-kern_return_t chudxnu_timer_callback_enter(chud_timer_t timer, uint32_t param1, uint32_t time, uint32_t units)
-{
-    uint64_t t_delay;
-    clock_interval_to_deadline(time, units, &t_delay);
-    thread_call_enter1_delayed((thread_call_t)timer, (thread_call_param_t)param1, t_delay);
-    return KERN_SUCCESS;
-}
-
-__private_extern__
-kern_return_t chudxnu_timer_callback_cancel(chud_timer_t timer)
-{
-    thread_call_cancel((thread_call_t)timer);
-    return KERN_SUCCESS;
-}
-
-__private_extern__
-kern_return_t chudxnu_timer_free(chud_timer_t timer)
-{
-    thread_call_cancel((thread_call_t)timer);
-    thread_call_free((thread_call_t)timer);
-    return KERN_SUCCESS;
-}
-
-#pragma mark **** CHUD syscall (PPC) ****
-
-typedef int (*PPCcallEnt)(struct savearea *save);
-extern PPCcallEnt      PPCcalls[];
-
-static chudxnu_syscall_callback_func_t syscall_callback_fn = NULL;
-
-static int chudxnu_private_syscall_callback(struct savearea *ssp)
-{
-	if(ssp) {
-		if(syscall_callback_fn) {
-			struct ppc_thread_state64 state;
-			kern_return_t retval;
-			mach_msg_type_number_t count = PPC_THREAD_STATE64_COUNT;
-			chudxnu_copy_savearea_to_threadstate(PPC_THREAD_STATE64, (thread_state_t)&state, &count, ssp);
-			ssp->save_r3 = (syscall_callback_fn)(PPC_THREAD_STATE64, (thread_state_t)&state, count);
-		} else {
-			ssp->save_r3 = KERN_FAILURE;
-		}
-	}
-	
-    return 1; // check for ASTs (always)
-}
-
-__private_extern__
-kern_return_t chudxnu_syscall_callback_enter(chudxnu_syscall_callback_func_t func)
-{
-	syscall_callback_fn = func;
-	PPCcalls[9] = chudxnu_private_syscall_callback;
-    __asm__ volatile("eieio");	/* force order */
-    __asm__ volatile("sync");	/* force to memory */
-    return KERN_SUCCESS;
-}
-
-__private_extern__
-kern_return_t chudxnu_syscall_callback_cancel(void)
-{
-	syscall_callback_fn = NULL;
-	PPCcalls[9] = NULL;
-    __asm__ volatile("eieio");	/* force order */
-    __asm__ volatile("sync");	/* force to memory */
-    return KERN_SUCCESS;
-}
-
-#pragma mark **** thread timer - DEPRECATED ****
-
-static thread_call_t thread_timer_call = NULL;
-static chudxnu_thread_timer_callback_func_t thread_timer_callback_fn = NULL;
-
-static void chudxnu_private_thread_timer_callback(thread_call_param_t param0, thread_call_param_t param1)
-{
-    if(thread_timer_call) {
-        thread_call_free(thread_timer_call);
-        thread_timer_call = NULL;
-
-        if(thread_timer_callback_fn) {
-            (thread_timer_callback_fn)((uint32_t)param0);
-        }
-    }
-}
-
-// DEPRECATED
-__private_extern__
-kern_return_t chudxnu_thread_timer_callback_enter(chudxnu_thread_timer_callback_func_t func, uint32_t param, uint32_t time, uint32_t units)
-{
-    if(!thread_timer_call) {
-        uint64_t t_delay;
-        thread_timer_callback_fn = func;
-        thread_timer_call = thread_call_allocate((thread_call_func_t)chudxnu_private_thread_timer_callback, (thread_call_param_t)param);
-        clock_interval_to_deadline(time, units, &t_delay);
-        thread_call_enter_delayed(thread_timer_call, t_delay);
-        return KERN_SUCCESS;
-    } else {
-        return KERN_FAILURE; // thread timer call already pending
-    }
-}
-
-// DEPRECATED
-__private_extern__
-kern_return_t chudxnu_thread_timer_callback_cancel(void)
-{
-    if(thread_timer_call) {
-    	thread_call_cancel(thread_timer_call);
-        thread_call_free(thread_timer_call);
-        thread_timer_call = NULL;
-    }
-    thread_timer_callback_fn = NULL;
-    return KERN_SUCCESS;
-}
diff --git a/osfmk/ppc/chud/chud_spr.h b/osfmk/chud/ppc/chud_spr.h
similarity index 100%
rename from osfmk/ppc/chud/chud_spr.h
rename to osfmk/chud/ppc/chud_spr.h
diff --git a/osfmk/ppc/chud/chud_thread.c b/osfmk/chud/ppc/chud_thread_ppc.c
similarity index 75%
rename from osfmk/ppc/chud/chud_thread.c
rename to osfmk/chud/ppc/chud_thread_ppc.c
index 585653203..b822fbeae 100644
--- a/osfmk/ppc/chud/chud_thread.c
+++ b/osfmk/chud/ppc/chud_thread_ppc.c
@@ -38,8 +38,8 @@
 #include <vm/vm_map.h>
 #include <vm/pmap.h>
 
-#include <ppc/chud/chud_xnu.h>
-#include <ppc/chud/chud_xnu_private.h>
+#include <chud/chud_xnu.h>
+#include <chud/chud_xnu_private.h>
 
 #include <ppc/misc_protos.h>
 #include <ppc/proc_reg.h>
@@ -53,29 +53,6 @@ extern kern_return_t machine_thread_get_kern_state( thread_t                thre
 													mach_msg_type_number_t  *count);
 
 
-#pragma mark **** thread binding ****
-
-__private_extern__
-kern_return_t chudxnu_bind_thread(thread_t thread, int cpu)
-{
-    if(cpu>=0 && cpu<chudxnu_avail_cpu_count()) { /* make sure cpu # is sane */
-        thread_bind(thread, cpu_to_processor(cpu));
-		if(thread==current_thread()) {
-			(void)thread_block(THREAD_CONTINUE_NULL);
-		}
-        return KERN_SUCCESS;
-    } else {
-        return KERN_FAILURE;
-    }
-}
-
-__private_extern__
-kern_return_t chudxnu_unbind_thread(thread_t thread)
-{
-    thread_bind(thread, PROCESSOR_NULL);
-    return KERN_SUCCESS;
-}
-
 #pragma mark **** thread state ****
 
 __private_extern__
@@ -516,7 +493,12 @@ kern_return_t chudxnu_kern_write(vm_offset_t dstaddr, void *srcaddr, vm_size_t s
 #define SUPERVISOR_MODE(msr) ((msr) & MASK(MSR_PR) ? FALSE : TRUE)
 #endif
 
-#define VALID_STACK_ADDRESS(addr)   (addr>=0x1000ULL && (addr&STACK_ALIGNMENT_MASK)==0x0 && (supervisor ? (addr>=kernStackMin && addr<=kernStackMax) : TRUE))
+#define VALID_STACK_ADDRESS(addr)   (addr>=0x1000ULL &&			 \
+				     (addr&STACK_ALIGNMENT_MASK)==0x0 && \
+				     (supervisor ?			 \
+					 (addr>=kernStackMin &&		 \
+					  addr<=kernStackMax) :		 \
+					 TRUE))
 
 
 __private_extern__
@@ -805,362 +787,6 @@ kern_return_t chudxnu_thread_get_callstack( thread_t thread,
     return KERN_SUCCESS;
 }
 
-#pragma mark **** task and thread info ****
-
-__private_extern__
-boolean_t chudxnu_is_64bit_task(task_t task)
-{
-	return (task_has_64BitAddr(task));
-}
-
-#define THING_TASK		0
-#define THING_THREAD	1
-
-// an exact copy of processor_set_things() except no mig conversion at the end!
-static kern_return_t chudxnu_private_processor_set_things(	processor_set_t	pset,
-															mach_port_t				**thing_list,
-															mach_msg_type_number_t	*count,
-															int						type)
-{
-	unsigned int actual;	/* this many things */
-	unsigned int maxthings;
-	unsigned int i;
-
-	vm_size_t size, size_needed;
-	void  *addr;
-
-	if (pset == PROCESSOR_SET_NULL)
-		return (KERN_INVALID_ARGUMENT);
-
-	size = 0; addr = 0;
-
-	for (;;) {
-		pset_lock(pset);
-		if (!pset->active) {
-			pset_unlock(pset);
-
-			return (KERN_FAILURE);
-		}
-
-		if (type == THING_TASK)
-			maxthings = pset->task_count;
-		else
-			maxthings = pset->thread_count;
-
-		/* do we have the memory we need? */
-
-		size_needed = maxthings * sizeof (mach_port_t);
-		if (size_needed <= size)
-			break;
-
-		/* unlock the pset and allocate more memory */
-		pset_unlock(pset);
-
-		if (size != 0)
-			kfree(addr, size);
-
-		assert(size_needed > 0);
-		size = size_needed;
-
-		addr = kalloc(size);
-		if (addr == 0)
-			return (KERN_RESOURCE_SHORTAGE);
-	}
-
-	/* OK, have memory and the processor_set is locked & active */
-
-	actual = 0;
-	switch (type) {
-
-	case THING_TASK:
-	{
-		task_t		task, *tasks = (task_t *)addr;
-
-		for (task = (task_t)queue_first(&pset->tasks);
-				!queue_end(&pset->tasks, (queue_entry_t)task);
-					task = (task_t)queue_next(&task->pset_tasks)) {
-			task_reference_internal(task);
-			tasks[actual++] = task;
-		}
-
-		break;
-	}
-
-	case THING_THREAD:
-	{
-		thread_t	thread, *threads = (thread_t *)addr;
-
-		for (i = 0, thread = (thread_t)queue_first(&pset->threads);
-				!queue_end(&pset->threads, (queue_entry_t)thread);
-					thread = (thread_t)queue_next(&thread->pset_threads)) {
-			thread_reference_internal(thread);
-			threads[actual++] = thread;
-		}
-
-		break;
-	}
-	}
-		
-	pset_unlock(pset);
-
-	if (actual < maxthings)
-		size_needed = actual * sizeof (mach_port_t);
-
-	if (actual == 0) {
-		/* no things, so return null pointer and deallocate memory */
-		*thing_list = 0;
-		*count = 0;
-
-		if (size != 0)
-			kfree(addr, size);
-	}
-	else {
-		/* if we allocated too much, must copy */
-
-		if (size_needed < size) {
-			void *newaddr;
-
-			newaddr = kalloc(size_needed);
-			if (newaddr == 0) {
-				switch (type) {
-
-				case THING_TASK:
-				{
-					task_t		*tasks = (task_t *)addr;
-
-					for (i = 0; i < actual; i++)
-						task_deallocate(tasks[i]);
-					break;
-				}
-
-				case THING_THREAD:
-				{
-					thread_t	*threads = (thread_t *)addr;
-
-					for (i = 0; i < actual; i++)
-						thread_deallocate(threads[i]);
-					break;
-				}
-				}
-
-				kfree(addr, size);
-				return (KERN_RESOURCE_SHORTAGE);
-			}
-
-			bcopy((void *) addr, (void *) newaddr, size_needed);
-			kfree(addr, size);
-			addr = newaddr;
-		}
-
-		*thing_list = (mach_port_t *)addr;
-		*count = actual;
-	}
-
-	return (KERN_SUCCESS);
-}
-
-// an exact copy of task_threads() except no mig conversion at the end!
-static kern_return_t chudxnu_private_task_threads(task_t	task,
-								    thread_act_array_t      *threads_out,
-    								mach_msg_type_number_t  *count)
-{
-	mach_msg_type_number_t	actual;
-	thread_t				*threads;
-	thread_t				thread;
-	vm_size_t				size, size_needed;
-	void					*addr;
-	unsigned int			i, j;
-
-	if (task == TASK_NULL)
-		return (KERN_INVALID_ARGUMENT);
-
-	size = 0; addr = 0;
-
-	for (;;) {
-		task_lock(task);
-		if (!task->active) {
-			task_unlock(task);
-
-			if (size != 0)
-				kfree(addr, size);
-
-			return (KERN_FAILURE);
-		}
-
-		actual = task->thread_count;
-
-		/* do we have the memory we need? */
-		size_needed = actual * sizeof (mach_port_t);
-		if (size_needed <= size)
-			break;
-
-		/* unlock the task and allocate more memory */
-		task_unlock(task);
-
-		if (size != 0)
-			kfree(addr, size);
-
-		assert(size_needed > 0);
-		size = size_needed;
-
-		addr = kalloc(size);
-		if (addr == 0)
-			return (KERN_RESOURCE_SHORTAGE);
-	}
-
-	/* OK, have memory and the task is locked & active */
-	threads = (thread_t *)addr;
-
-	i = j = 0;
-
-	for (thread = (thread_t)queue_first(&task->threads); i < actual;
-				++i, thread = (thread_t)queue_next(&thread->task_threads)) {
-		thread_reference_internal(thread);
-		threads[j++] = thread;
-	}
-
-	assert(queue_end(&task->threads, (queue_entry_t)thread));
-
-	actual = j;
-	size_needed = actual * sizeof (mach_port_t);
-
-	/* can unlock task now that we've got the thread refs */
-	task_unlock(task);
-
-	if (actual == 0) {
-		/* no threads, so return null pointer and deallocate memory */
-
-		*threads_out = 0;
-		*count = 0;
-
-		if (size != 0)
-			kfree(addr, size);
-	}
-	else {
-		/* if we allocated too much, must copy */
-
-		if (size_needed < size) {
-			void *newaddr;
-
-			newaddr = kalloc(size_needed);
-			if (newaddr == 0) {
-				for (i = 0; i < actual; ++i)
-					thread_deallocate(threads[i]);
-				kfree(addr, size);
-				return (KERN_RESOURCE_SHORTAGE);
-			}
-
-			bcopy(addr, newaddr, size_needed);
-			kfree(addr, size);
-			threads = (thread_t *)newaddr;
-		}
-
-		*threads_out = threads;
-		*count = actual;
-	}
-
-	return (KERN_SUCCESS);
-}
-
-
-__private_extern__
-kern_return_t chudxnu_all_tasks(task_array_t		*task_list,
-								mach_msg_type_number_t	*count)
-{
-	return chudxnu_private_processor_set_things(&default_pset, (mach_port_t **)task_list, count, THING_TASK);	
-}
-
-__private_extern__
-kern_return_t chudxnu_free_task_list(task_array_t	*task_list,
-									 mach_msg_type_number_t	*count)
-{
-	vm_size_t size = (*count)*sizeof(mach_port_t);
-	void *addr = *task_list;
-
-	if(addr) {
-		int i, maxCount = *count;
-		for(i=0; i<maxCount; i++) {
-			task_deallocate((*task_list)[i]);
-		}		
-		kfree(addr, size);
-		*task_list = NULL;
-		*count = 0;
-		return KERN_SUCCESS;
-	} else {
-		return KERN_FAILURE;
-	}
-}
-
-__private_extern__
-kern_return_t chudxnu_all_threads(	thread_array_t		*thread_list,
-									mach_msg_type_number_t	*count)
-{
-	return chudxnu_private_processor_set_things(&default_pset, (mach_port_t **)thread_list, count, THING_THREAD);
-}
-
-__private_extern__
-kern_return_t chudxnu_task_threads(	task_t task,
-			 						thread_array_t *thread_list,
-			 						mach_msg_type_number_t *count)
-{
-	return chudxnu_private_task_threads(task, thread_list, count);
-}
-
-__private_extern__
-kern_return_t chudxnu_free_thread_list(thread_array_t	*thread_list,
-									 mach_msg_type_number_t	*count)
-{
-	vm_size_t size = (*count)*sizeof(mach_port_t);
-	void *addr = *thread_list;
-
-	if(addr) {
-		int i, maxCount = *count;
-		for(i=0; i<maxCount; i++) {
-			thread_deallocate((*thread_list)[i]);
-		}		
-		kfree(addr, size);
-		*thread_list = NULL;
-		*count = 0;
-		return KERN_SUCCESS;
-	} else {
-		return KERN_FAILURE;
-	}
-}
-
-__private_extern__
-task_t chudxnu_current_task(void)
-{
-	return current_task();
-}
-
-__private_extern__
-thread_t chudxnu_current_thread(void)
-{
-	return current_thread();
-}
-
-__private_extern__
-task_t chudxnu_task_for_thread(thread_t thread)
-{
-    return get_threadtask(thread);
-}
-
-__private_extern__
-kern_return_t chudxnu_thread_info(thread_t thread,
-        						thread_flavor_t flavor,
-        						thread_info_t thread_info_out,
-        						mach_msg_type_number_t *thread_info_count)
-{
-	return thread_info(thread, flavor, thread_info_out, thread_info_count);
-}
-
-__private_extern__
-kern_return_t chudxnu_thread_last_context_switch(thread_t thread, uint64_t *timestamp)
-{
-    *timestamp = thread->last_switch;
-    return KERN_SUCCESS;
-}
-
 #pragma mark **** DEPRECATED ****
 
 // DEPRECATED
diff --git a/osfmk/chud/ppc/chud_xnu_glue.h b/osfmk/chud/ppc/chud_xnu_glue.h
new file mode 100644
index 000000000..7bc509e16
--- /dev/null
+++ b/osfmk/chud/ppc/chud_xnu_glue.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
diff --git a/osfmk/ppc/chud/chud_xnu_private.h b/osfmk/chud/ppc/chud_xnu_private.h
similarity index 71%
rename from osfmk/ppc/chud/chud_xnu_private.h
rename to osfmk/chud/ppc/chud_xnu_private.h
index aef3d761a..46b563af5 100644
--- a/osfmk/ppc/chud/chud_xnu_private.h
+++ b/osfmk/chud/ppc/chud_xnu_private.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -29,22 +29,27 @@
 #ifndef _PPC_CHUD_XNU_PRIVATE_H_
 #define _PPC_CHUD_XNU_PRIVATE_H_
 
-#include <stdint.h>
-#include <mach/boolean.h>
-#include <mach/mach_types.h>
-
 #pragma mark **** thread ****
-// ********************************************************************************
+// *****************************************************************************
 // thread
-// ********************************************************************************
-extern kern_return_t chudxnu_copy_savearea_to_threadstate(  thread_flavor_t flavor, 
-							    thread_state_t tstate,
-							    mach_msg_type_number_t *count,
-							    struct savearea *sv);
+// *****************************************************************************
+extern kern_return_t chudxnu_copy_savearea_to_threadstate(
+			thread_flavor_t flavor, 
+			thread_state_t tstate,
+			mach_msg_type_number_t *count,
+			struct savearea *sv);
 							    
-extern kern_return_t chudxnu_copy_threadstate_to_savearea(  struct savearea *sv,
-							    thread_flavor_t flavor,
-							    thread_state_t tstate,
-							    mach_msg_type_number_t *count);
+extern kern_return_t chudxnu_copy_threadstate_to_savearea(
+			struct savearea *sv,
+			thread_flavor_t flavor,
+			thread_state_t tstate,
+			mach_msg_type_number_t *count);
+
+#pragma mark **** cpu timer ****
+typedef struct {
+	timer_call_data_t			cpu_timer_call;
+	uint64_t				t_deadline;
+	chudxnu_cpu_timer_callback_func_t	cpu_timer_callback_fn;
+} chudcpu_data_t;
 
 #endif /* _PPC_CHUD_XNU_PRIVATE_H_ */
diff --git a/osfmk/conf/MASTER.i386 b/osfmk/conf/MASTER.i386
index 1dc361d30..1e9f170b6 100644
--- a/osfmk/conf/MASTER.i386
+++ b/osfmk/conf/MASTER.i386
@@ -12,7 +12,7 @@
 #  osfmk = [intel pc mach small event vol pst gdb fixpri simple_clock mkernserv uxpr kernstack ipc_compat ipc_debug mk30 mk30_i386]
 #  RELEASE = [intel pc iokit mach_pe mach mach_kdp small event vol hd pst gdb fixpri simple_clock mkernserv uxpr kernstack ipc_compat ipc_debug fb mk30 mk30_i386]
 #  DEBUG_KDP = [intel pc iokit mach_pe mach mach_kdp small event vol hd pst gdb fixpri simple_clock mkernserv uxpr kernstack ipc_compat ipc_debug fb mk30 mk30_i386 osf_debug debug]
-#  DEBUG= [intel pc iokit mach_pe mach mach_kdp small event vol hd pst gdb fixpri simple_clock mkernserv uxpr kernstack ipc_compat ipc_debug fb mk30 mk30_i386 osf_debug debug]
+#  DEBUG= [intel pc iokit mach_pe mach mach_kdp small event vol hd pst gdb fixpri simple_clock mkernserv uxpr kernstack ipc_compat ipc_debug fb mk30 mk30_i386 osf_debug debug mach_kdb]
 #  PROFILE = [ RELEASE profile ]
 #
 ######################################################################
@@ -48,7 +48,8 @@ options		MACH_BSD
 options         IOKIT		#				# <iokit>
 options         MACH_PE		#				# <mach_pe>
 
-#options	DDB		# Inline debugger		# <debug>
+options		DDB		# Inline debugger		# <debug>
+options		MACH_KDB	#				# <mach_kdb>
 options		MACH_KDP	# KDP				# <mach_kdp>
-#options		PAE
-
+options		PAE
+options		X86_64
diff --git a/osfmk/conf/Makefile b/osfmk/conf/Makefile
index 38fc57eba..212081efc 100644
--- a/osfmk/conf/Makefile
+++ b/osfmk/conf/Makefile
@@ -90,8 +90,8 @@ export ipc_table.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export ipc_table.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export mach_debug.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export mach_debug.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export mach_msg.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export mach_msg.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export mach_msg.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export mach_msg.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export mach_port.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export mach_port.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export mig_log.o_CFLAGS_RM=$(CWARNFLAGS_STD)
@@ -100,24 +100,24 @@ export ipc_clock.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export ipc_clock.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export ipc_host.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export ipc_host.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_kobject.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_kobject.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export ipc_kobject.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export ipc_kobject.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export ipc_mig.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export ipc_mig.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export ipc_sync.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export ipc_sync.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_tt.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_tt.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export sync_lock.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export sync_lock.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export ipc_tt.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export ipc_tt.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export sync_lock.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export sync_lock.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export sync_sema.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export sync_sema.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export mach_port_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export mach_port_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export lock_set_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export lock_set_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export semaphore_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export semaphore_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export mach_port_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export mach_port_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export lock_set_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export lock_set_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export semaphore_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export semaphore_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 
 #
 # kern should be warning free (almost)
@@ -157,86 +157,86 @@ export mach_clock.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export mach_clock.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export mach_factor.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export mach_factor.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export machine.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export machine.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export machine.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export machine.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export mk_timer.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export mk_timer.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export profile.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export profile.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export priority.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export priority.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export processor.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export processor.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export processor.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export processor.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export processor_data.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export processor_data.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export queue.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export queue.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export sched_prim.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export sched_prim.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export sched_prim.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export sched_prim.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export sscanf.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export sscanf.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export stack.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export stack.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export stack.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export stack.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export startup.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export startup.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export syscall_subr.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export syscall_subr.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export syscall_subr.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export syscall_subr.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export syscall_sw.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export syscall_sw.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export task.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export task.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export task.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export task.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export task_policy.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export task_policy.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export task_swap.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export task_swap.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export thread.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export thread.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export thread.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export thread.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export thread_act.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export thread_act.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export thread_call.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export thread_call.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export thread_call.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export thread_call.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export thread_policy.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export thread_policy.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export thread_swap.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export thread_swap.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export timer.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export timer.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export timer_call.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export timer_call.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export wait_queue.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export wait_queue.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export timer_call.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export timer_call.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export wait_queue.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export wait_queue.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export zalloc.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export zalloc.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export clock_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export clock_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export clock_priv_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export clock_priv_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export clock_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export clock_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export clock_priv_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export clock_priv_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export clock_reply_user.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export clock_reply_user.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export exc_user.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export exc_user.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export exc_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export exc_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export host_priv_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export host_priv_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export host_security_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export host_security_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ledger_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ledger_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export mach_host_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export mach_host_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export exc_user.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export exc_user.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export exc_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export exc_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export host_priv_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export host_priv_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export host_security_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export host_security_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export ledger_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export ledger_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export mach_host_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export mach_host_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export mach_notify_user.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export mach_notify_user.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export processor_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export processor_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export processor_set_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export processor_set_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export processor_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export processor_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export processor_set_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export processor_set_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 export prof_user.o_CFLAGS_RM=$(CWARNFLAGS_STD)
 export prof_user.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export task_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export task_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export thread_act_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export thread_act_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export task_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export task_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
+#export thread_act_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
+#export thread_act_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
 
 include $(MakeInc_cmd)
 include $(MakeInc_def)
diff --git a/osfmk/conf/Makefile.i386 b/osfmk/conf/Makefile.i386
index e7e996b62..f6c752838 100644
--- a/osfmk/conf/Makefile.i386
+++ b/osfmk/conf/Makefile.i386
@@ -11,6 +11,8 @@ CWARNFLAGS= $(filter-out -Wbad-function-cast, $(CWARNFLAGS_STD))
 
 # Objects that don't compile cleanly:
 OBJS_NO_WERROR=				\
+	pms.o				\
+	etimer.o			\
 	ioconf.o			\
 	UNDRequest.o			\
 	KUNCUserNotifications.o		\
@@ -28,7 +30,84 @@ OBJS_NO_WERROR=				\
 	mk_sp.o				\
 	printf.o			\
 	syscall_emulation.o		\
-	mach_header.o
+	UNDReplyServer.o	\
+	ipc_kobject.o		\
+	ipc_tt.o			\
+	machine.o			\
+	processor.o			\
+	stack.o				\
+	sched_prim.o		\
+	sync_lock.o			\
+	task.o				\
+	syscall_subr.o		\
+	thread.o			\
+	thread_call.o		\
+	timer_call.o		\
+	hibernate.o			\
+	clock_server.o		\
+	clock_priv_server.o	\
+	exc_user.o			\
+	wait_queue.o		\
+	exc_server.o		\
+	host_priv_server.o	\
+	host_priv_server.o	\
+	ledger_server.o		\
+	host_security_server.o \
+	lock_set_server.o	\
+	mach_host_server.o	\
+	mach_port_server.o	\
+	processor_server.o	\
+	processor_set_server.o \
+	semaphore_server.o	\
+	task_server.o		\
+	thread_act_server.o	\
+	device_server.o		\
+	pmap.o				\
+	cpuid.o				\
+	bsd_i386.o			\
+	loose_ends.o		\
+	mp_desc.o			\
+	perfmon.o			\
+	pcb.o				\
+	rtclock.o			\
+	trap.o				\
+	bbclock.o			\
+	mp.o				\
+	acpi.o				\
+	serial_console.o	\
+	text_console.o		\
+	mtrr.o				\
+	hibernate_i386.o	\
+	mach_msg.o	\
+	mach_header.o \
+	chud_osfmk_callback_i386.o \
+	chud_thread.o 		   \
+	chud_thread_i386.o 	   \
+	chud_cpu.o 		   \
+	db_access.o 		   \
+	db_aout.o		   \
+	db_break.o 		   \
+	db_command.o 		   \
+	db_cond.o 		   \
+	db_disasm.o		   \
+	db_examine.o 		   \
+	db_expr.o 		   \
+	db_ext_symtab.o 	   \
+	db_input.o 		   \
+	db_interface.o		   \
+	db_lex.o 		   \
+	db_macro.o 		   \
+	db_output.o 		   \
+	db_print.o 		   \
+	db_run.o 		   \
+	db_sym.o 		   \
+	db_task_thread.o 	   \
+	db_trace.o		   \
+	db_trap.o 		   \
+	db_variables.o 		   \
+	db_watch.o 		   \
+	db_write_cmd.o		   \
+	xpr.o
 
 OBJS_WERROR=$(filter-out $(OBJS_NO_WERROR),$(OBJS))
 
diff --git a/osfmk/conf/files b/osfmk/conf/files
index 3a2b613e0..e9738b439 100644
--- a/osfmk/conf/files
+++ b/osfmk/conf/files
@@ -129,8 +129,10 @@ osfmk/ipc/mach_port.c			standard
 osfmk/ipc/mig_log.c			optional mig_debug
 osfmk/kern/ast.c			standard
 osfmk/kern/clock.c			standard
+osfmk/kern/clock_oldops.c		standard
 osfmk/kern/counters.c			standard
 osfmk/kern/debug.c			standard
+osfmk/kern/etimer.c			standard
 osfmk/kern/exception.c		standard
 osfmk/kern/host.c			standard
 osfmk/kern/host_notify.c		standard
@@ -147,6 +149,8 @@ osfmk/kern/mach_clock.c		standard
 osfmk/kern/machine.c			standard
 osfmk/kern/mk_sp.c			standard
 osfmk/kern/mk_timer.c		standard
+osfmk/kern/pms.c			standard
+osfmk/kern/page_decrypt.c	standard
 osfmk/kern/profile.c			standard
 osfmk/kern/printf.c			standard
 osfmk/kern/priority.c			standard
@@ -241,3 +245,14 @@ osfmk/kern/kmod.c			standard
 # funcs).
 #
 osfmk/device/subrs.c			standard
+
+#
+# MI CHUD:
+#
+osfmk/chud/chud_cpu.c			standard
+osfmk/chud/chud_glue.c			standard
+osfmk/chud/chud_memory.c		standard
+osfmk/chud/chud_osfmk_callback.c	standard
+osfmk/chud/chud_thread.c		standard
+
+osfmk/console/serial_general.c	standard
diff --git a/osfmk/conf/files.i386 b/osfmk/conf/files.i386
index 1121b7b74..2a3399191 100644
--- a/osfmk/conf/files.i386
+++ b/osfmk/conf/files.i386
@@ -16,10 +16,11 @@ OPTIONS/dynamic_num_nodes	optional dynamic_num_nodes
 OPTIONS/vtoc_compat	optional vtoc_compat
 OPTIONS/fddi		optional fddi
 
+osfmk/vm/vm_apple_protect.c	 standard
+
 osfmk/i386/hi_res_clock_map.c 	optional hi_res_clock
 
 osfmk/i386/pmap.c		standard
-osfmk/i386/read_fault.c	standard
 
 
 osfmk/ddb/db_aout.c		optional mach_kdb
@@ -53,9 +54,11 @@ osfmk/i386/loose_ends.c	standard
 osfmk/i386/locks_i386.c	standard
 osfmk/i386/locore.s	standard
 osfmk/i386/start.s	standard
+osfmk/i386/lowmem_vectors.s	standard
 osfmk/i386/cswitch.s	standard
 osfmk/i386/machine_routines.c		standard
 osfmk/i386/machine_routines_asm.s	standard
+osfmk/i386/machine_check.c	standard
 osfmk/i386/mcount.s		optional profile
 osfmk/i386/mp_desc.c		standard
 osfmk/i386/ntoh.s		standard
@@ -65,8 +68,13 @@ osfmk/i386/phys.c		standard
 osfmk/i386/rtclock.c		standard
 osfmk/i386/trap.c		standard
 osfmk/i386/user_ldt.c		standard
+osfmk/i386/Diagnostics.c	standard
+osfmk/i386/pmCPU.c		standard
+osfmk/i386/hpet.c		standard
+osfmk/i386/tsc.c		standard
 
 osfmk/i386/commpage/commpage.c	standard
+osfmk/i386/commpage/commpage_asm.s	standard
 osfmk/i386/commpage/atomic.s	standard
 osfmk/i386/commpage/commpage_mach_absolute_time.s	standard
 osfmk/i386/commpage/spinlocks.s	standard
@@ -74,12 +82,20 @@ osfmk/i386/commpage/pthreads.s	standard
 osfmk/i386/commpage/cacheflush.s	standard
 osfmk/i386/commpage/commpage_gettimeofday.s	standard
 osfmk/i386/commpage/bcopy_scalar.s	standard
+osfmk/i386/commpage/bcopy_sse3.s	standard
+osfmk/i386/commpage/bcopy_sse4.s	standard
+osfmk/i386/commpage/bcopy_sse4_64.s	standard
 osfmk/i386/commpage/bzero_scalar.s	standard
+osfmk/i386/commpage/bzero_sse3.s	standard
+osfmk/i386/commpage/bzero_sse3_64.s	standard
+osfmk/i386/commpage/memset_pattern_sse3.s	standard
+osfmk/i386/commpage/memset_pattern_sse3_64.s	standard
+osfmk/i386/commpage/longcopy_sse4.s	standard
+osfmk/i386/commpage/longcopy_sse4_64.s	standard
+osfmk/i386/commpage/commpage_sigs.c	standard
 
 osfmk/i386/AT386/autoconf.c	standard
-osfmk/i386/AT386/bbclock.c	standard
 osfmk/i386/AT386/conf.c		standard
-osfmk/i386/AT386/himem.c	optional himem
 osfmk/i386/AT386/model_dep.c	standard
 osfmk/i386/AT386/physmem.c	optional physmem device-driver
 
@@ -114,6 +130,10 @@ osfmk/kdp/ml/i386/kdp_vm.c	optional	mach_kdp
 osfmk/i386/hibernate_i386.c		standard
 osfmk/i386/hibernate_restore.s		standard
 
+osfmk/chud/i386/chud_osfmk_callback_i386.c	standard
+osfmk/chud/i386/chud_cpu_i386.c			standard
+osfmk/chud/i386/chud_thread_i386.c		standard
+
 # DUMMIES TO FORCE GENERATION OF .h FILES
 osfmk/OPTIONS/ln		optional ln
 osfmk/OPTIONS/eisa		optional eisa
@@ -121,3 +141,7 @@ osfmk/OPTIONS/himem		optional himem
 osfmk/OPTIONS/ec		optional ec
 osfmk/OPTIONS/hi_res_clock	optional hi_res_clock
 
+
+osfmk/i386/startup64.c			optional x86_64
+osfmk/i386/start64.s			optional x86_64
+osfmk/i386/idt64.s			optional x86_64
diff --git a/osfmk/conf/files.ppc b/osfmk/conf/files.ppc
index e03ce61b9..ffc13d1d8 100644
--- a/osfmk/conf/files.ppc
+++ b/osfmk/conf/files.ppc
@@ -64,7 +64,6 @@ osfmk/ppc/Diagnostics.c		standard
 osfmk/ppc/PPCcalls.c		standard
 osfmk/ppc/vmachmon.c		standard
 osfmk/ppc/vmachmon_asm.s	standard
-osfmk/ppc/pms.c				standard
 osfmk/ppc/pmsCPU.c			standard
 
 osfmk/ppc/Firmware.s		standard
@@ -94,12 +93,10 @@ osfmk/ppc/commpage/memset_g3.s  standard
 osfmk/ppc/commpage/memset_g4.s  standard
 osfmk/ppc/commpage/memset_g5.s  standard
 
-osfmk/ppc/chud/chud_osfmk_callback.c	standard
-osfmk/ppc/chud/chud_cpu.c		standard
-osfmk/ppc/chud/chud_cpu_asm.s	standard
-osfmk/ppc/chud/chud_memory.c	standard
-osfmk/ppc/chud/chud_thread.c	standard
-osfmk/ppc/chud/chud_glue.c		standard
+osfmk/chud/ppc/chud_cpu_asm.s			standard
+osfmk/chud/ppc/chud_cpu_ppc.c			standard
+osfmk/chud/ppc/chud_osfmk_callback_ppc.c	standard
+osfmk/chud/ppc/chud_thread_ppc.c		standard
 
 osfmk/kdp/ml/ppc/kdp_machdep.c        optional mach_kdp
 osfmk/kdp/ml/ppc/kdp_vm.c             optional mach_kdp
diff --git a/osfmk/console/i386/serial_console.c b/osfmk/console/i386/serial_console.c
index a4d0b5376..5a1c8d424 100644
--- a/osfmk/console/i386/serial_console.c
+++ b/osfmk/console/i386/serial_console.c
@@ -33,6 +33,7 @@
 #include <i386/misc_protos.h>
 #include <vm/vm_kern.h>
 #include <console/video_console.h>
+#include <console/serial_protos.h>
 #include <kern/kalloc.h>
 
 static struct {
@@ -45,6 +46,9 @@ static struct {
 	decl_simple_lock_data(,write_lock);
 } console_ring;
 
+hw_lock_data_t cnputc_lock;
+static volatile long console_output = 0;
+
 typedef struct console_buf {
 	char	*buf_base;
 	char	*buf_end;
@@ -53,6 +57,35 @@ typedef struct console_buf {
 	char	buf[CPU_BUFFER_LEN];
 } console_buf_t;
 
+extern int serial_getc(void);
+extern void serial_putc(int);
+
+static void _serial_putc(int, int, int);
+
+int vcgetc(int, int, boolean_t, boolean_t);
+
+console_ops_t cons_ops[] = {
+    {_serial_putc, _serial_getc},
+    {vcputc, vcgetc}
+};
+
+uint32_t nconsops = (sizeof cons_ops / sizeof cons_ops[0]);
+
+uint32_t cons_ops_index = VC_CONS_OPS;
+
+/* This macro polls for pending TLB flushes while spinning on a lock
+ */
+#define SIMPLE_LOCK_NO_INTRS(l)				\
+MACRO_BEGIN						\
+	boolean_t istate = ml_get_interrupts_enabled();	\
+	while (!simple_lock_try((l)))			\
+	{						\
+		if (!istate)				\
+			handle_pending_TLB_flushes();	\
+		cpu_pause();				\
+	}						\
+MACRO_END
+
 void
 console_init(void)
 {
@@ -69,7 +102,7 @@ console_init(void)
 	console_ring.write_ptr = console_ring.buffer;
 	simple_lock_init(&console_ring.read_lock, 0);
 	simple_lock_init(&console_ring.write_lock, 0);
-
+	hw_lock_init(&cnputc_lock);
 }
 
 void *
@@ -100,6 +133,12 @@ console_cpu_free(void *buf)
 		kfree((void *) buf, sizeof(console_buf_t));
 }
 
+static inline int
+console_ring_space(void)
+{
+	return console_ring.len - console_ring.used;
+}
+
 static boolean_t
 console_ring_put(char ch)
 {
@@ -133,23 +172,83 @@ console_ring_get(void)
 static inline void
 cpu_buffer_put(console_buf_t *cbp, char ch)
 {
-	if (cbp->buf_ptr < cbp->buf_end)
+	if (ch != '\0' && cbp->buf_ptr < cbp->buf_end)
 		*(cbp->buf_ptr++) = ch;
 }
 
 static inline void
 _cnputc(char c)
 {
-	vcputc(0, 0, c);
+	/* The console device output routines are assumed to be
+	 * non-reentrant.
+	 */
+	mp_disable_preemption();
+	if (!hw_lock_to(&cnputc_lock, LockTimeOut*10)) {
+	/* If we timed out on the lock, and we're in the debugger,
+	 * break the lock.
+	 */
+		if (debug_mode) {
+			/* Since hw_lock_to takes a pre-emption count...*/
+			mp_enable_preemption(); 
+			hw_lock_init(&cnputc_lock);
+			hw_lock_lock(&cnputc_lock);
+		}
+		else
+			panic("Lock acquire timeout in _cnputc()");
+	}
+	cons_ops[cons_ops_index].putc(0, 0, c);
 	if (c == '\n')
-		vcputc(0, 0,'\r');
+            cons_ops[cons_ops_index].putc(0, 0, '\r');
+	hw_lock_unlock(&cnputc_lock);
+	mp_enable_preemption();
 }
 
 void
 cnputcusr(char c)
-{	
-	simple_lock(&console_ring.read_lock);
+{
+	/* Spin (with pre-emption enabled) waiting for console_ring_try_empty()
+	 * to complete output. There is a small window here where we could
+	 * end up with a stale value of console_output, but it's unlikely,
+	 * and _cnputc(), which outputs to the console device, is internally
+	 * synchronized. There's something of a conflict between the
+	 * character-at-a-time (with pre-emption enabled) unbuffered
+	 * output model here, and the buffered output from cnputc(),
+	 * whose consumers include printf() ( which outputs a sequence
+	 * with pre-emption disabled, and should be safe to call with
+	 * interrupts off); we don't want to disable pre-emption indefinitely
+	 * here, and spinlocks and mutexes are inappropriate.
+	 */
+	while (console_output != 0);
+
 	_cnputc(c);
+}
+
+static void
+console_ring_try_empty(void)
+{
+	boolean_t state = ml_get_interrupts_enabled();
+	/*
+	 * Try to get the read lock on the ring buffer to empty it.
+	 * If this fails someone else is already emptying...
+	 */
+	if (!simple_lock_try(&console_ring.read_lock))
+		return;
+	/* Indicate that we're in the process of writing a block of data
+	 * to the console.
+	 */
+	atomic_incl(&console_output, 1);
+	for (;;) {
+		char	ch;
+		if (!state)
+			handle_pending_TLB_flushes();
+   	 	SIMPLE_LOCK_NO_INTRS(&console_ring.write_lock);
+		ch = console_ring_get();
+    		simple_unlock(&console_ring.write_lock);
+		if (ch == 0)
+			break;
+		_cnputc(ch);
+	}
+	atomic_decl(&console_output, 1);
 	simple_unlock(&console_ring.read_lock);
 }
 
@@ -157,16 +256,29 @@ void
 cnputc(char c)
 {
 	console_buf_t	*cbp;
-
-	if (!(real_ncpus > 1)) {
+#if MACH_KDB
+	/* Bypass locking/buffering if in debugger */
+	if (kdb_cpu == cpu_number()) {
 		_cnputc(c);
 		return;
 	}
-
+#endif /* MACH_KDB */	
 	mp_disable_preemption();
-	/* add to stack buf */
 	cbp = (console_buf_t *) current_cpu_datap()->cpu_console_buf;
+	if (cbp == NULL) {
+		mp_enable_preemption();
+		/* Put directly if console ring is not initialized */
+		_cnputc(c);
+		return;
+	}
+
+	/* add to stack buf */
 	if (c != '\n') {
+		/* XXX - cpu_buffer_put() can fail silently if the buffer
+		 * is exhausted, as can happen if there's a long sequence
+		 * of data with no newlines. We should, instead, attempt
+		 * a flush.
+		 */
 		cpu_buffer_put(cbp, c);
 	} else {
 		boolean_t	state;
@@ -176,34 +288,70 @@ cnputc(char c)
 	
 		/* copy this buffer into the shared ring buffer */
 		state = ml_set_interrupts_enabled(FALSE);
-		simple_lock(&console_ring.write_lock);
-		for (cp = cbp->buf_base; cp < cbp->buf_ptr; cp++) {
-			while (!console_ring_put(*cp))
-				/* spin if share buffer full */
-				cpu_pause();
-		}
-		(void) console_ring_put('\n');
-		simple_unlock(&console_ring.write_lock);
-		ml_set_interrupts_enabled(state);
-		cbp->buf_ptr = cbp->buf_base;
+		SIMPLE_LOCK_NO_INTRS(&console_ring.write_lock);
 
 		/*
-		 * Try to get the read lock on the ring buffer to empty it.
-		 * If this fails someone else is already emptying...
+		 * Is there enough space in the shared ring buffer?
+		 * Try to empty if not.
+		 * Note, we want the entire local buffer to fit to
+		 * avoid another cpu interjecting.
 		 */
-		if (simple_lock_try(&console_ring.read_lock)) {
-			for (;;) {
-				char	ch;
-
-		   	 	simple_lock(&console_ring.write_lock);
-				ch = console_ring_get();
-		    		simple_unlock(&console_ring.write_lock);
-				if (ch == 0)
-					break;
-				_cnputc(ch);
-			}
-			simple_unlock(&console_ring.read_lock);
+		while (cbp->buf_ptr-cbp->buf_base + 1 > console_ring_space()) {
+			simple_unlock(&console_ring.write_lock);
+			console_ring_try_empty();
+			SIMPLE_LOCK_NO_INTRS(&console_ring.write_lock);
 		}
+		for (cp = cbp->buf_base; cp < cbp->buf_ptr; cp++)
+			console_ring_put(*cp);
+		console_ring_put('\n');
+		cbp->buf_ptr = cbp->buf_base;
+		simple_unlock(&console_ring.write_lock);
+		ml_set_interrupts_enabled(state);
 	}
+	console_ring_try_empty();
 	mp_enable_preemption();
 }
+
+int _serial_getc(__unused int a, __unused int b, boolean_t wait, __unused boolean_t raw)
+{
+    int c;
+    do {
+        c = serial_getc();
+    } while (wait && c < 0);
+
+    return c;
+}
+
+static void _serial_putc(__unused int a, __unused int b, int c)
+{
+    serial_putc(c);
+}
+
+
+int
+cngetc(void)
+{
+	return cons_ops[cons_ops_index].getc(0, 0,
+					     TRUE, FALSE);
+}
+
+int
+cnmaygetc(void)
+{
+	return cons_ops[cons_ops_index].getc(0, 0,
+					     FALSE, FALSE);
+}
+
+int
+vcgetc(__unused int l, 
+       __unused int u, 
+       __unused boolean_t wait, 
+       __unused boolean_t raw)
+{
+	char c;
+
+	if( 0 == (*PE_poll_input)( 0, &c))
+		return( c);
+	else
+		return( 0);
+}
diff --git a/osfmk/console/i386/text_console.c b/osfmk/console/i386/text_console.c
index cca64edcc..ba0fc3392 100644
--- a/osfmk/console/i386/text_console.c
+++ b/osfmk/console/i386/text_console.c
@@ -32,7 +32,7 @@
  * VGA text console support.
  */
 
-#include <i386/pio.h>
+#include <architecture/i386/pio.h>
 #include <console/video_console.h>
 #include "text_console.h"
 
@@ -104,6 +104,7 @@ move_up( csrpos_t  from,
          csrpos_t  to,
          int       count)
 {
+    if (vram_start == 0) return;
     kd_slmscu( vram_start + from, vram_start + to, count );
 }
 
@@ -117,6 +118,7 @@ move_down( csrpos_t  from,
            csrpos_t  to,
            int       count )
 {
+    if (vram_start == 0) return;
     kd_slmscd( vram_start + from, vram_start + to, count );
 }
 
@@ -130,6 +132,7 @@ clear_block( csrpos_t  start,
              int       size,
              char      attr)
 {
+    if (vram_start == 0) return;
     kd_slmwd( vram_start + start, size,
               ((unsigned short) attr << 8) + SPACE_CHAR);
 }
@@ -177,6 +180,7 @@ display_char( csrpos_t    pos,      /* where to put it */
               char        ch,       /* the character */
               char        attr )    /* its attribute */
 {
+    if (vram_start == 0) return;
     *(vram_start + pos)     = ch;
     *(vram_start + pos + 1) = attr;
 }
diff --git a/osfmk/console/i386/text_console.h b/osfmk/console/i386/text_console.h
index 8ab084516..cb3c8e36e 100644
--- a/osfmk/console/i386/text_console.h
+++ b/osfmk/console/i386/text_console.h
@@ -29,8 +29,6 @@
 #ifndef _TEXT_CONSOLE_H_
 #define _TEXT_CONSOLE_H_
 
-#define TEXT_MODE 0
-
 void tc_paint_char(int x, int y, unsigned char ch, int attrs, unsigned char ch_previous, int attrs_previous);
 void tc_scroll_down(int lines, int top, int bottom);
 void tc_scroll_up(int lines, int top, int bottom);
diff --git a/osfmk/console/panic_dialog.c b/osfmk/console/panic_dialog.c
index 7776f2d46..cad977f8a 100644
--- a/osfmk/console/panic_dialog.c
+++ b/osfmk/console/panic_dialog.c
@@ -28,6 +28,7 @@
 
 #include <vc.h>
 #include <console/video_console.h>
+#include <libkern/OSByteOrder.h>
 #include <kdp/kdp_udp.h>
 #include <kern/debug.h>
 #include <mach/mach_time.h>
@@ -523,9 +524,9 @@ blit_digit( int digit )
 		for( j=FONT_WIDTH-1; j>=0; j--) {
 
 			if ( bits & 0x80 )
-				rendered_font[row][j] = 0x0100 | panic_dialog->pd_info_color[0];
+				rendered_font[row][j] = OSSwapBigToHostInt16(0x0100 | panic_dialog->pd_info_color[0]);
 			else
-				rendered_font[row][j] = 0x0100 | panic_dialog->pd_info_color[1];
+				rendered_font[row][j] = OSSwapBigToHostInt16(0x0100 | panic_dialog->pd_info_color[1]);
 			bits <<= 1;
 		}
 	}
@@ -884,16 +885,16 @@ findbestgray( unsigned int color24 )
 static unsigned char
 color24togray8( unsigned int color24 )
 {       
-    float R, G, B;
-    float Gray;
+    int R, G, B;
+    int Gray;
     unsigned char gray8;
     
     R = (color24 & 0xFF0000) >> 16 ;
     G = (color24 & 0xFF00) >> 8 ;
     B = (color24 & 0xFF);
     
-    Gray = (R*.30) + (G*.59) + (B*.11);
-    gray8 = (unsigned char) ( Gray + .5);
+    Gray = (R*30) + (G*59) + (B*11);
+    gray8 = (unsigned char) ((Gray + 50) / 100);
     return gray8;
 }       
 
diff --git a/osfmk/console/ppc/serial_console.c b/osfmk/console/ppc/serial_console.c
index 3488e1093..6ea518c61 100644
--- a/osfmk/console/ppc/serial_console.c
+++ b/osfmk/console/ppc/serial_console.c
@@ -40,6 +40,7 @@
 #include <machine/machparam.h>		/* spl definitions */
 #include <types.h>
 #include <console/video_console.h>
+#include <console/serial_protos.h>
 #include <kern/kalloc.h>
 #include <kern/thread.h>
 #include <ppc/misc_protos.h>
@@ -62,29 +63,9 @@
  */
 
 const int console_unit = 0;
-const int console_chan_default = CONSOLE_PORT;
+const uint32_t console_chan_default = CONSOLE_PORT;
 #define console_chan (console_chan_default) /* ^ cpu_number()) */
 
-#define OPS(putc, getc, nosplputc, nosplgetc) putc, getc
-
-const struct console_ops {
-	int	(*putc)(int, int, int);
-	int	(*getc)(int, int, boolean_t, boolean_t);
-} cons_ops[] = {
-#define SCC_CONS_OPS 0
-	{OPS(scc_putc, scc_getc, no_spl_scputc, no_spl_scgetc)},
-#define VC_CONS_OPS 1
-	{OPS(vcputc, vcgetc, no_spl_vcputc, no_spl_vcgetc)},
-};
-#define NCONSOPS (sizeof cons_ops / sizeof cons_ops[0])
-
-#if SERIAL_CONSOLE_DEFAULT
-#define CONS_OPS SCC_CONS_OPS
-#define CONS_NAME "com"
-#else
-#define CONS_OPS VC_CONS_OPS
-#define CONS_NAME "vc"
-#endif
 
 #define MP_SAFE_CONSOLE 1	/* Set this to 1 to allow more than 1 processor to print at once */
 #if MP_SAFE_CONSOLE
@@ -103,8 +84,17 @@ volatile unsigned int sconowner=-1;								/* Mark who's actually writing */
 
 #endif
 
+#define OPS(putc, getc, nosplputc, nosplgetc) putc, getc
+
+console_ops_t cons_ops[] = {
+	{OPS(scc_putc, scc_getc, no_spl_scputc, no_spl_scgetc)},
+	{OPS(vcputc, vcgetc, no_spl_vcputc, no_spl_vcgetc)},
+};
+
+uint32_t nconsops = (sizeof cons_ops / sizeof cons_ops[0]);
+
+uint32_t cons_ops_index = VC_CONS_OPS;
 
-unsigned int cons_ops_index = CONS_OPS;
 unsigned int killprint = 0;
 unsigned int debcnputc = 0;
 extern unsigned int	mappingdeb0;
@@ -319,43 +309,6 @@ cnmaygetc()
 					     FALSE, FALSE);
 }
 
-boolean_t console_is_serial()
-{
-	return cons_ops_index == SCC_CONS_OPS;
-}
-
-int
-switch_to_video_console()
-{
-	int old_cons_ops = cons_ops_index;
-	cons_ops_index = VC_CONS_OPS;
-	return old_cons_ops;
-}
-
-int
-switch_to_serial_console()
-{
-	int old_cons_ops = cons_ops_index;
-	cons_ops_index = SCC_CONS_OPS;
-	return old_cons_ops;
-}
-
-/* The switch_to_{video,serial,kgdb}_console functions return a cookie that
-   can be used to restore the console to whatever it was before, in the
-   same way that splwhatever() and splx() work.  */
-void
-switch_to_old_console(int old_console)
-{
-	static boolean_t squawked;
-	unsigned int ops = old_console;
-
-	if (ops >= NCONSOPS && !squawked) {
-		squawked = TRUE;
-		printf("switch_to_old_console: unknown ops %d\n", ops);
-	} else
-		cons_ops_index = ops;
-}
-
 
 int
 vcgetc(__unused int l, 
diff --git a/osfmk/console/serial_general.c b/osfmk/console/serial_general.c
new file mode 100644
index 000000000..d9876f79f
--- /dev/null
+++ b/osfmk/console/serial_general.c
@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+/*
+ * @APPLE_FREE_COPYRIGHT@
+ */
+
+#include <mach_kdb.h>
+#include <platforms.h>
+#include <kern/spl.h>
+#include <mach/std_types.h>
+#include <types.h>
+#include <kern/thread.h>
+#include <console/serial_protos.h>
+
+extern unsigned int disableSerialOuput;
+extern void cons_cinput(char ch);		/* The BSD routine that gets characters */
+
+unsigned int serialmode;				/* Serial mode keyboard and console control */
+
+/*
+ *  This routine will start a thread that polls the serial port, listening for
+ *  characters that have been typed.
+ */
+
+void
+serial_keyboard_init(void)
+{
+	kern_return_t	result;
+	thread_t		thread;
+
+	if(!(serialmode & 2)) return;		/* Leave if we do not want a serial console */
+
+	kprintf("Serial keyboard started\n");
+	result = kernel_thread_start_priority((thread_continue_t)serial_keyboard_start, NULL, MAXPRI_KERNEL, &thread);
+	if (result != KERN_SUCCESS)
+		panic("serial_keyboard_init");
+
+	thread_deallocate(thread);
+}
+
+void
+serial_keyboard_start(void)
+{
+
+	serial_keyboard_poll();				/* Go see if there are any characters pending now */
+	panic("serial_keyboard_start: we can't get back here\n");
+
+}
+
+void
+serial_keyboard_poll(void)
+{
+	int chr;
+	uint64_t next;
+
+
+	while(1) {							/* Do this for a while */
+		chr = _serial_getc(0, 1, 0, 1);	/* Get a character if there is one */
+		if(chr < 0) break;				/* The serial buffer is empty */
+		cons_cinput((char)chr);			/* Buffer up the character */
+	}
+
+	clock_interval_to_deadline(16, 1000000, &next);	/* Get time of pop */
+
+	assert_wait_deadline((event_t)serial_keyboard_poll, THREAD_UNINT, next);	/* Show we are "waiting" */
+	thread_block((thread_continue_t)serial_keyboard_poll);	/* Wait for it */
+	panic("serial_keyboard_poll: Shouldn't never ever get here...\n");
+}
+
+boolean_t console_is_serial()
+{
+	return cons_ops_index == SERIAL_CONS_OPS;
+}
+
+int
+switch_to_video_console()
+{
+	int old_cons_ops = cons_ops_index;
+	cons_ops_index = VC_CONS_OPS;
+	return old_cons_ops;
+}
+
+int
+switch_to_serial_console()
+{
+	int old_cons_ops = cons_ops_index;
+	cons_ops_index = SERIAL_CONS_OPS;
+	return old_cons_ops;
+}
+
+/* The switch_to_{video,serial,kgdb}_console functions return a cookie that
+   can be used to restore the console to whatever it was before, in the
+   same way that splwhatever() and splx() work.  */
+void
+switch_to_old_console(int old_console)
+{
+	static boolean_t squawked;
+	uint32_t ops = old_console;
+
+	if ((ops >= nconsops) && !squawked) {
+		squawked = TRUE;
+		printf("switch_to_old_console: unknown ops %d\n", ops);
+	} else
+		cons_ops_index = ops;
+}
diff --git a/osfmk/console/serial_protos.h b/osfmk/console/serial_protos.h
new file mode 100644
index 000000000..d5296a683
--- /dev/null
+++ b/osfmk/console/serial_protos.h
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+/*
+ * @APPLE_FREE_COPYRIGHT@
+ */
+
+
+
+void serial_keyboard_init(void);
+void serial_keyboard_start(void);
+void serial_keyboard_poll(void);
+
+extern uint32_t serialmode;
+extern uint32_t cons_ops_index;
+extern uint32_t nconsops;
+
+extern int _serial_getc(int unit, int line, boolean_t wait, boolean_t raw);
+
+extern boolean_t console_is_serial(void);
+extern int switch_to_serial_console(void);
+extern int switch_to_video_console(void);
+extern void	 switch_to_old_console(int old_console);
+
+struct console_ops {
+	int	(*putc)(int, int, int);
+	int	(*getc)(int, int, boolean_t, boolean_t);
+} console_ops;
+typedef struct console_ops console_ops_t;
+
+
+#define SERIAL_CONS_OPS 0
+#define VC_CONS_OPS 1
diff --git a/osfmk/console/video_console.c b/osfmk/console/video_console.c
index ff2180297..c9cd390d6 100644
--- a/osfmk/console/video_console.c
+++ b/osfmk/console/video_console.c
@@ -101,6 +101,7 @@
 
 #include <vm/pmap.h>
 #include <vm/vm_kern.h>
+#include <machine/io_map_entries.h>
 
 #include <pexpert/pexpert.h>
 
@@ -258,6 +259,8 @@ gc_clear_screen(int xx, int yy, int top, int bottom, int which)
 {
 	spl_t    s;
 
+        if (!gc_buffer_size) return;
+
 	s = splhigh();
 	simple_lock(&gc_buffer_lock);
 
@@ -992,6 +995,8 @@ gc_scroll_down(int num, int top, int bottom)
 {
 	spl_t s;
 
+        if (!gc_buffer_size) return;
+
 	s = splhigh();
 	simple_lock(&gc_buffer_lock);
 
@@ -1072,6 +1077,8 @@ gc_scroll_up(int num, int top, int bottom)
 {
 	spl_t s;
 
+        if (!gc_buffer_size) return;
+
 	s = splhigh();
 	simple_lock(&gc_buffer_lock);
 
@@ -2112,14 +2119,16 @@ static void vc_progress_task( void * arg0, void * arg )
 
 #ifdef __i386__
 #include <console/i386/text_console.h>
+#include <pexpert/i386/boot.h>
 #endif /* __i386__ */
 
 static boolean_t gc_acquired      = FALSE;
 static boolean_t gc_graphics_boot = FALSE;
 
-static unsigned int lastVideoPhys = 0;
-static unsigned int lastVideoVirt = 0;
-static unsigned int lastVideoSize = 0;
+static unsigned int lastVideoPhys   = 0;
+static unsigned int lastVideoVirt   = 0;
+static unsigned int lastVideoSize   = 0;
+static boolean_t    lastVideoMapped = FALSE;
 
 void
 initialize_screen(Boot_Video * boot_vinfo, unsigned int op)
@@ -2135,16 +2144,34 @@ initialize_screen(Boot_Video * boot_vinfo, unsigned int op)
 		/* 
 		 *	First, check if we are changing the size and/or location of the framebuffer
 		 */
-
 		vinfo.v_name[0]  = 0;
 		vinfo.v_width    = boot_vinfo->v_width;
 		vinfo.v_height   = boot_vinfo->v_height;
 		vinfo.v_depth    = boot_vinfo->v_depth;
 		vinfo.v_rowbytes = boot_vinfo->v_rowBytes;
 		vinfo.v_physaddr = boot_vinfo->v_baseAddr;		/* Get the physical address */
+#ifdef __i386__
+                vinfo.v_type     = boot_vinfo->v_display;
+#else
+                vinfo.v_type = 0;
+#endif
+    
  
-		kprintf("initialize_screen: b=%08X, w=%08X, h=%08X, r=%08X\n",			/* (BRINGUP) */
-			vinfo.v_physaddr, vinfo.v_width,  vinfo.v_height,  vinfo.v_rowbytes);	/* (BRINGUP) */
+                kprintf("initialize_screen: b=%08X, w=%08X, h=%08X, r=%08X, d=%08X\n",                  /* (BRINGUP) */
+                        vinfo.v_physaddr, vinfo.v_width,  vinfo.v_height,  vinfo.v_rowbytes, vinfo.v_type);     /* (BRINGUP) */
+
+#ifdef __i386__
+                if ( (vinfo.v_type == VGA_TEXT_MODE) )
+                {
+                    if (vinfo.v_physaddr == 0) {
+                        vinfo.v_physaddr = 0xb8000;
+                        vinfo.v_width = 80;
+                        vinfo.v_height = 25;
+                        vinfo.v_depth = 8;
+                        vinfo.v_rowbytes = 0x8000;
+                    }
+                }
+#endif /* __i386__ */
 
 		if (!vinfo.v_physaddr)							/* Check to see if we have a framebuffer */
 		{
@@ -2171,39 +2198,43 @@ initialize_screen(Boot_Video * boot_vinfo, unsigned int op)
 			    vinfo.v_physaddr = (fbppage << 12) | (boot_vinfo->v_baseAddr & PAGE_MASK);			/* Get the physical address */
 		    }
     
-#ifdef __i386__
-		    vinfo.v_type     = boot_vinfo->v_display;
-#else
-		    vinfo.v_type = 0;
-#endif
-    
 		    fbsize = round_page_32(vinfo.v_height * vinfo.v_rowbytes);			/* Remember size */
     
 		    if ((lastVideoPhys != vinfo.v_physaddr) || (fbsize > lastVideoSize))		/* Did framebuffer change location or get bigger? */
 		    {
-			    newVideoVirt = io_map_spec((vm_offset_t)vinfo.v_physaddr, fbsize);	/* Allocate address space for framebuffer */
+			    unsigned int
+#if FALSE
+			    flags = (vinfo.v_type == VGA_TEXT_MODE) ? VM_WIMG_IO : VM_WIMG_WCOMB;
+#else
+			    flags = VM_WIMG_IO;
+#endif
+			    newVideoVirt = io_map_spec((vm_offset_t)vinfo.v_physaddr, fbsize, flags);	/* Allocate address space for framebuffer */
     
 			    if (lastVideoVirt)							/* Was the framebuffer mapped before? */
 			    {
-				    pmap_remove(kernel_pmap, trunc_page_64(lastVideoVirt),
-						round_page_64(lastVideoVirt + lastVideoSize));	/* Toss mappings */
-
-                                    if(lastVideoVirt <= vm_last_addr)                            /* Was this not a special pre-VM mapping? */
+#if FALSE
+                                    if(lastVideoMapped)                            /* Was this not a special pre-VM mapping? */
+#endif
+				    {
+					    pmap_remove(kernel_pmap, trunc_page_64(lastVideoVirt),
+						    round_page_64(lastVideoVirt + lastVideoSize));	/* Toss mappings */
+				    }
+                                    if(lastVideoMapped)                            /* Was this not a special pre-VM mapping? */
 				    {
 					    kmem_free(kernel_map, lastVideoVirt, lastVideoSize);	/* Toss kernel addresses */
 				    }
 			    }
-    
 			    lastVideoPhys = vinfo.v_physaddr;					/* Remember the framebuffer address */
 			    lastVideoSize = fbsize;							/* Remember the size */
 			    lastVideoVirt = newVideoVirt;						/* Remember the virtual framebuffer address */
+			    lastVideoMapped  = (NULL != kernel_map);
 		    }
 		}
 
 		vinfo.v_baseaddr = lastVideoVirt;				/* Set the new framebuffer address */
 
 #ifdef __i386__
-		if ( (vinfo.v_type == TEXT_MODE) )
+		if ( (vinfo.v_type == VGA_TEXT_MODE) )
 		{
 			// Text mode setup by the booter.
 
diff --git a/osfmk/ddb/db_command.c b/osfmk/ddb/db_command.c
index ccbcc097b..d8d8b5f4b 100644
--- a/osfmk/ddb/db_command.c
+++ b/osfmk/ddb/db_command.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -165,8 +165,6 @@ void db_command_list(
 	char			*last_modifp,	/* IN_OUT */
 	struct db_command	*cmd_table);
 
-
-
 /*
  * Search for command prefix.
  */
@@ -510,20 +508,33 @@ struct db_command db_command_table[] = {
 	{ "dmacro",	(db_func) db_del_macro_cmd,	CS_OWN,			0 },
 	{ "show",	0,				0,			db_show_cmds },
 	{ "cpu",	(db_func) db_switch_cpu,	0,			0 },
+	{ "dr",		db_display_real,		CS_MORE|CS_SET_DOT,	0 },
+	{ "di",		db_display_iokit,		CS_MORE,			0 },
+	{ "dk",		db_display_kmod,		CS_MORE,			0 },
+
 	{ "reboot",	(db_func) db_reboot,		0,			0 },
-#if defined(__ppc__)
+#if !defined(__ppc__)	
+	{ "pm",		db_pmgr,				CS_MORE,			0 },
+	{ "na",		db_nap,					CS_MORE,			0 },
+	{ "ms",		db_msr,					CS_MORE,			0 },
+	{ "cp",		db_cpuid,				CS_MORE,			0 },
+	{ "da",		db_apic,				CS_MORE,			0 },
+	{ "ts",		db_test,				CS_MORE,			0 },
+	{ "dn",		db_intcnt,				CS_MORE,			0 },
+	{ "hp",		db_hpet,				CS_MORE,			0 },
+	{ "cf",		db_cfg,					CS_MORE,			0 },
+	{ "dt",		db_dtimers,				CS_MORE,			0 },
+#endif
+#if defined(__ppc__)	
 	{ "lt",		db_low_trace,			CS_MORE|CS_SET_DOT,	0 },
 	{ "dl",		db_display_long,		CS_MORE|CS_SET_DOT,	0 },
 	{ "dc",		db_display_char,		CS_MORE|CS_SET_DOT,	0 },
-	{ "dr",		db_display_real,		CS_MORE|CS_SET_DOT,	0 },
 	{ "dv",		db_display_virtual,		CS_MORE|CS_SET_DOT,	0 },
 	{ "dm",		db_display_mappings,	CS_MORE|CS_SET_DOT,	0 },
 	{ "dh",		db_display_hash,		CS_MORE|CS_SET_DOT,	0 },
 	{ "dp",		db_display_pmap,		CS_MORE,			0 },
-	{ "di",		db_display_iokit,		CS_MORE,			0 },
 	{ "ds",		db_display_save,		CS_MORE|CS_SET_DOT,	0 },
 	{ "dx",		db_display_xregs,		CS_MORE|CS_SET_DOT,	0 },
-	{ "dk",		db_display_kmod,		CS_MORE,			0 },
 	{ "gs",		db_gsnoop,				CS_MORE,			0 },
 	{ "cm",		db_check_mappings,		CS_MORE,			0 },
 	{ "cp",		db_check_pmaps,			CS_MORE,			0 },
diff --git a/osfmk/ddb/db_run.c b/osfmk/ddb/db_run.c
index 10a0a23ed..f5536443d 100644
--- a/osfmk/ddb/db_run.c
+++ b/osfmk/ddb/db_run.c
@@ -77,6 +77,8 @@
 #include <kern/misc_protos.h>
 #include <kern/debug.h>
 
+#include <IOKit/IOPlatformExpert.h>
+
 boolean_t	db_sstep_print;
 int		db_loop_count;
 int		db_call_depth;
@@ -510,6 +512,19 @@ db_continue_cmd(
 	db_cmd_loop_done = 1;
 }
 
+
+/*
+ * Switch to gdb
+ */
+void
+db_to_gdb(
+	void)
+{
+	extern unsigned int switch_debugger;
+
+	switch_debugger=1;
+}
+
 /* gdb */
 void    
 db_continue_gdb(
@@ -518,9 +533,7 @@ db_continue_gdb(
 	db_expr_t	count,   
 	char *		modif)
 {
-#if defined(__ppc__)
 	db_to_gdb();
-#endif
 	db_run_mode = STEP_CONTINUE;
 	db_inst_count = 0;
 	db_last_inst_count = 0;   
@@ -531,9 +544,9 @@ db_continue_gdb(
 }
         
 
-
 boolean_t
 db_in_single_step(void)
 {
 	return(db_run_mode != STEP_NONE && db_run_mode != STEP_CONTINUE);
 }
+
diff --git a/osfmk/ddb/db_task_thread.c b/osfmk/ddb/db_task_thread.c
index af04e4566..01eca330e 100644
--- a/osfmk/ddb/db_task_thread.c
+++ b/osfmk/ddb/db_task_thread.c
@@ -75,8 +75,8 @@
 #define DB_MAX_THREADID	0x10000		/* max # of threads in a task */
 #define DB_MAX_PSETS	0x10000		/* max # of processor sets */
 
-task_t		db_default_task;	/* default target task */
-thread_t	db_default_act;		/* default target thr_act */
+task_t		db_default_task = TASK_NULL;	/* default target task */
+thread_t	db_default_act = THREAD_NULL;		/* default target thr_act */
 
 
 
diff --git a/osfmk/ddb/db_trap.c b/osfmk/ddb/db_trap.c
index 3ad99f7da..642fedd39 100644
--- a/osfmk/ddb/db_trap.c
+++ b/osfmk/ddb/db_trap.c
@@ -70,7 +70,7 @@
 #include <ddb/db_break.h>
 #include <ddb/db_command.h>
 #include <ddb/db_examine.h>
-#include <ddb/db_output.h>		/* For db_printf() */
+#include <ddb/db_output.h>             /* For db_printf() */
 #include <ddb/db_run.h>
 #include <ddb/db_task_thread.h>
 #include <ddb/db_trap.h>
@@ -105,8 +105,16 @@ db_task_trap(
 	 * but print symbols using a (task-specific) symbol table, found
 	 * using task.
 	 */
+
+	/* Elided since walking the thread/task lists before setting up
+	 * safe recovery points is incorrect, and could
+	 * potentially cause us to loop and fault indefinitely.
+	 */
+#if 0	
 	db_init_default_act();
+#endif       
 	db_check_breakpoint_valid();
+
 	if (db_stop_at_pc(&bkpt, task, task_space)) {
 	    if (db_inst_count) {
 		db_printf("After %d instructions (%d loads, %d stores),\n",
diff --git a/osfmk/default_pager/default_pager.c b/osfmk/default_pager/default_pager.c
index 40b91d9de..fc0219d8a 100644
--- a/osfmk/default_pager/default_pager.c
+++ b/osfmk/default_pager/default_pager.c
@@ -314,6 +314,8 @@ start_def_pager( __unused char *bs_device )
 	if (!default_pager_backing_store_monitor_callout)
 		panic("can't start backing store monitor thread");
 	thread_call_enter(default_pager_backing_store_monitor_callout);
+	
+	return (0);
 }
 
 /*
diff --git a/osfmk/default_pager/default_pager_internal.h b/osfmk/default_pager/default_pager_internal.h
index f17823b9f..ec579a15d 100644
--- a/osfmk/default_pager/default_pager_internal.h
+++ b/osfmk/default_pager/default_pager_internal.h
@@ -543,7 +543,7 @@ struct clmap {
 	(clm)->cl_alloc.clb_map >>= (VSCLSIZE(vs) - (clm)->cl_numpages)
 
 typedef struct vstruct_alias {
-	int *name;
+	memory_object_pager_ops_t name;
 	struct vstruct *vs;
 } vstruct_alias_t;
 
@@ -580,9 +580,11 @@ typedef struct vstruct_alias {
  * VM Object Structure:  This is the structure used to manage
  * default pager object associations with their control counter-
  * parts (VM objects).
+ *
+ * The start of this structure MUST match a "struct memory_object".
  */
 typedef struct vstruct {
-	int		 	*vs_mem_obj;	/* our memory obj - temp */
+	memory_object_pager_ops_t vs_pager_ops; /* == &default_pager_ops */
 	int			vs_mem_obj_ikot;/* JMM:fake ip_kotype() */
 	memory_object_control_t vs_control;	/* our mem obj control ref */
 	VS_LOCK_TYPE		vs_lock;	/* data for the lock */
@@ -733,9 +735,11 @@ __private_extern__ zone_t	vstruct_zone;
  */
 #ifdef MACH_KERNEL
 
-#define ISVS ((int *)123456)
+extern const struct memory_object_pager_ops default_pager_ops;
+
 #define mem_obj_is_vs(_mem_obj_)					\
-	(((_mem_obj_) != NULL) && ((_mem_obj_)->pager == ISVS))
+	(((_mem_obj_) != NULL) &&					\
+	 ((_mem_obj_)->mo_pager_ops == &default_pager_ops))
 #define mem_obj_to_vs(_mem_obj_)					\
 	((vstruct_t)(_mem_obj_))
 #define vs_to_mem_obj(_vs_) ((memory_object_t)(_vs_))
diff --git a/osfmk/default_pager/dp_backing_store.c b/osfmk/default_pager/dp_backing_store.c
index d1fb84b08..5432e7c49 100644
--- a/osfmk/default_pager/dp_backing_store.c
+++ b/osfmk/default_pager/dp_backing_store.c
@@ -569,7 +569,7 @@ default_pager_backing_store_create(
 				kalloc(sizeof (struct vstruct_alias));
 	if(alias_struct != NULL) {
 		alias_struct->vs = (struct vstruct *)bs;
-		alias_struct->name = ISVS;
+		alias_struct->name = &default_pager_ops;
 		port->alias = (int) alias_struct;
 	}
 	else {
@@ -720,7 +720,7 @@ ps_delete(
 			error = KERN_FAILURE;
 		else {
 			vm_object_t	transfer_object;
-			int		count;
+			unsigned int	count;
 			upl_t		upl;
 
 			transfer_object = vm_object_allocate((vm_object_size_t)VM_SUPER_CLUSTER);
@@ -1127,7 +1127,7 @@ vs_alloc_async(void)
 				kalloc(sizeof (struct vstruct_alias));
 			if(alias_struct != NULL) {
 				alias_struct->vs = (struct vstruct *)vsa;
-				alias_struct->name = ISVS;
+				alias_struct->name = &default_pager_ops;
 				reply_port->alias = (int) alias_struct;
 				vsa->reply_port = reply_port;
 				vs_alloc_async_count++;
@@ -1179,7 +1179,7 @@ vs_alloc_async(void)
 				kalloc(sizeof (struct vstruct_alias));
 			if(alias_struct != NULL) {
 				alias_struct->vs = reply_port;
-				alias_struct->name = ISVS;
+				alias_struct->name = &default_pager_ops;
 				reply_port->alias = (int) vsa;
 				vsa->reply_port = reply_port;
 				vs_alloc_async_count++;
@@ -1235,7 +1235,7 @@ ps_vstruct_create(
 	/*
 	 * The following fields will be provided later.
 	 */
-	vs->vs_mem_obj = NULL;
+	vs->vs_pager_ops = NULL;
 	vs->vs_control = MEMORY_OBJECT_CONTROL_NULL;
 	vs->vs_references = 1;
 	vs->vs_seqno = 0;
@@ -2475,6 +2475,7 @@ ps_read_device(
 	__unused int 				flags)
 {
   panic("ps_read_device not supported");
+  return KERN_FAILURE;
 }
 
 kern_return_t
@@ -2486,6 +2487,7 @@ ps_write_device(
 	__unused struct vs_async	*vsa)
 {
   panic("ps_write_device not supported");
+  return KERN_FAILURE;
 }
 
 #endif /* DEVICE_PAGING */
@@ -2522,7 +2524,7 @@ pvs_cluster_read(
 	upl_t			upl;
 	kern_return_t		error = KERN_SUCCESS;
 	int					size;
-	int			residual;
+	unsigned int		residual;
 	unsigned int		request_flags;
 	int					seg_index;
 	int					pages_in_cl;
@@ -2570,8 +2572,8 @@ pvs_cluster_read(
 	}
 
 	while (cnt && (error == KERN_SUCCESS)) {
-	        int     ps_info_valid;
-		int	page_list_count;
+	        int     	ps_info_valid;
+		unsigned int	page_list_count;
 
 		if((vs_offset & cl_mask) && 
 			(cnt > (VM_SUPER_CLUSTER - 
@@ -2789,7 +2791,7 @@ pvs_cluster_read(
 			        failed_size = xfer_size;
 
 				if (error == KERN_SUCCESS) {
-				        if (residual == xfer_size) {
+				        if ((signed) residual == xfer_size) {
 				        /*
 					 * If a read operation returns no error
 					 * and no data moved, we turn it into
@@ -2878,7 +2880,7 @@ vs_cluster_write(
 	cl_size = pages_in_cl * vm_page_size;
 	
 	if (!dp_internal) {
-		int	     page_list_count;
+		unsigned int page_list_count;
 		int	     request_flags;
 		unsigned int super_size;
 		int          first_dirty;
@@ -3431,7 +3433,7 @@ vs_cluster_transfer(
 	kern_return_t		error = KERN_SUCCESS;
 	unsigned int		size, size_wanted;
 	int			i;
-	unsigned int		residual;
+	unsigned int		residual = 0;
 	unsigned int		unavail_size;
 //	default_pager_thread_t	*dpt;
 //	boolean_t		dealloc;
@@ -3542,6 +3544,10 @@ vs_cluster_transfer(
 			original_read_vsmap = *vsmap_ptr;
 
 		if(ps->ps_segtype == PS_PARTITION) {
+			panic("swap partition not supported\n");
+			/*NOTREACHED*/
+			error = KERN_FAILURE;
+			residual = size;
 /*
 			NEED TO ISSUE WITH SYNC & NO COMMIT
 			error = ps_read_device(ps, actual_offset, &buffer,
diff --git a/osfmk/default_pager/dp_memory_object.c b/osfmk/default_pager/dp_memory_object.c
index 2664f3e06..8e80950e3 100644
--- a/osfmk/default_pager/dp_memory_object.c
+++ b/osfmk/default_pager/dp_memory_object.c
@@ -357,6 +357,20 @@ default_pager_add(
 
 #endif
 
+const struct memory_object_pager_ops default_pager_ops = {
+	dp_memory_object_reference,
+	dp_memory_object_deallocate,
+	dp_memory_object_init,
+	dp_memory_object_terminate,
+	dp_memory_object_data_request,
+	dp_memory_object_data_return,
+	dp_memory_object_data_initialize,
+	dp_memory_object_data_unlock,
+	dp_memory_object_synchronize,
+	dp_memory_object_unmap,
+	"default pager"
+};
+
 kern_return_t
 dp_memory_object_init(
 	memory_object_t		mem_obj,
@@ -715,7 +729,7 @@ dp_memory_object_data_return(
 		/* a synchronous interface */
 		/* return KERN_LOCK_OWNED; */
 		upl_t		upl;
-		int		page_list_count = 0;
+		unsigned int	page_list_count = 0;
 		memory_object_super_upl_request(vs->vs_control,
 					(memory_object_offset_t)offset,
 					size, size,
@@ -730,8 +744,8 @@ dp_memory_object_data_return(
 	if ((vs->vs_seqno != vs->vs_next_seqno++)
 			|| (vs->vs_readers)
 			|| (vs->vs_xfer_pending)) {
-		upl_t	upl;
-		int	page_list_count = 0;
+		upl_t		upl;
+		unsigned int	page_list_count = 0;
 
 		vs->vs_next_seqno--;
                 VS_UNLOCK(vs);
@@ -815,7 +829,7 @@ default_pager_memory_object_create(
 	 * and this default_pager structure
 	 */
 
-	vs->vs_mem_obj = ISVS;
+	vs->vs_pager_ops = &default_pager_ops;
 	vs->vs_mem_obj_ikot = IKOT_MEMORY_OBJECT;
 
 	/*
@@ -850,7 +864,7 @@ default_pager_object_create(
 	 * Set up associations between the default pager
 	 * and this vstruct structure
 	 */
-	vs->vs_mem_obj = ISVS;
+	vs->vs_pager_ops = &default_pager_ops;
 	vstruct_list_insert(vs);
 	*mem_objp = vs_to_mem_obj(vs);
 	return KERN_SUCCESS;
diff --git a/osfmk/device/device.defs b/osfmk/device/device.defs
index e465cf723..54067d2bb 100644
--- a/osfmk/device/device.defs
+++ b/osfmk/device/device.defs
@@ -88,6 +88,7 @@ type io_struct_inband_t = array[*:4096] of char;
 type io_scalar_inband_t = array[*:16] of int;
 type io_async_ref_t 	= array[*:8] of natural_t;
 type io_buf_ptr_t	= ^array[] of MACH_MSG_TYPE_INTEGER_8;
+type NDR_record_t       = struct[8] of char;
 
 type io_object_t = mach_port_t
 		ctype: mach_port_t
@@ -497,6 +498,16 @@ routine io_object_get_bundle_identifier(
 	out class_name          : io_name_t   
         );
 
+routine io_service_open_extended(
+	    service		: io_object_t;
+	in  owningTask		: task_t;
+	in  connect_type	: int;
+	in  ndr			: NDR_record_t;
+	in  properties		: io_buf_ptr_t, physicalcopy;
+        out result		: natural_t;
+	out connection		: io_connect_t
+	);
+
 #endif
 
 
diff --git a/osfmk/device/device_types.defs b/osfmk/device/device_types.defs
index fd1119241..788ab01f0 100644
--- a/osfmk/device/device_types.defs
+++ b/osfmk/device/device_types.defs
@@ -97,7 +97,7 @@ type io_done_queue_t	= mach_port_t
 #if	KERNEL_SERVER
 		intran: io_done_queue_t io_done_queue_port_lookup(mach_port_t)
 		outtran: mach_port_t convert_io_done_queue_to_port(io_done_queue_t)
-#endif	KERNEL_SERVER
+#endif	/* KERNEL_SERVER */
 		;
 
 import <device/device_types.h>;
diff --git a/osfmk/device/iokit_rpc.c b/osfmk/device/iokit_rpc.c
index 29ef1f998..a04a511a1 100644
--- a/osfmk/device/iokit_rpc.c
+++ b/osfmk/device/iokit_rpc.c
@@ -94,6 +94,9 @@ iokit_client_memory_for_type(
 	vm_address_t *	address,
 	vm_size_t    *	size );
 
+
+extern ppnum_t IOGetLastPageNumber(void);
+
 /*
  * Lookup a device by its port.
  * Doesn't consume the naked send right; produces a device reference.
@@ -400,36 +403,13 @@ iokit_notify( mach_msg_header_t * msg )
 /* need to create a pmap function to generalize */
 unsigned int IODefaultCacheBits(addr64_t pa)
 {
-    unsigned int	flags;
-#ifndef i386
-    struct phys_entry * pp;
-
-    // Find physical address
-    if ((pp = pmap_find_physentry(pa >> 12))) {
-	// Use physical attributes as default
-	// NOTE: DEVICE_PAGER_FLAGS are made to line up
-	flags = VM_MEM_COHERENT;						/* We only support coherent memory */
-	if(pp->ppLink & ppG) flags |= VM_MEM_GUARDED;	/* Add in guarded if it is */
-	if(pp->ppLink & ppI) flags |= VM_MEM_NOT_CACHEABLE;	/* Add in cache inhibited if so */
-    } else
-	// If no physical, just hard code attributes
-        flags = VM_WIMG_IO;
-#else
-    extern pmap_paddr_t	avail_end;
 
-    if (pa < avail_end)
-	flags = VM_WIMG_COPYBACK;
-    else
-	flags = VM_WIMG_IO;
-#endif
-
-    return flags;
+      return(pmap_cache_attributes(pa >> PAGE_SHIFT));
 }
 
-kern_return_t IOMapPages(vm_map_t map, vm_offset_t va, vm_offset_t pa,
-			vm_size_t length, unsigned int options)
+kern_return_t IOMapPages(vm_map_t map, mach_vm_address_t va, mach_vm_address_t pa,
+			mach_vm_size_t length, unsigned int options)
 {
-    vm_size_t	off;
     vm_prot_t	prot;
     unsigned int flags;
     pmap_t 	 pmap = map->pmap;
@@ -452,7 +432,7 @@ kern_return_t IOMapPages(vm_map_t map, vm_offset_t va, vm_offset_t pa,
 	    flags = VM_WIMG_WTHRU;
 	    break;
 
-	case kIOWriteCombineCache:
+	case kIOMapWriteCombineCache:
 	    flags = VM_WIMG_WCOMB;
 	    break;
 
@@ -460,33 +440,100 @@ kern_return_t IOMapPages(vm_map_t map, vm_offset_t va, vm_offset_t pa,
 	    flags = VM_WIMG_COPYBACK;
 	    break;
     }
-#if __ppc__
 
     // Set up a block mapped area
-    pmap_map_block(pmap, (addr64_t)va, (ppnum_t)(pa >> 12), (uint32_t)(length >> 12), prot, flags, 0);
+    pmap_map_block(pmap, va, (ppnum_t)atop_64(pa), (uint32_t) atop_64(round_page_64(length)), prot, flags, 0);
+
+    return( KERN_SUCCESS );
+}
+
+kern_return_t IOUnmapPages(vm_map_t map, mach_vm_address_t va, mach_vm_size_t length)
+{
+    pmap_t	pmap = map->pmap;
+
+    pmap_remove(pmap, trunc_page_64(va), round_page_64(va + length));
+
+    return( KERN_SUCCESS );
+}
+
+kern_return_t IOProtectCacheMode(vm_map_t map, mach_vm_address_t va,
+					mach_vm_size_t length, unsigned int options)
+{
+    mach_vm_size_t off;
+    vm_prot_t	   prot;
+    unsigned int   flags;
+    pmap_t 	   pmap = map->pmap;
+
+    prot = (options & kIOMapReadOnly)
+		? VM_PROT_READ : (VM_PROT_READ|VM_PROT_WRITE);
+
+    switch (options & kIOMapCacheMask)
+    {
+    /* What cache mode do we need? */
+	case kIOMapDefaultCache:
+	default:
+	    return (KERN_INVALID_ARGUMENT);
+
+	case kIOMapInhibitCache:
+	    flags = VM_WIMG_IO;
+	    break;
+
+	case kIOMapWriteThruCache:
+	    flags = VM_WIMG_WTHRU;
+	    break;
 
+	case kIOMapWriteCombineCache:
+	    flags = VM_WIMG_WCOMB;
+	    break;
+
+	case kIOMapCopybackCache:
+	    flags = VM_WIMG_COPYBACK;
+	    break;
+    }
+#if __ppc__
+    // can't remap block mappings, but ppc doesn't speculative read from WC
 #else
-//  enter each page's physical address in the target map
 
+    //  enter each page's physical address in the target map
     for (off = 0; off < length; off += page_size)
-	pmap_enter(pmap, va + off, (pa + off) >> 12, prot, flags, TRUE);
+    {
+	ppnum_t ppnum = pmap_find_phys(pmap, va + off);
+	if (ppnum)
+	    pmap_enter(pmap, va + off, ppnum, prot, flags, TRUE);
+    }
 
 #endif
 
-    return( KERN_SUCCESS );
+    return (KERN_SUCCESS);
 }
 
-kern_return_t IOUnmapPages(vm_map_t map, vm_offset_t va, vm_size_t length)
+ppnum_t IOGetLastPageNumber(void)
 {
-    pmap_t	pmap = map->pmap;
+    ppnum_t	 lastPage, highest = 0;
 
-    pmap_remove(pmap, trunc_page_64(va), round_page_64(va + length));
-
-    return( KERN_SUCCESS );
+#if __ppc__
+    int idx;
+    for (idx = 0; idx < pmap_mem_regions_count; idx++)
+    {
+	lastPage = pmap_mem_regions[idx].mrEnd;
+#elif __i386__
+    unsigned int idx;
+    for (idx = 0; idx < pmap_memory_region_count; idx++)
+    {
+	lastPage = pmap_memory_regions[idx].end - 1;
+#else
+#error arch
+#endif
+	if (lastPage > highest)
+	    highest = lastPage;
+    }
+    return (highest);
 }
 
+
 void IOGetTime( mach_timespec_t * clock_time);
 void IOGetTime( mach_timespec_t * clock_time)
 {
 	clock_get_system_nanotime(&clock_time->tv_sec, &clock_time->tv_nsec);
 }
+
diff --git a/osfmk/i386/AT386/bbclock.c b/osfmk/i386/AT386/bbclock.c
deleted file mode 100644
index 3849e7194..000000000
--- a/osfmk/i386/AT386/bbclock.c
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the License
- * may not be used to create, or enable the creation or redistribution of,
- * unlawful or unlicensed copies of an Apple operating system, or to
- * circumvent, violate, or enable the circumvention or violation of, any
- * terms of an Apple operating system software license agreement.
- * 
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- * 
- * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/*
-  Copyright 1988, 1989 by Intel Corporation, Santa Clara, California.
-
-		All Rights Reserved
-
-Permission to use, copy, modify, and distribute this software and
-its documentation for any purpose and without fee is hereby
-granted, provided that the above copyright notice appears in all
-copies and that both the copyright notice and this permission notice
-appear in supporting documentation, and that the name of Intel
-not be used in advertising or publicity pertaining to distribution
-of the software without specific, written prior permission.
-
-INTEL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
-INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS,
-IN NO EVENT SHALL INTEL BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
-CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
-NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
-WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-*/
-
-#include <types.h>
-#include <mach/message.h>
-#include <kern/thread.h>
-#include <kern/clock.h>
-#include <kern/spl.h>
-#include <kern/processor.h>
-#include <kern/misc_protos.h>
-#include <i386/cpu_data.h>
-#include <i386/cpu_number.h>
-#include <i386/pio.h>
-#include <i386/AT386/rtc.h>
-#include <i386/AT386/bbclock_entries.h>
-
-/* local data */
-static	int	month[12] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
-
-extern	char	dectohexdec(
-			int			n);
-extern int	hexdectodec(
-			char			c);
-extern int	yeartoday(
-			int			yr);
-extern void	rtcput(
-			struct rtc_st		* regs);
-extern int	rtcget(
-			struct rtc_st		* regs);
-
-#define	LOCK_BBC()	splclock()
-#define	UNLOCK_BBC(s)	splx(s)
-
-/*
- * Configure battery-backed clock.
- */
-int
-bbc_config(void)
-{
-	int		BbcFlag;
-	struct rtc_st	rtclk;
-
-	mp_disable_preemption();
-	if (cpu_number() != master_cpu) {
-		mp_enable_preemption();
-		return(1);
-	}
-
-	/*
-	 * Setup device.
-	 */
-	outb(RTC_ADDR, RTC_A);
-	outb(RTC_DATA, RTC_DIV2 | RTC_RATE6);
-	outb(RTC_ADDR, RTC_B);
-	outb(RTC_DATA, RTC_HM);
-
-	/*
-	 * Probe the device by trying to read it.
-	 */
-	BbcFlag = (rtcget(&rtclk) ? 0 : 1);
-	if (BbcFlag)
-		printf("battery clock configured\n");
-	else
-		printf("WARNING: Battery Clock Failure!\n");
-	mp_enable_preemption();
-	return (BbcFlag);
-}
-
-/*
- * Get the current clock time.
- */
-kern_return_t
-bbc_gettime(
-	mach_timespec_t	*cur_time)	/* OUT */
-{
-	struct rtc_st	rtclk;
-	time_t		n;
-	int		sec, min, hr, dom, mon, yr;
-	int		i, days = 0;
-	spl_t		s;
-	thread_t	thread;
-
-	if ((thread = current_thread()) != THREAD_NULL) {
-		thread_bind(thread, master_processor);
-		mp_disable_preemption();
-		if (current_processor() != master_processor) {
-			mp_enable_preemption();
-			thread_block(THREAD_CONTINUE_NULL);
-		} else {
-			mp_enable_preemption();
-		}
-	}
-
-	s = LOCK_BBC();
-	rtcget(&rtclk);
-	sec = hexdectodec(rtclk.rtc_sec);
-	min = hexdectodec(rtclk.rtc_min);
-	hr = hexdectodec(rtclk.rtc_hr);
-	dom = hexdectodec(rtclk.rtc_dom);
-	mon = hexdectodec(rtclk.rtc_mon);
-	yr = hexdectodec(rtclk.rtc_yr);
-	yr = (yr < 70) ? yr+100 : yr;
-	n = sec + 60 * min + 3600 * hr;
-	n += (dom - 1) * 3600 * 24;
-	if (yeartoday(yr) == 366)
-		month[1] = 29;
-	for (i = mon - 2; i >= 0; i--)
-		days += month[i];
-	month[1] = 28;
-	for (i = 70; i < yr; i++)
-		days += yeartoday(i);
-	n += days * 3600 * 24;
-	cur_time->tv_sec  = n;
-	cur_time->tv_nsec = 0;
-	UNLOCK_BBC(s);
-
-	if (thread != THREAD_NULL)
-		thread_bind(thread, PROCESSOR_NULL);
-	return (KERN_SUCCESS);
-}
-
-/*
- * Set the current clock time.
- */
-kern_return_t
-bbc_settime(
-	mach_timespec_t	*new_time)
-{
-	struct rtc_st	rtclk;
-	time_t		n;
-	int		diff, i, j;
-	spl_t		s;
-	thread_t	thread;
-
-	if ((thread = current_thread()) != THREAD_NULL) {
-		thread_bind(thread, master_processor);
-		mp_disable_preemption();
-		if (current_processor() != master_processor) {
-			mp_enable_preemption();
-			thread_block(THREAD_CONTINUE_NULL);
-		} else { 
-			mp_enable_preemption();
-		}
-	}
-
-	s = LOCK_BBC();
-	rtcget(&rtclk);
-	diff = 0;
-	n = (new_time->tv_sec - diff) % (3600 * 24);   /* hrs+mins+secs */
-	rtclk.rtc_sec = dectohexdec(n%60);
-	n /= 60;
-	rtclk.rtc_min = dectohexdec(n%60);
-	rtclk.rtc_hr = dectohexdec(n/60);
-	n = (new_time->tv_sec - diff) / (3600 * 24);	/* days */
-	rtclk.rtc_dow = (n + 4) % 7;  /* 1/1/70 is Thursday */
-	for (j = 70; n >= (i = yeartoday(j)); j++)
-		n -= i;
-	rtclk.rtc_yr = dectohexdec(j % 100);
-	if (yeartoday(j) == 366)
-		month[1] = 29;
-	for (i = 0; n >= month[i]; i++)
-		n -= month[i];
-	month[1] = 28;
-	rtclk.rtc_mon = dectohexdec(++i);
-	rtclk.rtc_dom = dectohexdec(++n);
-	rtcput(&rtclk);
-	UNLOCK_BBC(s);
-
-	if (thread != THREAD_NULL)
-		thread_bind(current_thread(), PROCESSOR_NULL);
-
-	return (KERN_SUCCESS);
-}
-
-
-/* DEVICE SPECIFIC ROUTINES */
-
-int
-rtcget(
-	struct rtc_st	* regs)
-{
-	outb(RTC_ADDR, RTC_D); 
-	if ((inb(RTC_DATA) & RTC_VRT) == 0)
-		return (-1);
-	outb(RTC_ADDR, RTC_A);	
-	while (inb(RTC_DATA) & RTC_UIP)		/* busy wait */
-		outb(RTC_ADDR, RTC_A);	
-	load_rtc((unsigned char *)regs);
-	return (0);
-}	
-
-void
-rtcput(
-	struct rtc_st	* regs)
-{
-	register unsigned char	x;
-
-	outb(RTC_ADDR, RTC_B);
-	x = inb(RTC_DATA);
-	outb(RTC_ADDR, RTC_B);
-	outb(RTC_DATA, x | RTC_SET); 	
-	save_rtc((unsigned char *)regs);
-	outb(RTC_ADDR, RTC_B);
-	outb(RTC_DATA, x & ~RTC_SET); 
-}
-
-int
-yeartoday(
-	int	year)
-{
-        year += 1900;
-	return((year % 4) ? 365 :
-	       ((year % 100) ? 366 : ((year % 400) ? 365: 366)));
-}
-
-int
-hexdectodec(
-	char	n)
-{
-	return ((((n >> 4) & 0x0F) * 10) + (n & 0x0F));
-}
-
-char
-dectohexdec(
-	int	n)
-{
-	return ((char)(((n / 10) << 4) & 0xF0) | ((n % 10) & 0x0F));
-}
diff --git a/osfmk/i386/AT386/conf.c b/osfmk/i386/AT386/conf.c
index 779bc024b..598a237c7 100644
--- a/osfmk/i386/AT386/conf.c
+++ b/osfmk/i386/AT386/conf.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -78,9 +78,9 @@ extern	struct clock_ops	sysclk_ops, calend_ops;
 struct	clock	clock_list[] = {
 
 	/* SYSTEM_CLOCK */
-	{ &sysclk_ops, 0, 0, {0} },
+	{ &sysclk_ops, 0, 0 },
 
 	/* CALENDAR_CLOCK */
-	{ &calend_ops, 0, 0, {0} }
+	{ &calend_ops, 0, 0 }
 };
 int	clock_count = sizeof(clock_list) / sizeof(clock_list[0]);
diff --git a/osfmk/i386/AT386/misc_protos.h b/osfmk/i386/AT386/misc_protos.h
index 4b88dfa53..1ef5c8575 100644
--- a/osfmk/i386/AT386/misc_protos.h
+++ b/osfmk/i386/AT386/misc_protos.h
@@ -39,7 +39,6 @@
  */
 
 extern void		i386_init(void);
-extern void		i386_vm_init(unsigned int maxmem, KernelBootArgs_t *args);
 extern void		machine_init(void);
 extern void		machine_startup(void);
 
@@ -50,13 +49,6 @@ extern void		machine_startup(void);
 extern void		cninit(void);
 extern void		kdreboot(void);
 
-/*
- * i386/locore.s
- */
-
-extern void		kdb_kintr(void);
-extern void		kgdb_kintr(void);
-
 /*
  * i386/db_interface.c
  */
diff --git a/osfmk/i386/AT386/model_dep.c b/osfmk/i386/AT386/model_dep.c
index 824265de3..7c9575171 100644
--- a/osfmk/i386/AT386/model_dep.c
+++ b/osfmk/i386/AT386/model_dep.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -68,7 +68,6 @@
 
 #include <platforms.h>
 #include <mach_kdb.h>
-#include <himem.h>
 
 #include <mach/i386/vm_param.h>
 
@@ -87,26 +86,39 @@
 #include <kern/machine.h>
 #include <i386/fpu.h>
 #include <i386/ipl.h>
-#include <i386/pio.h>
 #include <i386/misc_protos.h>
-#include <i386/mp.h>
 #include <i386/mtrr.h>
+#include <i386/machine_routines.h>
+#include <i386/pmCPU.h>
 #include <i386/postcode.h>
+#include <architecture/i386/pio.h> /* inb() */
 #include <pexpert/i386/boot.h>
 #if	MACH_KDB
 #include <ddb/db_aout.h>
 #endif /* MACH_KDB */
 
+#include <vm/pmap.h>
+#include <vm/vm_map.h>
+#include <vm/vm_kern.h>
+
 #include <i386/mp_desc.h>
 #include <i386/mp.h>
+#include <i386/cpuid.h>
 
 #include <IOKit/IOPlatformExpert.h>
+#include <IOKit/IOHibernatePrivate.h>
+
+#include <pexpert/i386/efi.h>
+
+#include <kern/thread.h>
+#include <i386/thread.h>
+#include <mach-o/loader.h>
+#include <mach-o/nlist.h>
 
 void	enable_bluebox(void);
 void	disable_bluebox(void);
 
 static void machine_conf(void);
-#include <i386/cpuid.h>
 
 extern int		default_preemption_rate;
 extern int		max_unsafe_quanta;
@@ -114,6 +126,26 @@ extern int		max_poll_quanta;
 extern int		idlehalt;
 extern unsigned int	panic_is_inited;
 
+int db_run_mode;
+
+static int packAsc (uint8_t *inbuf, unsigned int length);
+
+volatile int pbtcpu = -1;
+hw_lock_data_t pbtlock;		/* backtrace print lock */
+uint32_t pbtcnt = 0;
+
+extern const char version[];
+
+typedef struct _cframe_t {
+    struct _cframe_t	*prev;
+    unsigned		caller;
+    unsigned		args[0];
+} cframe_t;
+
+void panic_i386_backtrace(void *_frame, int nframes);
+
+static unsigned panic_io_port = 0;
+
 void
 machine_startup()
 {
@@ -134,8 +166,8 @@ machine_startup()
 
 #if NOTYET
 	hw_lock_init(&debugger_lock);	/* initialize debugger lock */
-	hw_lock_init(&pbtlock);		/* initialize print backtrace lock */
 #endif
+	hw_lock_init(&pbtlock);		/* initialize print backtrace lock */
 
 #if	MACH_KDB
 	/*
@@ -176,6 +208,32 @@ machine_startup()
 	if (PE_parse_boot_arg("idlehalt", &boot_arg)) {
 		idlehalt = boot_arg;
 	}
+/* The I/O port to issue a read from, in the event of a panic. Useful for
+ * triggering logic analyzers.
+ */
+	if (PE_parse_boot_arg("panic_io_port", &boot_arg)) {
+		/*I/O ports range from 0 through 0xFFFF */
+		panic_io_port = boot_arg & 0xffff;
+	}
+
+/*
+ *	fn is used to force napping.
+ *		fn=0 means no napping allowed
+ *		fn=1 means forces napping on, normal C2 and C4 transitions
+ *		fn=2 means forces napping on, but C4 is disabled
+ *		fn=3 means forces napping on, but use halt
+ *		fn=4 means forces napping on and will always use C4
+ *
+ *		Note that this will take effect only when the system normally starts napping.
+ *
+ */ 
+
+	if (!PE_parse_boot_arg("fn", &forcenap)) forcenap = 0;	/* If force nap not set, make 0 */
+	else {
+		if(forcenap < 5) forcenap = forcenap + 1;		/* See comments above for decode, this is set to fn + 1 */
+		else forcenap = 0;								/* Clear for error case */
+	}
+	machine_nap_policy();								/* Make sure the nap policy reflects the user's choice */
 
 	machine_conf();
 
@@ -197,18 +255,363 @@ machine_conf(void)
 	machine_info.memory_size = mem_size;
 }
 
+
+extern void *gPEEFIRuntimeServices;
+extern void *gPEEFISystemTable;
+
+/*-
+ *  COPYRIGHT (C) 1986 Gary S. Brown.  You may use this program, or
+ *  code or tables extracted from it, as desired without restriction.
+ *
+ *  First, the polynomial itself and its table of feedback terms.  The
+ *  polynomial is
+ *  X^32+X^26+X^23+X^22+X^16+X^12+X^11+X^10+X^8+X^7+X^5+X^4+X^2+X^1+X^0
+ *
+ *  Note that we take it "backwards" and put the highest-order term in
+ *  the lowest-order bit.  The X^32 term is "implied"; the LSB is the
+ *  X^31 term, etc.  The X^0 term (usually shown as "+1") results in
+ *  the MSB being 1
+ *
+ *  Note that the usual hardware shift register implementation, which
+ *  is what we're using (we're merely optimizing it by doing eight-bit
+ *  chunks at a time) shifts bits into the lowest-order term.  In our
+ *  implementation, that means shifting towards the right.  Why do we
+ *  do it this way?  Because the calculated CRC must be transmitted in
+ *  order from highest-order term to lowest-order term.  UARTs transmit
+ *  characters in order from LSB to MSB.  By storing the CRC this way
+ *  we hand it to the UART in the order low-byte to high-byte; the UART
+ *  sends each low-bit to hight-bit; and the result is transmission bit
+ *  by bit from highest- to lowest-order term without requiring any bit
+ *  shuffling on our part.  Reception works similarly
+ *
+ *  The feedback terms table consists of 256, 32-bit entries.  Notes
+ *
+ *      The table can be generated at runtime if desired; code to do so
+ *      is shown later.  It might not be obvious, but the feedback
+ *      terms simply represent the results of eight shift/xor opera
+ *      tions for all combinations of data and CRC register values
+ *
+ *      The values must be right-shifted by eight bits by the "updcrc
+ *      logic; the shift must be unsigned (bring in zeroes).  On some
+ *      hardware you could probably optimize the shift in assembler by
+ *      using byte-swap instructions
+ *      polynomial $edb88320
+ *
+ *
+ * CRC32 code derived from work by Gary S. Brown.
+ */
+
+static uint32_t crc32_tab[] = {
+	0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f,
+	0xe963a535, 0x9e6495a3,	0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
+	0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2,
+	0xf3b97148, 0x84be41de,	0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
+	0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,	0x14015c4f, 0x63066cd9,
+	0xfa0f3d63, 0x8d080df5,	0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
+	0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,	0x35b5a8fa, 0x42b2986c,
+	0xdbbbc9d6, 0xacbcf940,	0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
+	0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423,
+	0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
+	0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,	0x76dc4190, 0x01db7106,
+	0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
+	0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d,
+	0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
+	0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
+	0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
+	0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7,
+	0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
+	0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa,
+	0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
+	0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81,
+	0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
+	0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84,
+	0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
+	0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
+	0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
+	0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e,
+	0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
+	0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55,
+	0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
+	0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28,
+	0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
+	0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f,
+	0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
+	0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
+	0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
+	0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69,
+	0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
+	0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc,
+	0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
+	0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693,
+	0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
+	0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d
+};
+
+static uint32_t
+crc32(uint32_t crc, const void *buf, size_t size)
+{
+	const uint8_t *p;
+
+	p = buf;
+	crc = crc ^ ~0U;
+
+	while (size--)
+		crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8);
+
+	return crc ^ ~0U;
+}
+
+static void
+efi_set_tables_64(EFI_SYSTEM_TABLE_64 * system_table)
+{
+    EFI_RUNTIME_SERVICES_64 *runtime;
+    uint32_t hdr_cksum;
+    uint32_t cksum;
+
+    kprintf("Processing 64-bit EFI tables at 0x%x\n", (unsigned int)system_table);
+    do {
+        if (system_table->Hdr.Signature != EFI_SYSTEM_TABLE_SIGNATURE) {
+	    kprintf("Bad EFI system table signature\n");
+            break;
+        }
+        // Verify signature of the system table
+        hdr_cksum = system_table->Hdr.CRC32;
+        system_table->Hdr.CRC32 = 0;
+        cksum = crc32(0L, system_table, system_table->Hdr.HeaderSize);
+
+        //kprintf("System table calculated CRC32 = 0x%x, header = 0x%x\n", cksum, hdr_cksum);
+        system_table->Hdr.CRC32 = hdr_cksum;
+        if (cksum != hdr_cksum) {
+            kprintf("Bad EFI system table checksum\n");
+            break;
+        }
+
+	gPEEFISystemTable     = system_table;
+
+        kprintf("RuntimeServices table at 0x%qx\n", system_table->RuntimeServices);
+        runtime = (EFI_RUNTIME_SERVICES_64 *) (uint32_t)system_table->RuntimeServices; // XXX
+        kprintf("Checking runtime services table 0x%x\n", runtime);
+	if (runtime->Hdr.Signature != EFI_RUNTIME_SERVICES_SIGNATURE) {
+	    kprintf("Bad EFI runtime table signature\n");
+	    break;
+	}
+
+	// Verify signature of runtime services table
+	hdr_cksum = runtime->Hdr.CRC32;
+	runtime->Hdr.CRC32 = 0;
+	cksum = crc32(0L, runtime, runtime->Hdr.HeaderSize);
+
+	//kprintf("Runtime table calculated CRC32 = 0x%x, header = 0x%x\n", cksum, hdr_cksum);
+	runtime->Hdr.CRC32 = hdr_cksum;
+	if (cksum != hdr_cksum) {
+	    kprintf("Bad EFI runtime table checksum\n");
+	    break;
+	}
+
+	gPEEFIRuntimeServices = runtime;
+    }
+    while (FALSE);
+}
+
+static void
+efi_set_tables_32(EFI_SYSTEM_TABLE * system_table)
+{
+    EFI_RUNTIME_SERVICES *runtime;
+    uint32_t hdr_cksum;
+    uint32_t cksum;
+
+    kprintf("Processing 32-bit EFI tables at 0x%x\n", (unsigned int)system_table);
+    do {
+        if (system_table->Hdr.Signature != EFI_SYSTEM_TABLE_SIGNATURE) {
+	    kprintf("Bad EFI system table signature\n");
+            break;
+        }
+        // Verify signature of the system table
+        hdr_cksum = system_table->Hdr.CRC32;
+        system_table->Hdr.CRC32 = 0;
+        cksum = crc32(0L, system_table, system_table->Hdr.HeaderSize);
+
+        //kprintf("System table calculated CRC32 = 0x%x, header = 0x%x\n", cksum, hdr_cksum);
+        system_table->Hdr.CRC32 = hdr_cksum;
+        if (cksum != hdr_cksum) {
+            kprintf("Bad EFI system table checksum\n");
+            break;
+        }
+
+	gPEEFISystemTable     = system_table;
+
+        runtime = (EFI_RUNTIME_SERVICES *) system_table->RuntimeServices;
+	if (runtime->Hdr.Signature != EFI_RUNTIME_SERVICES_SIGNATURE) {
+	    kprintf("Bad EFI runtime table signature\n");
+	    break;
+	}
+
+	// Verify signature of runtime services table
+	hdr_cksum = runtime->Hdr.CRC32;
+	runtime->Hdr.CRC32 = 0;
+	cksum = crc32(0L, runtime, runtime->Hdr.HeaderSize);
+
+	//kprintf("Runtime table calculated CRC32 = 0x%x, header = 0x%x\n", cksum, hdr_cksum);
+	runtime->Hdr.CRC32 = hdr_cksum;
+	if (cksum != hdr_cksum) {
+	    kprintf("Bad EFI runtime table checksum\n");
+	    break;
+	}
+
+	gPEEFIRuntimeServices = runtime;
+    }
+    while (FALSE);
+}
+
+
+/* Map in EFI runtime areas. */
+static void
+efi_init(void)
+{
+    boot_args *args = (boot_args *)PE_state.bootArgs;
+
+    kprintf("Initializing EFI runtime services\n");
+
+    do
+    {
+        vm_offset_t vm_size, vm_addr;
+	vm_map_offset_t phys_addr;
+	EfiMemoryRange *mptr;
+	unsigned int msize, mcount;
+	unsigned int i;
+
+	msize = args->MemoryMapDescriptorSize;
+	mcount = args->MemoryMapSize / msize;
+
+	mptr = (EfiMemoryRange *)args->MemoryMap;
+	for (i=0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
+	    if (((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME) ) {
+		vm_size = i386_ptob((uint32_t)mptr->NumberOfPages);
+		vm_addr =   (vm_offset_t) mptr->VirtualStart;
+		phys_addr = (vm_map_offset_t) mptr->PhysicalStart;
+		pmap_map(vm_addr, phys_addr, phys_addr + round_page(vm_size),
+		     (mptr->Type == kEfiRuntimeServicesCode) ? VM_PROT_READ | VM_PROT_EXECUTE : VM_PROT_READ|VM_PROT_WRITE,
+		     (mptr->Type == EfiMemoryMappedIO)       ? VM_WIMG_IO   : VM_WIMG_USE_DEFAULT);
+	    }
+	}
+
+        if (args->Version > 1)
+                panic("Incompatible boot args version %d\n", args->Version);
+
+        kprintf("Boot args version %d revision %d mode %d\n", args->Version, args->Revision, args->efiMode);
+        if (args->Revision >= 4 && args->efiMode == kBootArgsEfiMode64) {
+                efi_set_tables_64((EFI_SYSTEM_TABLE_64 *) args->efiSystemTable);
+        } else {
+                efi_set_tables_32((EFI_SYSTEM_TABLE *) args->efiSystemTable);
+        }
+    }
+    while (FALSE);
+
+    return;
+}
+
+/* Remap EFI runtime areas. */
+void
+hibernate_newruntime_map(void * map, vm_size_t map_size, uint32_t system_table_offset)
+{
+    boot_args *args = (boot_args *)PE_state.bootArgs;
+
+    kprintf("Reinitializing EFI runtime services\n");
+
+    if (args->Revision < 3)
+	return;
+    do
+    {
+	vm_offset_t vm_size, vm_addr;
+	vm_map_offset_t phys_addr;
+	EfiMemoryRange *mptr;
+	unsigned int msize, mcount;
+	unsigned int i;
+
+	gPEEFISystemTable     = 0;
+	gPEEFIRuntimeServices = 0;
+
+	system_table_offset += ptoa_32(args->efiRuntimeServicesPageStart);
+
+	kprintf("Old system table %p, new %p\n",
+	    args->efiSystemTable,    (void *) system_table_offset);
+
+	args->efiSystemTable    = (uint32_t) system_table_offset;
+
+	kprintf("Old map:\n");
+	msize = args->MemoryMapDescriptorSize;
+	mcount = args->MemoryMapSize / msize;
+	mptr = (EfiMemoryRange *)args->MemoryMap;
+	for (i=0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
+	    if ((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME) {
+
+		vm_size = i386_ptob((uint32_t)mptr->NumberOfPages);
+		vm_addr =   (vm_offset_t) mptr->VirtualStart;
+		phys_addr = (vm_map_offset_t) mptr->PhysicalStart;
+
+		kprintf("mapping[%d] %qx @ %x, %x\n", mptr->Type, phys_addr, vm_addr, mptr->NumberOfPages);
+	    }
+	}
+
+	pmap_remove(kernel_pmap, i386_ptob(args->efiRuntimeServicesPageStart), 
+				 i386_ptob(args->efiRuntimeServicesPageStart + args->efiRuntimeServicesPageCount));
+
+	kprintf("New map:\n");
+	msize = args->MemoryMapDescriptorSize;
+	mcount = map_size / msize;
+	mptr = map;
+	for (i=0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
+	    if ((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME) {
+
+		vm_size = i386_ptob((uint32_t)mptr->NumberOfPages);
+		vm_addr =   (vm_offset_t) mptr->VirtualStart;
+		phys_addr = (vm_map_offset_t) mptr->PhysicalStart;
+
+		kprintf("mapping[%d] %qx @ %x, %x\n", mptr->Type, phys_addr, vm_addr, mptr->NumberOfPages);
+
+		pmap_map(vm_addr, phys_addr, phys_addr + round_page(vm_size),
+			 (mptr->Type == kEfiRuntimeServicesCode) ? VM_PROT_READ | VM_PROT_EXECUTE : VM_PROT_READ|VM_PROT_WRITE,
+			 (mptr->Type == EfiMemoryMappedIO)       ? VM_WIMG_IO   : VM_WIMG_USE_DEFAULT);
+	    }
+	}
+
+        if (args->Version > 1)
+                panic("Incompatible boot args version %d\n", args->Version);
+
+        kprintf("Boot args version %d revision %d mode %d\n", args->Version, args->Revision, args->efiMode);
+        if (args->Revision >= 4 && args->efiMode == kBootArgsEfiMode64) {
+                efi_set_tables_64((EFI_SYSTEM_TABLE_64 *) args->efiSystemTable);
+        } else {
+                efi_set_tables_32((EFI_SYSTEM_TABLE *) args->efiSystemTable);
+        }
+    }
+    while (FALSE);
+
+    kprintf("Done reinitializing EFI runtime services\n");
+
+    return;
+}
+
 /*
  * Find devices.  The system is alive.
  */
 void
 machine_init(void)
 {
+        /* Ensure panic buffer is initialized. */
+        debug_log_init();
+
 	/*
 	 * Display CPU identification
 	 */
-	cpuid_cpu_display("CPU identification", 0);
-	cpuid_feature_display("CPU features", 0);
+	cpuid_cpu_display("CPU identification");
+	cpuid_feature_display("CPU features");
+	cpuid_extfeature_display("CPU extended features");
 
+        /*
+         * Initialize EFI runtime services.
+         */
+        efi_init();
 
 	smp_init();
 
@@ -256,14 +659,6 @@ void
 halt_all_cpus(boolean_t reboot)
 {
 	if (reboot) {
-		/*
-		 * Tell the BIOS not to clear and test memory.
-		 */
-#if 0 /* XXX fixme */
-		if (!reset_mem_on_reboot)
-			*(unsigned short *)phystokv(0x472) = 0x1234;
-#endif
-
 		printf("MACH Reboot\n");
 		PEHaltRestart( kPERestartCPU );
 	} else {
@@ -273,37 +668,128 @@ halt_all_cpus(boolean_t reboot)
 	while(1);
 }
 
-/*XXX*/
-void fc_get(mach_timespec_t *ts);
-#include <kern/clock.h>
-extern kern_return_t	sysclk_gettime(
-			mach_timespec_t	*cur_time);
-void fc_get(mach_timespec_t *ts) {
-	(void )sysclk_gettime(ts);
+/* Issue an I/O port read if one has been requested - this is an event logic
+ * analyzers can use as a trigger point.
+ */
+
+void
+panic_io_port_read(void) {
+	if (panic_io_port)
+		(void)inb(panic_io_port);
+}
+
+/* For use with the MP rendezvous mechanism
+ */
+
+static void
+machine_halt_cpu(__unused void *arg) {
+	panic_io_port_read();
+	__asm__ volatile("hlt");
 }
 
 void
 Debugger(
 	const char	*message)
 {
+	unsigned long pi_size = 0;
+	void *stackptr;
 
+	hw_atomic_add(&debug_mode, 1);   
 	if (!panic_is_inited) {
 		postcode(PANIC_HLT);
 		asm("hlt");
 	}
 
+
 	printf("Debugger called: <%s>\n", message);
 	kprintf("Debugger called: <%s>\n", message);
 
-	draw_panic_dialog();
+	/*
+	 * Skip the graphical panic box if no panic string.
+	 * This is the case if we're being called from
+	 *   host_reboot(,HOST_REBOOT_DEBUGGER)
+	 * as a quiet way into the debugger.
+	 */
+
+	if (panicstr) {
+		disable_preemption();
+
+/* Issue an I/O port read if one has been requested - this is an event logic
+ * analyzers can use as a trigger point.
+ */
+		panic_io_port_read();
+
+		/* Obtain current frame pointer */
+		__asm__ volatile("movl %%ebp, %0" : "=m" (stackptr));
+
+		/* Print backtrace - callee is internally synchronized */
+		panic_i386_backtrace(stackptr, 16);
+
+		/* everything should be printed now so copy to NVRAM
+		 */
+
+		if( debug_buf_size > 0) {
+		    /* Do not compress the panic log 
+		     * or save to NVRAM unless kernel debugging 
+		     * is disabled. The NVRAM shim doesn't
+		     * sync to the store until haltRestart is called.
+		     */
+		    if (!panicDebugging) {
+			unsigned int bufpos;
+			
+                        debug_putc(0);
+
+			/* Now call the compressor */
+			/* XXX Consider using the WKdm compressor in the
+			 * future, rather than just packing - would need to
+			 * be co-ordinated with crashreporter, which decodes
+			 * this post-restart.
+			 */
+			bufpos = packAsc ((uint8_t *)debug_buf,
+			    (unsigned int) (debug_buf_ptr - debug_buf) );
+			/* If compression was successful,
+			 * use the compressed length
+			 */
+			if (bufpos) {
+			    debug_buf_ptr = debug_buf + bufpos;
+                        }
+			/* Save panic log to non-volatile store
+			 * Panic info handler must truncate data that is 
+			 * too long for this platform.
+			 * This call must save data synchronously,
+			 * since we can subsequently halt the system.
+			 */
+                        pi_size = debug_buf_ptr - debug_buf;
+                        pi_size = PESavePanicInfo((unsigned char *)debug_buf,
+			    pi_size );
+                    }
+                }
+		draw_panic_dialog();
+
+		if (!panicDebugging) {
+			/* Clear the MP rendezvous function lock, in the event
+			 * that a panic occurred while in that codepath.
+			 */
+			mp_rendezvous_break_lock();
+			/* Force all CPUs to disable interrupts and HLT.
+			 * We've panicked, and shouldn't depend on the
+			 * PEHaltRestart() mechanism, which relies on several
+			 * bits of infrastructure.
+			 */
+			mp_rendezvous_no_intrs(machine_halt_cpu, NULL);
+			/* NOT REACHED */
+		}
+        }
 
 	__asm__("int3");
+	hw_atomic_sub(&debug_mode, 1);   
 }
 
 void
 enable_bluebox(void)
 {
 }
+
 void
 disable_bluebox(void)
 {
@@ -316,3 +802,257 @@ machine_boot_info(char *buf, __unused vm_size_t size)
 	return buf;
 }
 
+
+struct pasc {
+    unsigned a: 7;
+    unsigned b: 7;
+    unsigned c: 7;
+    unsigned d: 7;
+    unsigned e: 7;
+    unsigned f: 7;
+    unsigned g: 7;
+    unsigned h: 7;
+}  __attribute__((packed));
+
+typedef struct pasc pasc_t;
+
+static int packAsc (unsigned char *inbuf, unsigned int length)
+{
+  unsigned int i, j = 0;
+  unsigned int extra;
+  pasc_t pack;
+
+  for (i = 0; i < length; i+=8)
+    {
+      pack.a = inbuf[i];
+      pack.b = inbuf[i+1];
+      pack.c = inbuf[i+2];
+      pack.d = inbuf[i+3];
+      pack.e = inbuf[i+4];
+      pack.f = inbuf[i+5];
+      pack.g = inbuf[i+6];
+      pack.h = inbuf[i+7];
+      bcopy ((char *) &pack, inbuf + j, 7);
+      j += 7;
+    }
+  extra = (i - length);
+  if (extra > 0) {
+    inbuf[j - extra] &= (0xFF << (8-extra));
+  }
+  return j-((extra == 7) ? 6 : extra);
+}
+
+/* Routines for address - symbol translation. Not called unless the "keepsyms"
+ * boot-arg is supplied.
+ */
+
+static int
+panic_print_macho_symbol_name(struct mach_header *mh, vm_address_t search)
+{
+    struct nlist			*sym = NULL;
+    struct load_command		*cmd;
+    struct segment_command	*orig_ts = NULL, *orig_le = NULL;
+    struct symtab_command	*orig_st = NULL;
+    unsigned int			i;
+    char					*strings, *bestsym = NULL;
+    vm_address_t			bestaddr = 0, diff, curdiff;
+    
+    if (mh->magic != MH_MAGIC) {
+        /* bad magic number */
+        return 0;
+    }
+    
+    cmd = (struct load_command *) &mh[1];
+    for (i = 0; i < mh->ncmds; i++) {
+        if (cmd->cmd == LC_SEGMENT) {
+            struct segment_command *orig_sg = (struct segment_command *) cmd;
+            
+            if (strcmp(SEG_TEXT, orig_sg->segname) == 0)
+                orig_ts = orig_sg;
+            else if (strcmp(SEG_LINKEDIT, orig_sg->segname) == 0)
+                orig_le = orig_sg;
+            else if (strcmp("", orig_sg->segname) == 0)
+                orig_ts = orig_sg; /* kexts have a single unnamed segment */
+        }
+        else if (cmd->cmd == LC_SYMTAB)
+            orig_st = (struct symtab_command *) cmd;
+        
+        cmd = (struct load_command *) ((caddr_t) cmd + cmd->cmdsize);
+    }
+    
+    if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL))
+        return 0;
+    
+    /* kexts don't have a LINKEDIT segment for now, so we'll never get this far for kexts */
+    
+    vm_address_t slide = ((vm_address_t)mh) - orig_ts->vmaddr;
+    if (slide != 0)
+        search -= slide; /* adjusting search since the binary has slid */
+    
+    if ((search < orig_ts->vmaddr) ||
+        (search >= orig_ts->vmaddr + orig_ts->vmsize)) {
+        /* search out of range for this mach header */
+        return 0;
+    }
+    
+    sym = (struct nlist *)orig_le->vmaddr;
+    strings = ((char *)sym) + orig_st->nsyms * sizeof(struct nlist);
+    diff = search;
+    
+    for (i = 0; i < orig_st->nsyms; i++) {
+        if (sym[i].n_value <= search) {
+            curdiff = search - (vm_address_t)sym[i].n_value;
+            if (curdiff < diff) {
+                diff = curdiff;
+                bestaddr = sym[i].n_value;
+                bestsym = strings + sym[i].n_un.n_strx;
+            }
+        }
+    }
+    
+    if (bestsym != NULL) {
+        if (diff != 0) {
+            kdb_printf("%s + 0x%08x ", bestsym, diff);
+        } else {
+            kdb_printf("%s ", bestsym);
+        }
+        return 1;
+    }
+    return 0;
+}
+
+extern kmod_info_t * kmod; /* the list of modules */
+
+static void
+panic_print_kmod_symbol_name(vm_address_t search)
+{
+    kmod_info_t *			current_kmod = kmod;
+    
+    while (current_kmod != NULL) {
+        if ((current_kmod->address <= search) &&
+            (current_kmod->address + current_kmod->size > search))
+            break;
+        current_kmod = current_kmod->next;
+    }
+    if (current_kmod != NULL) {
+        /* if kexts had symbol table loaded, we'd call search_symbol_name again; alas, they don't */
+        kdb_printf("%s + %d ", current_kmod->name, search - current_kmod->address);
+    }
+}
+
+extern struct mach_header _mh_execute_header; /* the kernel's mach header */
+
+static void
+panic_print_symbol_name(vm_address_t search)
+{
+    /* try searching in the kernel */
+    if (panic_print_macho_symbol_name(&_mh_execute_header, search) == 0) {
+        /* that failed, now try to search for the right kext */
+        panic_print_kmod_symbol_name(search);
+    }
+}
+
+/* Generate a backtrace, given a frame pointer - this routine
+ * should walk the stack safely. The trace is appended to the panic log
+ * and conditionally, to the console. If the trace contains kernel module
+ * addresses, display the module name, load address and dependencies.
+ */
+
+#define DUMPFRAMES 32
+#define PBT_TIMEOUT_CYCLES (5 * 1000 * 1000 * 1000ULL)
+void
+panic_i386_backtrace(void *_frame, int nframes)
+{
+	cframe_t	*frame = (cframe_t *)_frame;
+	vm_offset_t raddrs[DUMPFRAMES];
+	int frame_index;
+	volatile uint32_t *ppbtcnt = &pbtcnt;
+	uint64_t bt_tsc_timeout;
+	boolean_t keepsyms = FALSE;
+
+	if(pbtcpu != cpu_number()) {
+		hw_atomic_add(&pbtcnt, 1);
+		/* Spin on print backtrace lock, which serializes output
+		 * Continue anyway if a timeout occurs.
+		 */
+		hw_lock_to(&pbtlock, LockTimeOut*100);
+		pbtcpu = cpu_number();
+	}
+
+	PE_parse_boot_arg("keepsyms", &keepsyms);
+
+	kdb_printf("Backtrace, "
+	    "Format - Frame : Return Address (4 potential args on stack) ");
+
+	for (frame_index = 0; frame_index < nframes; frame_index++) {
+		vm_offset_t curframep = (vm_offset_t) frame;
+
+		if (!curframep)
+			break;
+
+		if (curframep & 0x3) {
+			kdb_printf("Unaligned frame\n");
+			goto invalid;
+		}
+
+		if (!kvtophys(curframep) ||
+		    !kvtophys(curframep + sizeof(cframe_t))) {
+			kdb_printf("No mapping exists for frame pointer\n");
+			goto invalid;
+		}
+
+		kdb_printf("\n0x%x : 0x%x ",
+		    frame, frame->caller);
+		if (frame_index < DUMPFRAMES)
+			raddrs[frame_index] = frame->caller;
+
+		if (kvtophys((vm_offset_t)&(frame->args[3])))
+			kdb_printf("(0x%x 0x%x 0x%x 0x%x) ",
+			    frame->args[0], frame->args[1],
+			    frame->args[2], frame->args[3]);
+
+		/* Display address-symbol translation only if the "keepsyms"
+		 * boot-arg is suppplied, since we unload LINKEDIT otherwise.
+		 * This routine is potentially unsafe; also, function
+		 * boundary identification is unreliable after a strip -x.
+		 */
+		if (keepsyms)
+			panic_print_symbol_name((vm_address_t)frame->caller);
+		
+		/* Stack grows downward */
+		if (frame->prev < frame) {
+			frame = frame->prev;
+			goto invalid;
+		}
+		frame = frame->prev;
+	}
+
+	if (frame_index >= nframes)
+		kdb_printf("\tBacktrace continues...\n");
+
+	goto out;
+
+invalid:
+	kdb_printf("Backtrace terminated-invalid frame pointer 0x%x\n",frame);
+out:
+
+	/* Identify kernel modules in the backtrace and display their
+	 * load addresses and dependencies. This routine should walk
+	 * the kmod list safely.
+	 */
+	if (frame_index)
+		kmod_dump((vm_offset_t *)&raddrs[0], frame_index);
+
+	kdb_printf("\nKernel version:\n%s\n\n",version);
+
+	/* Release print backtrace lock, to permit other callers in the
+	 * event of panics on multiple processors.
+	 */
+	hw_lock_unlock(&pbtlock);
+	hw_atomic_sub(&pbtcnt, 1);
+	/* Wait for other processors to complete output
+	 * Timeout and continue after PBT_TIMEOUT_CYCLES.
+	 */
+	bt_tsc_timeout = rdtsc64() + PBT_TIMEOUT_CYCLES;
+	while(*ppbtcnt && (rdtsc64() < bt_tsc_timeout));
+}
diff --git a/osfmk/i386/AT386/rtc.h b/osfmk/i386/AT386/rtc.h
deleted file mode 100644
index 30030a2db..000000000
--- a/osfmk/i386/AT386/rtc.h
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the License
- * may not be used to create, or enable the creation or redistribution of,
- * unlawful or unlicensed copies of an Apple operating system, or to
- * circumvent, violate, or enable the circumvention or violation of, any
- * terms of an Apple operating system software license agreement.
- * 
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- * 
- * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/*
- * HISTORY
- * 
- * Revision 1.1.1.1  1998/09/22 21:05:39  wsanchez
- * Import of Mac OS X kernel (~semeria)
- *
- * Revision 1.1.1.1  1998/03/07 02:25:40  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.1.6.1  1994/09/23  01:47:30  ezf
- * 	change marker to not FREE
- * 	[1994/09/22  21:20:22  ezf]
- *
- * Revision 1.1.2.3  1993/08/09  19:39:04  dswartz
- * 	Add ANSI prototypes - CR#9523
- * 	[1993/08/06  17:51:17  dswartz]
- * 
- * Revision 1.1.2.2  1993/06/02  23:21:32  jeffc
- * 	Added to OSF/1 R1.3 from NMK15.0.
- * 	[1993/06/02  21:03:17  jeffc]
- * 
- * Revision 1.1  1992/09/30  02:27:20  robert
- * 	Initial revision
- * 
- * $EndLog$
- */
-/* CMU_HIST */
-/*
- * Revision 2.7  91/05/14  16:30:03  mrt
- * 	Correcting copyright
- * 
- * Revision 2.6  91/03/16  14:47:03  rpd
- * 	Fixed ioctl definitions for ANSI C.
- * 	[91/02/20            rpd]
- * 
- * Revision 2.5  91/02/05  17:20:25  mrt
- * 	Changed to new Mach copyright
- * 	[91/02/01  17:47:16  mrt]
- * 
- * Revision 2.4  90/11/26  14:51:02  rvb
- * 	jsb bet me to XMK34, sigh ...
- * 	[90/11/26            rvb]
- * 	Synched 2.5 & 3.0 at I386q (r1.5.1.3) & XMK35 (r2.4)
- * 	[90/11/15            rvb]
- * 
- * Revision 1.5.1.2  90/07/27  11:27:06  rvb
- * 	Fix Intel Copyright as per B. Davies authorization.
- * 	[90/07/27            rvb]
- * 
- * Revision 2.2  90/05/03  15:46:11  dbg
- * 	First checkin.
- * 
- * Revision 1.5.1.1  90/01/08  13:29:46  rvb
- * 	Add Intel copyright.
- * 	[90/01/08            rvb]
- * 
- * Revision 1.5  89/09/25  12:27:37  rvb
- * 	File was provided by Intel 9/18/89.
- * 	[89/09/23            rvb]
- * 
- */
-/* CMU_ENDHIST */
-/* 
- * Mach Operating System
- * Copyright (c) 1991,1990,1989 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
-/* 
- */
-
-/*
- *   Copyright 1988, 1989 by Intel Corporation, Santa Clara, California.
- * 
- * 		All Rights Reserved
- * 
- * Permission to use, copy, modify, and distribute this software and
- * its documentation for any purpose and without fee is hereby
- * granted, provided that the above copyright notice appears in all
- * copies and that both the copyright notice and this permission notice
- * appear in supporting documentation, and that the name of Intel
- * not be used in advertising or publicity pertaining to distribution
- * of the software without specific, written prior permission.
- * 
- * INTEL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
- * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS,
- * IN NO EVENT SHALL INTEL BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
- * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
- * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
- * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
- * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
- */
-
-#define RTC_ADDR	0x70	/* I/O port address for register select */
-#define RTC_DATA	0x71	/* I/O port address for data read/write */
-
-/*
- * Register A definitions
- */
-#define RTC_A		0x0a	/* register A address */
-#define RTC_UIP		0x80	/* Update in progress bit */
-#define RTC_DIV0	0x00	/* Time base of 4.194304 MHz */
-#define RTC_DIV1	0x10	/* Time base of 1.048576 MHz */
-#define RTC_DIV2	0x20	/* Time base of 32.768 KHz */
-#define RTC_RATE6	0x06	/* interrupt rate of 976.562 */
-
-/*
- * Register B definitions
- */
-#define RTC_B		0x0b	/* register B address */
-#define RTC_SET		0x80	/* stop updates for time set */
-#define RTC_PIE		0x40	/* Periodic interrupt enable */
-#define RTC_AIE		0x20	/* Alarm interrupt enable */
-#define RTC_UIE		0x10	/* Update ended interrupt enable */
-#define RTC_SQWE	0x08	/* Square wave enable */
-#define RTC_DM		0x04	/* Date mode, 1 = binary, 0 = BCD */
-#define RTC_HM		0x02	/* hour mode, 1 = 24 hour, 0 = 12 hour */
-#define RTC_DSE		0x01	/* Daylight savings enable */
-
-/* 
- * Register C definitions
- */
-#define RTC_C		0x0c	/* register C address */
-#define RTC_IRQF	0x80	/* IRQ flag */
-#define RTC_PF		0x40	/* PF flag bit */
-#define RTC_AF		0x20	/* AF flag bit */
-#define RTC_UF		0x10	/* UF flag bit */
-
-/*
- * Register D definitions
- */
-#define RTC_D		0x0d	/* register D address */
-#define RTC_VRT		0x80	/* Valid RAM and time bit */
-
-#define RTC_NREG	0x0e	/* number of RTC registers */
-#define RTC_NREGP	0x0a	/* number of RTC registers to set time */
-
-#define RTCRTIME	_IOR('c', 0x01, struct rtc_st) /* Read time from RTC */
-#define RTCSTIME	_IOW('c', 0x02, struct rtc_st) /* Set time into RTC */
-
-struct rtc_st {
-	char	rtc_sec;
-	char	rtc_asec;
-	char	rtc_min;
-	char	rtc_amin;
-	char	rtc_hr;
-	char	rtc_ahr;
-	char	rtc_dow;
-	char	rtc_dom;
-	char	rtc_mon;
-	char	rtc_yr;
-	char	rtc_statusa;
-	char	rtc_statusb;
-	char	rtc_statusc;
-	char	rtc_statusd;
-};
-
-/*
- * this macro reads contents of real time clock to specified buffer 
- */
-#define load_rtc(regs) \
-{\
-	register int i; \
-	\
-	for (i = 0; i < RTC_NREG; i++) { \
-		outb(RTC_ADDR, i); \
-		(regs)[i] = inb(RTC_DATA); \
-	} \
-}
-
-/*
- * this macro writes contents of specified buffer to real time clock 
- */ 
-#define save_rtc(regs) \
-{ \
-	register int i; \
-	for (i = 0; i < RTC_NREGP; i++) { \
-		outb(RTC_ADDR, i); \
-		outb(RTC_DATA, (regs)[i]);\
-	} \
-}	
-
-
diff --git a/osfmk/i386/Diagnostics.c b/osfmk/i386/Diagnostics.c
new file mode 100644
index 000000000..911e5e437
--- /dev/null
+++ b/osfmk/i386/Diagnostics.c
@@ -0,0 +1,166 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_FREE_COPYRIGHT@
+ */
+/*
+ * @APPLE_FREE_COPYRIGHT@
+ */
+
+/*
+ *	Author: Bill Angell, Apple
+ *	Date:	10/auht-five
+ *
+ *	Random diagnostics
+ *
+ *	Try to keep the x86 selectors in-sync with the ppc selectors.
+ *
+ */
+
+
+#include <kern/machine.h>
+#include <kern/processor.h>
+#include <mach/machine.h>
+#include <mach/processor_info.h>
+#include <mach/mach_types.h>
+#include <mach/boolean.h>
+#include <kern/thread.h>
+#include <kern/task.h>
+#include <kern/ipc_kobject.h>
+#include <mach/vm_param.h>
+#include <ipc/port.h>
+#include <ipc/ipc_entry.h>
+#include <ipc/ipc_space.h>
+#include <ipc/ipc_object.h>
+#include <ipc/ipc_port.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_map.h>
+#include <vm/vm_page.h>
+#include <vm/pmap.h>
+#include <pexpert/pexpert.h>
+#include <console/video_console.h>
+#include <i386/cpu_data.h>
+#include <i386/Diagnostics.h>
+#include <i386/mp.h>
+#include <i386/pmCPU.h>
+#include <i386/tsc.h>
+#include <i386/hpet.h>
+#include <mach/i386/syscall_sw.h>
+
+extern uint64_t lastNapClear;
+
+diagWork        dgWork;
+uint64_t        lastNapClear = 0ULL;
+uint64_t        lastRuptClear = 0ULL;
+
+typedef struct pmdata {
+	uint64_t        pmNapDur;	/* Time since last query */
+	pmStats_t       pmd;	/* Powermanagement statistics */
+}               pmdata;
+
+
+
+int 
+diagCall64(__unused x86_saved_state_t * regs)
+{
+        panic("diagCall not yet supported for 64 bit tasks\n");
+}
+
+
+int 
+diagCall(x86_saved_state_t * state)
+{
+
+	uint32_t        stk, curpos, i, j;
+	uint32_t        selector, data;
+	int             err;
+	uint64_t        currNap, durNap;
+	x86_saved_state32_t	*regs;
+
+	assert(is_saved_state32(state));
+	regs = saved_state32(state);
+
+	if (!(dgWork.dgFlags & enaDiagSCs))
+		return 0;	/* If not enabled, cause an exception */
+
+	stk = regs->uesp;	/* Point to the stack */
+	err = copyin((user_addr_t) (stk + 4), (char *) &selector, sizeof(uint32_t));	/* Get the selector */
+	if (err) {
+		return 0;	/* Failed to fetch stack */
+	}
+	switch (selector) {	/* Select the routine */
+
+	case dgRuptStat:	/* Suck Interruption statistics */
+
+		err = copyin((user_addr_t) (stk + 8), (char *) &data, sizeof(uint32_t));	/* Get the selector */
+
+		if (data == 0) {/* If number of processors is 0, clear all
+				 * counts */
+			for (i = 0; i < real_ncpus; i++) {	/* Cycle through
+								 * processors */
+				for (j = 0; j < 256; j++)
+					cpu_data_ptr[i]->cpu_hwIntCnt[j] = 0;
+			}
+
+			lastRuptClear = mach_absolute_time();	/* Get the time of clear */
+			return 1;	/* Normal return */
+		}
+		err = copyin((user_addr_t) (stk + 8), (char *) &data, sizeof(uint32_t));	/* Get the selector */
+
+		(void) copyout((char *) &real_ncpus, data, sizeof(real_ncpus));	/* Copy out number of
+										 * processors */
+
+		currNap = mach_absolute_time();	/* Get the time now */
+		durNap = currNap - lastRuptClear;	/* Get the last interval
+							 * duration */
+		if (durNap == 0)
+			durNap = 1;	/* This is a very short time, make it
+					 * bigger */
+
+		curpos = data + sizeof(real_ncpus);	/* Point to the next
+							 * available spot */
+
+		for (i = 0; i < real_ncpus; i++) {	/* Move 'em all out */
+			(void) copyout((char *) &durNap, curpos, 8);	/* Copy out the time
+									 * since last clear */
+			(void) copyout((char *) &cpu_data_ptr[i]->cpu_hwIntCnt, curpos + 8, 256 * sizeof(uint32_t));	/* Copy out interrupt
+															 * data for this
+															 * processor */
+			curpos = curpos + (256 * sizeof(uint32_t) + 8);	/* Point to next out put
+									 * slot */
+		}
+
+		break;
+
+	default:		/* Handle invalid ones */
+		return 0;	/* Return an exception */
+
+	}
+
+	return 1;		/* Normal non-ast check return */
+}
diff --git a/osfmk/i386/Diagnostics.h b/osfmk/i386/Diagnostics.h
new file mode 100644
index 000000000..c8d385c7b
--- /dev/null
+++ b/osfmk/i386/Diagnostics.h
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_FREE_COPYRIGHT@
+ */
+/*
+ * @APPLE_FREE_COPYRIGHT@
+ */
+
+/*
+ *	Here are the Diagnostic interface interfaces
+ *	Lovingly crafted by Bill Angell using traditional methods
+ *	Keep selectors in sync with the PPC version where possible.	
+ */
+#ifdef	KERNEL_PRIVATE
+
+#ifndef _DIAGNOSTICS_H_
+#define _DIAGNOSTICS_H_
+
+#ifdef __ppc__
+#error This file is not useful on PowerPC.
+#endif
+
+int diagCall(x86_saved_state_t *regs);
+int diagCall64(x86_saved_state_t *regs);
+
+#define diagSCnum 0x00006000
+
+#define dgAdjTB 0
+#define dgLRA 1
+#define dgpcpy 2
+#define dgreset 3
+#define dgtest 4
+#define dgBMphys 5
+#define dgUnMap 6
+#define dgBootScreen 7
+#define dgFlush 8
+#define dgAlign 9
+#define dgprw 10
+#define dgmck 11
+#define dg64 12
+#define dgProbeRead 13
+#define dgCPNull 14
+#define dgPerfMon 15
+#define dgMapPage 16
+#define dgScom 17
+#define dgBind 18
+#define dgPproc 19
+#define dgAcntg 20
+#define dgKlra 21
+#define dgKfree 22
+#define	dgWar 23
+#define dgNapStat 24
+#define dgRuptStat 25
+
+
+typedef struct diagWork {			/* Diagnostic work area */
+
+	unsigned int dgLock;			/* Lock if needed */
+	unsigned int dgFlags;			/* Flags */
+#define enaExpTrace 0x00000001
+#define enaExpTraceb 31
+#define enaUsrFCall 0x00000002
+#define enaUsrFCallb 30
+#define enaUsrPhyMp 0x00000004
+#define enaUsrPhyMpb 29
+#define enaDiagSCs  0x00000008
+#define enaDiagSCsb  28
+#define enaDiagDM  0x00000010
+#define enaDiagSDMb  27
+#define enaDiagEM  0x00000020
+#define enaDiagEMb  26
+#define enaDiagTrap  0x00000040
+#define enaDiagTrapb  25
+#define enaNotifyEM  0x00000080
+#define enaNotifyEMb  24
+	
+	unsigned int dgMisc0;
+	unsigned int dgMisc1;
+	unsigned int dgMisc2;
+	unsigned int dgMisc3;
+	unsigned int dgMisc4;
+	unsigned int dgMisc5;
+
+} diagWork;
+
+extern diagWork dgWork;
+
+
+#endif /* _DIAGNOSTICS_H_ */
+
+#endif /* KERNEL_PRIVATE */
diff --git a/osfmk/i386/Makefile b/osfmk/i386/Makefile
index ae6a4000d..2ce73da27 100644
--- a/osfmk/i386/Makefile
+++ b/osfmk/i386/Makefile
@@ -9,9 +9,13 @@ include $(MakeInc_def)
 
 EXPORT_ONLY_FILES = 	\
 		    apic.h \
+		    asm.h \
 		    cpu_number.h \
 		    cpu_capabilities.h	\
+		    cpu_data.h \
 		    cpuid.h \
+		    eflags.h \
+		    hpet.h \
 		    io_map_entries.h \
 		    lock.h \
 		    locks.h \
@@ -21,13 +25,20 @@ EXPORT_ONLY_FILES = 	\
 		    mp.h \
 		    mp_desc.h \
 		    mp_events.h \
+		    pmCPU.h \
+		    pmap.h \
 		    proc_reg.h \
+		    rtclock.h \
 		    seg.h \
 		    simple_lock.h \
+		    tsc.h \
 		    tss.h
 
 INSTALL_MD_DIR = i386
 
+INSTALL_MD_LIST = eflags.h \
+		  user_ldt.h
+
 INSTALL_MD_LCL_LIST = cpu_capabilities.h
 
 EXPORT_MD_LIST = ${EXPORT_ONLY_FILES}
diff --git a/osfmk/i386/acpi.c b/osfmk/i386/acpi.c
index 006fe9d9b..a70821010 100644
--- a/osfmk/i386/acpi.c
+++ b/osfmk/i386/acpi.c
@@ -31,7 +31,10 @@
 #include <i386/pmap.h>
 #include <i386/mtrr.h>
 #include <i386/acpi.h>
+#include <i386/fpu.h>
 #include <i386/mp.h>
+#include <i386/mp_desc.h>
+#include <i386/machine_check.h>
 
 #include <kern/cpu_data.h>
 
@@ -47,11 +50,13 @@ extern unsigned int disableSerialOuput;
 
 extern void        set_kbd_leds(int leds);
 
+extern void 	fpinit(void);
+
 vm_offset_t
 acpi_install_wake_handler(void)
 {
 	/* copy wake code to ACPI_WAKE_ADDR in low memory */
-	bcopy_phys((addr64_t) kvtophys((vm_offset_t)acpi_wake_start),
+	bcopy_phys(kvtophys((vm_offset_t)acpi_wake_start),
 		   (addr64_t) ACPI_WAKE_ADDR,
 		   acpi_wake_end - acpi_wake_start);
 
@@ -70,12 +75,28 @@ typedef struct acpi_hibernate_callback_data {
 static void
 acpi_hibernate(void *refcon)
 {
-    boolean_t hib;
+    boolean_t dohalt;
 
     acpi_hibernate_callback_data *data = (acpi_hibernate_callback_data *)refcon;
 
     if (current_cpu_datap()->cpu_hibernate) {
-        hib = hibernate_write_image();
+
+        dohalt = hibernate_write_image();
+	if (dohalt)
+	{
+	    // off
+	    HIBLOG("power off\n");
+	    if (PE_halt_restart) 
+		(*PE_halt_restart)(kPEHaltCPU);
+	}
+	else
+	{
+	    // sleep
+	    HIBLOG("sleep\n");
+
+	    // should we come back via regular wake, set the state in memory.
+	    cpu_datap(0)->cpu_hibernate = 0;
+	}
     }
 
     (data->func)(data->refcon);
@@ -89,51 +110,85 @@ acpi_sleep_kernel(acpi_sleep_callback func, void *refcon)
     acpi_hibernate_callback_data data;
     boolean_t did_hibernate;
 
-	/* shutdown local APIC before passing control to BIOS */
-	lapic_shutdown();
+    kprintf("acpi_sleep_kernel hib=%d\n", current_cpu_datap()->cpu_hibernate);
+
+    /* shutdown local APIC before passing control to BIOS */
+    lapic_shutdown();
 
     data.func = func;
     data.refcon = refcon;
 
-	/*
-	 * Save master CPU state and sleep platform.
-	 * Will not return until platform is woken up,
-	 * or if sleep failed.
-	 */
+    /* Save HPET state */
+    hpet_save();
+
+    /*
+     * If we're in 64-bit mode, drop back into legacy mode during sleep.
+     */
+    if (cpu_mode_is64bit()) {
+	cpu_IA32e_disable(current_cpu_datap());
+	kprintf("acpi_sleep_kernel legacy mode re-entered\n");
+    }
+
+    /*
+     * Save master CPU state and sleep platform.
+     * Will not return until platform is woken up,
+     * or if sleep failed.
+     */
     acpi_sleep_cpu(acpi_hibernate, &data);
 
-	/* reset UART if kprintf is enabled */
-	if (FALSE == disableSerialOuput)
-		serial_init();
+    /* reset UART if kprintf is enabled */
+    if (FALSE == disableSerialOuput)
+	    serial_init();
+
+    kprintf("ret from acpi_sleep_cpu hib=%d\n", current_cpu_datap()->cpu_hibernate);
 
     if (current_cpu_datap()->cpu_hibernate) {
-        * (int *) CM1 = 0;
-        * (int *) CM2 = 0;
-        * (int *) CM3 = 0;
+	int i;
+	for (i=0; i<PMAP_NWINDOWS; i++) {
+	    *current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CMAP = 0;
+	}
+	current_cpu_datap()->cpu_hibernate = 0;
+	did_hibernate = TRUE;
 
-        current_cpu_datap()->cpu_hibernate = 0;
+    } else {
+	did_hibernate = FALSE;
+    }
 
-        did_hibernate = TRUE;
+    /* Re-enable 64-bit mode if necessary. */
+    if (cpu_mode_is64bit()) {
+	cpu_IA32e_enable(current_cpu_datap());
+	cpu_desc_load64(current_cpu_datap());
+	kprintf("acpi_sleep_kernel 64-bit mode re-enabled\n");
+	fast_syscall_init64();
     } else {
-        did_hibernate = FALSE;
+	fast_syscall_init();
     }
 
-	/* restore MTRR settings */
-	mtrr_update_cpu();
+    /* Re-enable machine check handling */
+    mca_cpu_init();
 
-	/* set up PAT following boot processor power up */
-	pat_init();
+    /* restore MTRR settings */
+    mtrr_update_cpu();
+
+    /* set up PAT following boot processor power up */
+    pat_init();
 
     if (did_hibernate) {
         hibernate_machine_init();
     }
         
-	/* re-enable and re-init local apic */
-	if (lapic_probe())
-		lapic_init();
+    /* re-enable and re-init local apic */
+    if (lapic_probe())
+	lapic_init();
+
+    /* Restore HPET state */
+    hpet_restore();
+
+    /* let the realtime clock reset */
+    rtc_sleep_wakeup();
 
-	/* let the realtime clock reset */
-	rtc_sleep_wakeup();
+    fpinit();
+    clear_fpu();
 
     if (did_hibernate) {
         enable_preemption();
diff --git a/osfmk/i386/acpi_wakeup.s b/osfmk/i386/acpi_wakeup.s
index 80ae08ad3..0d20c8063 100644
--- a/osfmk/i386/acpi_wakeup.s
+++ b/osfmk/i386/acpi_wakeup.s
@@ -42,7 +42,7 @@
 	.long	address - EXT(acpi_wake_start)	;\
 	.word	segment
 
-#define	PA(addr)	((addr)-KERNELBASE)
+#define	PA(addr)	(addr)
 
 /*
  * acpi_wake_start
@@ -151,6 +151,10 @@ ENTRY(acpi_sleep_cpu)
 	pushal
 	movl	%esp, saved_esp
 
+	/* make sure tlb is flushed */
+	movl	%cr3,%eax
+	movl	%eax,%cr3
+	
 	/* save control registers */
 	movl	%cr0, %eax
 	movl	%eax, saved_cr0
@@ -192,43 +196,27 @@ ENTRY(acpi_sleep_cpu)
 	jmp	wake_restore
 
 wake_prot:
-
 	/* protected mode, paging disabled */
 	POSTCODE(ACPI_WAKE_PROT_ENTRY)
 
+	movl	PA(saved_cr3), %ebx
+	movl	PA(saved_cr4), %ecx
+	/*
+	 * restore cr3, PAE and NXE states in an orderly fashion
+	 */
+	movl	%ebx, %cr3
+	movl	%ecx, %cr4
+
+	movl	$(MSR_IA32_EFER), %ecx			/* MSR number in ecx */
+	rdmsr						/* MSR value return in edx: eax */
+	orl	$(MSR_IA32_EFER_NXE), %eax		/* Set NXE bit in low 32-bits */
+	wrmsr						/* Update Extended Feature Enable reg */
+
 	/* restore kernel GDT */
 	lgdt	PA(saved_gdt)
 
-	/* restore control registers */
 	movl	PA(saved_cr2), %eax
 	movl	%eax, %cr2
-	
-#ifdef PAE
-	movl	PA(EXT(IdlePDPT)), %eax
-	movl	(%eax), %esi		/* save orig */
-	movl	24(%eax), %ebx
-	movl	%ebx, (%eax)	/* identity map low mem */
-	movl	%eax, %cr3
-
-	movl	PA(saved_cr4), %eax
-	movl	%eax, %cr4
-#else
-	movl	PA(saved_cr4), %eax
-	movl	%eax, %cr4
-
-	/*
-	 * Temporarily use the page tables at IdlePTD
-	 * to enable paging.  Copy the KPTDI entry to
-	 * entry 0 in the PTD to identity map the kernel.
-	 */
-	movl	PA(EXT(IdlePTD)), %eax
-	movl	%eax, %ebx
-	addl	$(KPTDI << PTEINDX), %ebx  /* bytes per PDE */
-	movl	(%ebx), %ebx		/* IdlePTD[KPTDI]  */
-	movl	(%eax), %esi		/* save original IdlePTD[0] */
-	movl	%ebx, (%eax)		/* update IdlePTD[0] */   
-	movl	%eax, %cr3		/* CR3 = IdlePTD */
-#endif
 
 	/* restore CR0, paging enabled */
 	movl	PA(saved_cr0), %eax
@@ -246,20 +234,6 @@ wake_paged:
 	movw	$(KERNEL_DS), %ax
 	movw	%ax, %ds
 
-	/* undo changes to IdlePTD */
-#ifdef PAE
-	movl	EXT(IdlePDPT), %eax
-#else	
-	movl	EXT(IdlePTD), %eax
-#endif
-	addl	$(KERNELBASE), %eax	/* make virtual */
-	movl	%esi, (%eax)
-
-	/* restore real PDE base */
-	movl	saved_cr3, %eax
-	movl	%eax, %cr3
-
-
 	/* restore local and interrupt descriptor tables */
 	lldt	saved_ldt
 	lidt	saved_idt
@@ -299,29 +273,36 @@ wake_restore:
         .globl EXT(acpi_wake_prot_entry)
 ENTRY(acpi_wake_prot_entry)
 	/* protected mode, paging enabled */
+
 	POSTCODE(ACPI_WAKE_PAGED_ENTRY)
 
 	/* restore kernel GDT */
-	lgdt	PA(saved_gdt)
-	
+	lgdt	saved_gdt
+
 	POSTCODE(0x40)
+
 	/* restore control registers */
+
+	movl	saved_cr0, %eax
+	movl	%eax, %cr0
+
 	movl	saved_cr2, %eax
 	movl	%eax, %cr2
 
         POSTCODE(0x3E)
-	/* switch to kernel data segment */
-	movw	$(KERNEL_DS), %ax
-	movw	%ax, %ds
 
-        POSTCODE(0x3D)
 	/* restore real PDE base */
 	movl	saved_cr3, %eax
 	movl	saved_cr4, %edx
 	movl	%eax, %cr3
 	movl	%edx, %cr4
+	movl	%eax, %cr3
 
-        POSTCODE(0x3C)
+	/* switch to kernel data segment */
+	movw	$(KERNEL_DS), %ax
+	movw	%ax, %ds
+
+	POSTCODE(0x3C)
 	/* restore local and interrupt descriptor tables */
 	lldt	saved_ldt
 	lidt	saved_idt
@@ -368,6 +349,7 @@ ENTRY(acpi_wake_prot_entry)
 /*
  * CPU registers saved across sleep/wake.
  */
+
 saved_esp:	.long 0
 saved_es:	.word 0
 saved_fs:	.word 0
diff --git a/osfmk/i386/asm.h b/osfmk/i386/asm.h
index 20d579ef5..363bdeee5 100644
--- a/osfmk/i386/asm.h
+++ b/osfmk/i386/asm.h
@@ -78,6 +78,7 @@
 #define S_ARG1	 8(%esp)
 #define S_ARG2	12(%esp)
 #define S_ARG3	16(%esp)
+#define S_ARG4	20(%esp)
 
 #define FRAME	pushl %ebp; movl %esp, %ebp
 #define EMARF	leave
@@ -91,7 +92,7 @@
 
 /* There is another definition of ALIGN for .c sources */
 #ifdef ASSEMBLER
-#define ALIGN 2,0x90
+#define ALIGN 4,0x90
 #endif /* ASSEMBLER */
 
 #ifndef FALIGN
diff --git a/osfmk/i386/asm64.h b/osfmk/i386/asm64.h
new file mode 100644
index 000000000..bb1a51c0b
--- /dev/null
+++ b/osfmk/i386/asm64.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/* Helper macros for 64-bit mode switching */
+
+/*
+ * Long jump to 64-bit space from 32-bit compatibility mode.
+ */
+#define	ENTER_64BIT_MODE()			\
+	.code32					;\
+	.byte   0xea	/* far jump longmode */	;\
+	.long   1f				;\
+	.word   KERNEL64_CS			;\
+        .code64					;\
+1:
+
+/*
+ * Here in long mode but still running below 4G.
+ * "Near" jump into uber-space.
+ */
+#define	ENTER_UBERSPACE()			\
+        mov     2f,%rax				;\
+        jmp     *%rax				;\
+2:      .long   3f				;\
+        .long   KERNEL_UBER_BASE_HI32		;\
+3:     
+
+/*
+ * Long jump to 32-bit compatibility mode from 64-bit space.
+ */
+#define ENTER_COMPAT_MODE()			\
+	ljmp	*4f				;\
+4:	.long	5f				;\
+	.word	KERNEL_CS			;\
+	.code32					;\
+5:
+
diff --git a/osfmk/i386/ast.h b/osfmk/i386/ast.h
index 5b98d83f9..efa1f6b35 100644
--- a/osfmk/i386/ast.h
+++ b/osfmk/i386/ast.h
@@ -62,13 +62,6 @@
 /*
  * Machine-dependent AST file for machines with no hardware AST support.
  *
- * For the I386, we define AST_I386_FP to handle delayed
- * floating-point exceptions.  The FPU may interrupt on errors
- * while the user is not running (in kernel or other thread running).
  */
 
-#define	AST_I386_FP	0x80000000
-#define MACHINE_AST_PER_THREAD  AST_I386_FP
-
-
 #endif	/* _I386_AST_H_ */
diff --git a/osfmk/i386/bsd_i386.c b/osfmk/i386/bsd_i386.c
index 6568929ed..896ebd976 100644
--- a/osfmk/i386/bsd_i386.c
+++ b/osfmk/i386/bsd_i386.c
@@ -65,12 +65,16 @@
 #include <i386/cpu_number.h>
 #include <i386/mp_desc.h>
 #include <i386/vmparam.h>
+#include <i386/trap.h>
+#include <mach/i386/syscall_sw.h>
 #include <sys/syscall.h>
 #include <sys/kdebug.h>
 #include <sys/ktrace.h>
+#include <sys/errno.h>
 #include <../bsd/sys/sysent.h>
 
 extern struct proc *current_proc(void);
+extern struct proc * kernproc;
 
 kern_return_t
 thread_userstack(
@@ -79,7 +83,7 @@ thread_userstack(
     thread_state_t,
     unsigned int,
     mach_vm_offset_t *,
-	int *
+    int *
 );
 
 kern_return_t
@@ -91,6 +95,8 @@ thread_entrypoint(
     mach_vm_offset_t *
 ); 
 
+void * find_user_regs(thread_t);
+
 unsigned int get_msr_exportmask(void);
 
 unsigned int get_msr_nbits(void);
@@ -110,110 +116,99 @@ void IOSleep(int);
  */
 kern_return_t
 thread_userstack(
-    __unused thread_t   thread,
-    int                 flavor,
-    thread_state_t      tstate,
-    unsigned int        count,
-    user_addr_t    *user_stack,
-	int					*customstack
-)
+		 __unused thread_t   thread,
+		 int                 flavor,
+		 thread_state_t      tstate,
+		 __unused unsigned int        count,
+		 user_addr_t    *user_stack,
+		 int		*customstack
+		 )
 {
-        struct i386_saved_state *state;
-	i386_thread_state_t *state25;
-	vm_offset_t	uesp;
-
         if (customstack)
-			*customstack = 0;
+	        *customstack = 0;
 
         switch (flavor) {
-	case i386_THREAD_STATE:	/* FIXME */
-                state25 = (i386_thread_state_t *) tstate;
+	    case OLD_i386_THREAD_STATE:
+	    case x86_THREAD_STATE32:
+	    {
+	        x86_thread_state32_t *state25;
+
+                state25 = (x86_thread_state32_t *) tstate;
+
 		if (state25->esp)
 			*user_stack = state25->esp;
 		else 
-			*user_stack = USRSTACK;
+			*user_stack = VM_USRSTACK32;
 		if (customstack && state25->esp)
 			*customstack = 1;
 		else
 			*customstack = 0;
 		break;
+	    }
 
-        case i386_NEW_THREAD_STATE:
-                if (count < i386_NEW_THREAD_STATE_COUNT)
-                        return (KERN_INVALID_ARGUMENT);
-		else {
-                	state = (struct i386_saved_state *) tstate;
-			uesp = state->uesp;
-    		}
+	    case x86_THREAD_STATE64:
+	    {
+	        x86_thread_state64_t *state25;
+
+                state25 = (x86_thread_state64_t *) tstate;
 
-                /* If a valid user stack is specified, use it. */
-		if (uesp)
-			*user_stack = uesp;
+		if (state25->rsp)
+			*user_stack = state25->rsp;
 		else 
-			*user_stack = USRSTACK;
-		if (customstack && uesp)
+			*user_stack = VM_USRSTACK64;
+		if (customstack && state25->rsp)
 			*customstack = 1;
 		else
 			*customstack = 0;
-                break;
-        default :
+		break;
+	    }
+
+	    default :
                 return (KERN_INVALID_ARGUMENT);
         }
                 
         return (KERN_SUCCESS);
 }    
 
+
 kern_return_t
 thread_entrypoint(
-    __unused thread_t   thread,
-    int                 flavor,
-    thread_state_t      tstate,
-    unsigned int        count,
-    mach_vm_offset_t    *entry_point
-)
+		  __unused thread_t   thread,
+		  int                 flavor,
+		  thread_state_t      tstate,
+		  __unused unsigned int        count,
+		  mach_vm_offset_t    *entry_point
+		  )
 { 
-    struct i386_saved_state	*state;
-    i386_thread_state_t *state25;
-
-    /*
-     * Set a default.
-     */
-    if (*entry_point == 0)
-	*entry_point = VM_MIN_ADDRESS;
+        /*
+	 * Set a default.
+	 */
+        if (*entry_point == 0)
+	        *entry_point = VM_MIN_ADDRESS;
 		
-    switch (flavor) {
-    case i386_THREAD_STATE:
-	state25 = (i386_thread_state_t *) tstate;
-	*entry_point = state25->eip ? state25->eip: VM_MIN_ADDRESS;
-	break;
-
-    case i386_NEW_THREAD_STATE:
-	if (count < i386_THREAD_STATE_COUNT)
-	    return (KERN_INVALID_ARGUMENT);
-	else {
-		state = (struct i386_saved_state *) tstate;
-
-		/*
-	 	* If a valid entry point is specified, use it.
-	 	*/
-		*entry_point = state->eip ? state->eip: VM_MIN_ADDRESS;
-	}
-	break;
-    }
+	switch (flavor) {
+	    case OLD_i386_THREAD_STATE:
+	    case x86_THREAD_STATE32:
+	    {
+	        x86_thread_state32_t *state25;
+
+		state25 = (x86_thread_state32_t *) tstate;
+		*entry_point = state25->eip ? state25->eip: VM_MIN_ADDRESS;
+		break;
+	    }
 
+	    case x86_THREAD_STATE64:
+	    {
+	        x86_thread_state64_t *state25;
+
+		state25 = (x86_thread_state64_t *) tstate;
+		*entry_point = state25->rip ? state25->rip: VM_MIN_ADDRESS64;
+		break;
+	    }
+    }
     return (KERN_SUCCESS);
 }   
 
-struct i386_saved_state *
-get_user_regs(thread_t th)
-{
-	if (th->machine.pcb)
-		return(USER_REGS(th));
-	else {
-		printf("[get_user_regs: thread does not have pcb]");
-		return NULL;
-	}
-}
 
 /*
  * Duplicate parent state in child
@@ -225,43 +220,49 @@ machine_thread_dup(
     thread_t		child
 )
 {
-	struct i386_float_state		floatregs;
-
-#ifdef	XXX
-	/* Save the FPU state */
-	if ((pcb_t)(per_proc_info[cpu_number()].fpu_pcb) == parent->machine.pcb) {
-		fp_state_save(parent);
-	}
-#endif
+	
+	pcb_t		parent_pcb;
+	pcb_t		child_pcb;
 
-	if (child->machine.pcb == NULL || parent->machine.pcb == NULL)
+	if ((child_pcb = child->machine.pcb) == NULL ||
+	    (parent_pcb = parent->machine.pcb) == NULL)
 		return (KERN_FAILURE);
+	/*
+	 * Copy over the i386_saved_state registers
+	 */
+	if (cpu_mode_is64bit()) {
+	        if (thread_is_64bit(parent))
+		        bcopy(USER_REGS64(parent), USER_REGS64(child), sizeof(x86_saved_state64_t));
+		else
+		        bcopy(USER_REGS32(parent), USER_REGS32(child), sizeof(x86_saved_state_compat32_t));
+	} else
+	        bcopy(USER_REGS32(parent), USER_REGS32(child), sizeof(x86_saved_state32_t));
 
-	/* Copy over the i386_saved_state registers */
-	child->machine.pcb->iss = parent->machine.pcb->iss;
-
-	/* Check to see if parent is using floating point
+	/*
+	 * Check to see if parent is using floating point
 	 * and if so, copy the registers to the child
-	 * FIXME - make sure this works.
 	 */
+        fpu_dup_fxstate(parent, child);
 
-	if (parent->machine.pcb->ims.ifps)  {
-		if (fpu_get_state(parent, &floatregs) == KERN_SUCCESS)
-			fpu_set_state(child, &floatregs);
-	}
-	
-	/* FIXME - should a user specified LDT, TSS and V86 info
+#ifdef	MACH_BSD
+	/*
+	 * Copy the parent's cthread id and USER_CTHREAD descriptor, if 32-bit.
+	 */
+	child_pcb->cthread_self = parent_pcb->cthread_self;
+	if (!thread_is_64bit(parent))
+		child_pcb->cthread_desc = parent_pcb->cthread_desc;
+
+	/*
+	 * FIXME - should a user specified LDT, TSS and V86 info
 	 * be duplicated as well?? - probably not.
 	 */
         // duplicate any use LDT entry that was set I think this is appropriate.
-#ifdef	MACH_BSD
-        if (parent->machine.pcb->uldt_selector!= 0) {
-            child->machine.pcb->uldt_selector = parent->machine.pcb->uldt_selector;
-            child->machine.pcb->uldt_desc = parent->machine.pcb->uldt_desc;
+        if (parent_pcb->uldt_selector!= 0) {
+	        child_pcb->uldt_selector = parent_pcb->uldt_selector;
+		child_pcb->uldt_desc = parent_pcb->uldt_desc;
         }
 #endif
             
-            
 	return (KERN_SUCCESS);
 }
 
@@ -273,17 +274,49 @@ void thread_set_child(thread_t child, int pid);
 void
 thread_set_child(thread_t child, int pid)
 {
-	child->machine.pcb->iss.eax = pid;
-	child->machine.pcb->iss.edx = 1;
-	child->machine.pcb->iss.efl &= ~EFL_CF;
+
+        if (thread_is_64bit(child)) {
+	        x86_saved_state64_t	*iss64;
+		
+		iss64 = USER_REGS64(child);
+
+		iss64->rax = pid;
+		iss64->rdx = 1;
+		iss64->isf.rflags &= ~EFL_CF;
+	} else {
+	        x86_saved_state32_t	*iss32;
+		
+		iss32 = USER_REGS32(child);
+
+		iss32->eax = pid;
+		iss32->edx = 1;
+		iss32->efl &= ~EFL_CF;
+	}
 }
+
+
 void thread_set_parent(thread_t parent, int pid);
 void
 thread_set_parent(thread_t parent, int pid)
 {
-	parent->machine.pcb->iss.eax = pid;
-	parent->machine.pcb->iss.edx = 0;
-	parent->machine.pcb->iss.efl &= ~EFL_CF;
+
+        if (thread_is_64bit(parent)) {
+	        x86_saved_state64_t	*iss64;
+		
+		iss64 = USER_REGS64(parent);
+
+		iss64->rax = pid;
+		iss64->rdx = 0;
+		iss64->isf.rflags &= ~EFL_CF;
+	} else {
+	        x86_saved_state32_t	*iss32;
+		
+		iss32 = USER_REGS32(parent);
+
+		iss32->eax = pid;
+		iss32->edx = 0;
+		iss32->efl &= ~EFL_CF;
+	}
 }
 
 
@@ -292,24 +325,10 @@ thread_set_parent(thread_t parent, int pid)
  * System Call handling code
  */
 
-#define	ERESTART	-1		/* restart syscall */
-#define	EJUSTRETURN	-2		/* don't modify regs, just return */
-
-
-#define NO_FUNNEL 0
-#define KERNEL_FUNNEL 1
-
-extern funnel_t * kernel_flock;
-
-extern int set_bsduthreadargs (thread_t, struct i386_saved_state *, void *);
-extern void * get_bsduthreadarg(thread_t);
-extern int * get_bsduthreadrval(thread_t th);
-extern int * get_bsduthreadlowpridelay(thread_t th);
+extern struct proc * i386_current_proc(void);
 
 extern long fuword(vm_offset_t);
 
-extern void unix_syscall(struct i386_saved_state *);
-extern void unix_syscall_return(int);
 
 /* following implemented in bsd/dev/i386/unix_signal.c */
 int __pthread_cset(struct sysent *);
@@ -318,243 +337,117 @@ void __pthread_creset(struct sysent *);
 
 
 void
-unix_syscall_return(int error)
+machdep_syscall(x86_saved_state_t *state)
 {
-    thread_t		thread;
-	volatile int *rval;
-	struct i386_saved_state *regs;
-	struct proc *p;
-	unsigned short code;
-	vm_offset_t params;
-	struct sysent *callp;
-	volatile int *lowpri_delay;
-
-    thread = current_thread();
-    rval = get_bsduthreadrval(thread);
-    lowpri_delay = get_bsduthreadlowpridelay(thread);
-	p = current_proc();
-
-	regs = USER_REGS(thread);
-
-	/* reconstruct code for tracing before blasting eax */
-	code = regs->eax;
-	params = (vm_offset_t) ((caddr_t)regs->uesp + sizeof (int));
-	callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
-	if (callp == sysent) {
-	  code = fuword(params);
-	}
-
-	if (error == ERESTART) {
-		regs->eip -= 7;
-	}
-	else if (error != EJUSTRETURN) {
-		if (error) {
-		    regs->eax = error;
-		    regs->efl |= EFL_CF;	/* carry bit */
-		} else { /* (not error) */
-		    regs->eax = rval[0];
-		    regs->edx = rval[1];
-		    regs->efl &= ~EFL_CF;
-		} 
-	}
-
-	ktrsysret(p, code, error, rval[0], (callp->sy_funnel & FUNNEL_MASK));
-
-	__pthread_creset(callp);
-
-	if ((callp->sy_funnel & FUNNEL_MASK) != NO_FUNNEL)
-    		(void) thread_funnel_set(current_thread()->funnel_lock, FALSE);
-
-	if (*lowpri_delay) {
-	        /*
-		 * task is marked as a low priority I/O type
-		 * and the I/O we issued while in this system call
-		 * collided with normal I/O operations... we'll
-		 * delay in order to mitigate the impact of this
-		 * task on the normal operation of the system
-		 */
-		IOSleep(*lowpri_delay);
-	        *lowpri_delay = 0;
-	}
-	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
-		error, rval[0], rval[1], 0, 0);
-
-    thread_exception_return();
-    /* NOTREACHED */
-}
-
-
-void
-unix_syscall(struct i386_saved_state *regs)
-{
-    thread_t		thread;
-    void	*vt; 
-    unsigned short	code;
-    struct sysent		*callp;
-	int	nargs;
-	int	error;
-	int *rval;
-	int funnel_type;
-    vm_offset_t		params;
-	struct proc *p;
-	volatile int *lowpri_delay;
-
-    thread = current_thread();
-    p = current_proc();
-    rval = get_bsduthreadrval(thread);
-    lowpri_delay = get_bsduthreadlowpridelay(thread);
-
-    thread->task->syscalls_unix++;		/* MP-safety ignored */
-
-    //printf("[scall : eax %x]",  regs->eax);
-    code = regs->eax;
-    params = (vm_offset_t) ((caddr_t)regs->uesp + sizeof (int));
-    callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
-    if (callp == sysent) {
-	code = fuword(params);
-	params += sizeof (int);
-	callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
-    }
+	int			args[machdep_call_count];
+        int			trapno;
+	int			nargs;
+	machdep_call_t		*entry;
+	x86_saved_state32_t	*regs;
+
+	assert(is_saved_state32(state));
+	regs = saved_state32(state);
     
-    vt = get_bsduthreadarg(thread);
+	trapno = regs->eax;
+#if DEBUG_TRACE
+	kprintf("machdep_syscall(0x%08x) code=%d\n", regs, trapno);
+#endif
 
-    if ((nargs = (callp->sy_narg * sizeof (int))) &&
-	    (error = copyin((user_addr_t) params, (char *) vt, nargs)) != 0) {
-	regs->eax = error;
-	regs->efl |= EFL_CF;
-	thread_exception_return();
-	/* NOTREACHED */
-    }
-    
-    rval[0] = 0;
-    rval[1] = regs->edx;
+	if (trapno < 0 || trapno >= machdep_call_count) {
+	        regs->eax = (unsigned int)kern_invalid(NULL);
 
-	if ((error = __pthread_cset(callp))) {
-		/* cancelled system call; let it returned with EINTR for handling */
-		regs->eax = error;
-		regs->efl |= EFL_CF;
 		thread_exception_return();
 		/* NOTREACHED */
 	}
+	entry = &machdep_call_table[trapno];
+	nargs = entry->nargs;
 
-	funnel_type = (callp->sy_funnel & FUNNEL_MASK);
-	if(funnel_type == KERNEL_FUNNEL)
-		(void) thread_funnel_set(kernel_flock, TRUE);
-	
-    (void) set_bsduthreadargs(thread, regs, NULL);
-
-    if (callp->sy_narg > 8)
-	panic("unix_syscall max arg count exceeded (%d)", callp->sy_narg);
-
-	ktrsyscall(p, code, callp->sy_narg, vt, funnel_type);
+	if (nargs != 0) {
+	        if (copyin((user_addr_t) regs->uesp + sizeof (int),
+			   (char *) args, (nargs * sizeof (int)))) {
+		        regs->eax = KERN_INVALID_ADDRESS;
 
-	{ 
-	  int *ip = (int *)vt;
-	  KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
-	      *ip, *(ip+1), *(ip+2), *(ip+3), 0);
+			thread_exception_return();
+			/* NOTREACHED */
+		}
 	}
+	switch (nargs) {
+	    case 0:
+	        regs->eax = (*entry->routine.args_0)();
+		break;
+	    case 1:
+		regs->eax = (*entry->routine.args_1)(args[0]);
+		break;
+	    case 2:
+		regs->eax = (*entry->routine.args_2)(args[0], args[1]);
+		break;
+	    case 3:
+	        if (!entry->bsd_style)
+		        regs->eax = (*entry->routine.args_3)(args[0], args[1], args[2]);
+		else {
+		        int	error;
+			int	rval;
+
+		        error = (*entry->routine.args_bsd_3)(&rval, args[0], args[1], args[2]);
+			if (error) {
+			        regs->eax = error;
+				regs->efl |= EFL_CF;	/* carry bit */
+			} else {
+			        regs->eax = rval;
+			        regs->efl &= ~EFL_CF;
+			}
+		}
+		break;
+	    case 4:
+		regs->eax = (*entry->routine.args_4)(args[0], args[1], args[2], args[3]);
+		break;
 
-    error = (*(callp->sy_call))((void *) p, (void *) vt, &rval[0]);
-	
-#if 0
-	/* May be needed with vfork changes */
-	regs = USER_REGS(thread);
-#endif
-	if (error == ERESTART) {
-		regs->eip -= 7;
-	}
-	else if (error != EJUSTRETURN) {
-		if (error) {
-		    regs->eax = error;
-		    regs->efl |= EFL_CF;	/* carry bit */
-		} else { /* (not error) */
-		    regs->eax = rval[0];
-		    regs->edx = rval[1];
-		    regs->efl &= ~EFL_CF;
-		} 
-	}
-
-	ktrsysret(p, code, error, rval[0], funnel_type);
-
-        __pthread_creset(callp);
-
-	if(funnel_type != NO_FUNNEL)
-    		(void) thread_funnel_set(current_thread()->funnel_lock, FALSE);
-
-	if (*lowpri_delay) {
-	        /*
-		 * task is marked as a low priority I/O type
-		 * and the I/O we issued while in this system call
-		 * collided with normal I/O operations... we'll
-		 * delay in order to mitigate the impact of this
-		 * task on the normal operation of the system
-		 */
-		IOSleep(*lowpri_delay);
-	        *lowpri_delay = 0;
+	    default:
+	        panic("machdep_syscall: too many args");
 	}
-	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
-		error, rval[0], rval[1], 0, 0);
+	if (current_thread()->funnel_lock)
+	        (void) thread_funnel_set(current_thread()->funnel_lock, FALSE);
 
-    thread_exception_return();
-    /* NOTREACHED */
+	thread_exception_return();
+	/* NOTREACHED */
 }
 
 
 void
-machdep_syscall( struct i386_saved_state *regs)
+machdep_syscall64(x86_saved_state_t *state)
 {
-    int				trapno, nargs;
-    machdep_call_t		*entry;
-    
-    trapno = regs->eax;
-    if (trapno < 0 || trapno >= machdep_call_count) {
-	regs->eax = (unsigned int)kern_invalid(NULL);
+        int			trapno;
+	machdep_call_t		*entry;
+	x86_saved_state64_t	*regs;
 
-	thread_exception_return();
-	/* NOTREACHED */
-    }
+	assert(is_saved_state64(state));
+	regs = saved_state64(state);
     
-    entry = &machdep_call_table[trapno];
-    nargs = entry->nargs;
+	trapno = regs->rax & SYSCALL_NUMBER_MASK;
 
-    if (nargs > 0) {
-	int			args[nargs];
+	if (trapno < 0 || trapno >= machdep_call_count) {
+	        regs->rax = (unsigned int)kern_invalid(NULL);
 
-	if (copyin((user_addr_t) regs->uesp + sizeof (int),
-		    (char *) args,
-		    nargs * sizeof (int))) {
-
-	    regs->eax = KERN_INVALID_ADDRESS;
-
-	    thread_exception_return();
-	    /* NOTREACHED */
+		thread_exception_return();
+		/* NOTREACHED */
 	}
+	entry = &machdep_call_table64[trapno];
 
-	switch (nargs) {
-	    case 1:
-		regs->eax = (*entry->routine.args_1)(args[0]);
-		break;
-	    case 2:
-		regs->eax = (*entry->routine.args_2)(args[0],args[1]);
-		break;
-	    case 3:
-		regs->eax = (*entry->routine.args_3)(args[0],args[1],args[2]);
+	switch (entry->nargs) {
+	    case 0:
+	        regs->rax = (*entry->routine.args_0)();
 		break;
-	    case 4:
-		regs->eax = (*entry->routine.args_4)(args[0],args[1],args[2],args[3]);
+	    case 1:
+		regs->rax = (*entry->routine.args64_1)(regs->rdi);
 		break;
 	    default:
-		panic("machdep_syscall(): too many args");
+	        panic("machdep_syscall64: too many args");
 	}
-    }
-    else
-	regs->eax = (*entry->routine.args_0)();
-
-    if (current_thread()->funnel_lock)
-   	(void) thread_funnel_set(current_thread()->funnel_lock, FALSE);
+	if (current_thread()->funnel_lock)
+	        (void) thread_funnel_set(current_thread()->funnel_lock, FALSE);
 
-    thread_exception_return();
-    /* NOTREACHED */
+	thread_exception_return();
+	/* NOTREACHED */
 }
 
 
@@ -583,7 +476,7 @@ thread_compose_cthread_desc(unsigned int addr, pcb_t pcb)
 kern_return_t
 thread_set_cthread_self(uint32_t self)
 {
-   current_thread()->machine.pcb->cthread_self = self;
+   current_thread()->machine.pcb->cthread_self = (uint64_t) self;
    
    return (KERN_SUCCESS);
 }
@@ -597,11 +490,38 @@ thread_get_cthread_self(void)
 kern_return_t
 thread_fast_set_cthread_self(uint32_t self)
 {
-  pcb_t pcb;
-  pcb = (pcb_t)current_thread()->machine.pcb;
-  thread_compose_cthread_desc(self, pcb);
-  pcb->cthread_self = self; /* preserve old func too */
-  return (USER_CTHREAD);
+	pcb_t			pcb;
+	x86_saved_state32_t	*iss;
+
+	pcb = (pcb_t)current_thread()->machine.pcb;
+	thread_compose_cthread_desc(self, pcb);
+	pcb->cthread_self = (uint64_t) self; /* preserve old func too */
+	iss = saved_state32(pcb->iss);
+	iss->gs = USER_CTHREAD;
+
+	return (USER_CTHREAD);
+}
+
+kern_return_t
+thread_fast_set_cthread_self64(uint64_t self)
+{
+	pcb_t			pcb;
+	x86_saved_state64_t	*iss;
+
+	pcb = current_thread()->machine.pcb;
+
+	/* check for canonical address, set 0 otherwise  */
+	if (!IS_USERADDR64_CANONICAL(self))
+		self = 0ULL;
+	pcb->cthread_self = self;
+	current_cpu_datap()->cpu_uber.cu_user_gs_base = self;
+
+	/* XXX for 64-in-32 */
+	iss = saved_state64(pcb->iss);
+	iss->gs = USER_CTHREAD;
+	thread_compose_cthread_desc((uint32_t) self, pcb);
+
+	return (USER_CTHREAD);
 }
 
 /*
@@ -671,137 +591,177 @@ thread_set_user_ldt(uint32_t address, uint32_t size, uint32_t flags)
 
     return USER_SETTABLE;
 }
-void
-mach25_syscall(struct i386_saved_state *regs)
-{
-	printf("*** Atttempt to execute a Mach 2.5 system call at EIP=%x EAX=%x(%d)\n",
-			regs->eip, regs->eax, -regs->eax);
-	panic("FIXME!");
-}
-#endif	/* MACH_BSD */
-
 
-/* This routine is called from assembly before each and every mach trap.
- */
+#endif	/* MACH_BSD */
 
-extern unsigned int mach_call_start(unsigned int, unsigned int *);
 
-__private_extern__
-unsigned int
-mach_call_start(unsigned int call_number, unsigned int *args)
-{
-	int i, argc;
-	unsigned int kdarg[3];
+typedef kern_return_t (*mach_call_t)(void *);
 
-	current_thread()->task->syscalls_mach++;	/* MP-safety ignored */
+struct mach_call_args {
+	syscall_arg_t arg1;
+	syscall_arg_t arg2;
+	syscall_arg_t arg3;
+	syscall_arg_t arg4;
+	syscall_arg_t arg5;
+	syscall_arg_t arg6;
+	syscall_arg_t arg7;
+	syscall_arg_t arg8;
+	syscall_arg_t arg9;
+};
 
-/* Always prepare to trace mach system calls */
 
-	kdarg[0]=0;
-	kdarg[1]=0;
-	kdarg[2]=0;
+static kern_return_t
+mach_call_arg_munger32(uint32_t sp, int nargs, int call_number, struct mach_call_args *args);
 
-	argc = mach_trap_table[call_number>>4].mach_trap_arg_count;
-	
-	if (argc > 3)
-		argc = 3;
-	
-	for (i=0; i < argc; i++)
-	  kdarg[i] = (int)*(args + i);
-	
-	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number>>4)) | DBG_FUNC_START,
-			      kdarg[0], kdarg[1], kdarg[2], 0, 0);
 
-	return call_number; /* pass this back thru */
-}
+static kern_return_t
+mach_call_arg_munger32(uint32_t sp, int nargs, int call_number, struct mach_call_args *args)
+{
+	unsigned int args32[9];
 
-/* This routine is called from assembly after each mach system call
- */
+	if (copyin((user_addr_t)(sp + sizeof(int)), (char *)args32, nargs * sizeof (int)))
+	        return KERN_INVALID_ARGUMENT;
 
-extern unsigned int mach_call_end(unsigned int, unsigned int);
+	switch (nargs) {
+	    case 9: args->arg9 = args32[8];
+	    case 8: args->arg8 = args32[7];
+	    case 7: args->arg7 = args32[6];
+	    case 6: args->arg6 = args32[5];
+	    case 5: args->arg5 = args32[4];
+	    case 4: args->arg4 = args32[3];
+	    case 3: args->arg3 = args32[2];
+	    case 2: args->arg2 = args32[1];
+	    case 1: args->arg1 = args32[0];
+	}
+	if (call_number == 90) {
+	        /* munge_l for mach_wait_until_trap() */
+	        args->arg1 = (((uint64_t)(args32[0])) | ((((uint64_t)(args32[1]))<<32)));
+	}
+	if (call_number == 93) {
+	        /* munge_wl for mk_timer_arm_trap() */
+	        args->arg2 = (((uint64_t)(args32[1])) | ((((uint64_t)(args32[2]))<<32)));
+	}
 
-__private_extern__
-unsigned int
-mach_call_end(unsigned int call_number, unsigned int retval)
-{
-  KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_SC,(call_number>>4)) | DBG_FUNC_END,
-		retval, 0, 0, 0, 0);
-	return retval;  /* pass this back thru */
+	return KERN_SUCCESS;
 }
 
-typedef kern_return_t (*mach_call_t)(void *);
 
-extern __attribute__((regparm(1))) kern_return_t
-mach_call_munger(unsigned int call_number, 
-	unsigned int arg1,
-	unsigned int arg2,
-	unsigned int arg3,
-	unsigned int arg4,
-	unsigned int arg5,
-	unsigned int arg6,
-	unsigned int arg7,
-	unsigned int arg8,
-	unsigned int arg9
-);
+__private_extern__ void
+mach_call_munger(x86_saved_state_t *state);
+
 
-struct mach_call_args {
-	unsigned int arg1;
-	unsigned int arg2;
-	unsigned int arg3;
-	unsigned int arg4;
-	unsigned int arg5;
-	unsigned int arg6;
-	unsigned int arg7;
-	unsigned int arg8;
-	unsigned int arg9;
-};
 __private_extern__
-__attribute__((regparm(1))) kern_return_t
-mach_call_munger(unsigned int call_number, 
-	unsigned int arg1,
-	unsigned int arg2,
-	unsigned int arg3,
-	unsigned int arg4,
-	unsigned int arg5,
-	unsigned int arg6,
-	unsigned int arg7,
-	unsigned int arg8,
-	unsigned int arg9
-)
+void
+mach_call_munger(x86_saved_state_t *state)
 {
 	int argc;
+	int call_number;
 	mach_call_t mach_call;
 	kern_return_t retval;
 	struct mach_call_args args = { 0, 0, 0, 0, 0, 0, 0, 0, 0 };
- 
-	current_thread()->task->syscalls_mach++;	/* MP-safety ignored */
-	call_number >>= 4;
+	x86_saved_state32_t	*regs;
+
+	assert(is_saved_state32(state));
+	regs = saved_state32(state);
+
+	call_number = -(regs->eax);
+#if DEBUG_TRACE
+	kprintf("mach_call_munger(0x%08x) code=%d\n", regs, call_number);
+#endif
+
+	if (call_number < 0 || call_number >= mach_trap_count) {
+	        i386_exception(EXC_SYSCALL, call_number, 1);
+		/* NOTREACHED */
+	}
+	mach_call = (mach_call_t)mach_trap_table[call_number].mach_trap_function;
 	
+	if (mach_call == (mach_call_t)kern_invalid) {
+	        i386_exception(EXC_SYSCALL, call_number, 1);
+		/* NOTREACHED */
+	}
 	argc = mach_trap_table[call_number].mach_trap_arg_count;
-	switch (argc) {
-		case 9: args.arg9 = arg9;
-		case 8: args.arg8 = arg8;
-		case 7: args.arg7 = arg7;
-		case 6: args.arg6 = arg6;
-		case 5: args.arg5 = arg5;
-		case 4: args.arg4 = arg4;
-		case 3: args.arg3 = arg3;
-		case 2: args.arg2 = arg2;
-		case 1: args.arg1 = arg1;
+
+	if (argc) {
+	        retval = mach_call_arg_munger32(regs->uesp, argc, call_number, &args);
+		
+		if (retval != KERN_SUCCESS) {
+		        regs->eax = retval;
+	
+			thread_exception_return();
+			/* NOTREACHED */
+		}
 	}
+	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_START,
+			      (int) args.arg1, (int) args.arg2, (int) args.arg3, (int) args.arg4, 0);
 	
+	retval = mach_call(&args);
+
+	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_SC,(call_number)) | DBG_FUNC_END,
+			      retval, 0, 0, 0, 0);
+	regs->eax = retval;
+	
+	thread_exception_return();
+	/* NOTREACHED */
+}
+
+
+
+__private_extern__ void
+mach_call_munger64(x86_saved_state_t *state);
+
+
+__private_extern__
+void
+mach_call_munger64(x86_saved_state_t *state)
+{
+	int call_number;
+	int argc;
+	mach_call_t mach_call;
+	x86_saved_state64_t	*regs;
+
+	assert(is_saved_state64(state));
+	regs = saved_state64(state);
+
+	call_number = regs->rax & SYSCALL_NUMBER_MASK;
+
 	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_START,
-			      args.arg1, args.arg2, args.arg3, 0, 0);
+			      (int) regs->rdi, (int) regs->rsi, (int) regs->rdx, (int) regs->r10, 0);
 	
+	if (call_number < 0 || call_number >= mach_trap_count) {
+	        i386_exception(EXC_SYSCALL, regs->rax, 1);
+		/* NOTREACHED */
+	}
 	mach_call = (mach_call_t)mach_trap_table[call_number].mach_trap_function;
-	retval = mach_call(&args);
 
+	if (mach_call == (mach_call_t)kern_invalid) {
+	        i386_exception(EXC_SYSCALL, regs->rax, 1);
+		/* NOTREACHED */
+	}
+	argc = mach_trap_table[call_number].mach_trap_arg_count;
+
+	if (argc > 6) {
+	        int copyin_count;
+
+		copyin_count = (argc - 6) * sizeof(uint64_t);
+
+	        if (copyin((user_addr_t)(regs->isf.rsp + sizeof(user_addr_t)), (char *)&regs->v_arg6, copyin_count)) {
+		        regs->rax = KERN_INVALID_ARGUMENT;
+			
+			thread_exception_return();
+			/* NOTREACHED */
+		}
+	}
+	regs->rax = (uint64_t)mach_call((void *)(&regs->rdi));
+	
 	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_SC,(call_number)) | DBG_FUNC_END,
-		retval, 0, 0, 0, 0);
+			      (int)regs->rax, 0, 0, 0, 0);
 
-	return retval;
+	thread_exception_return();
+	/* NOTREACHED */
 }
 
+
+
 /*
  * thread_setuserstack:
  *
@@ -813,9 +773,19 @@ thread_setuserstack(
 	thread_t	thread,
 	mach_vm_address_t	user_stack)
 {
-	struct i386_saved_state *ss = get_user_regs(thread);
+        if (thread_is_64bit(thread)) {
+	        x86_saved_state64_t	*iss64;
+
+		iss64 = USER_REGS64(thread);
 
-	ss->uesp = CAST_DOWN(unsigned int,user_stack);
+		iss64->isf.rsp = (uint64_t)user_stack;
+	} else {
+	        x86_saved_state32_t	*iss32;
+		
+		iss32 = USER_REGS32(thread);
+
+		iss32->uesp = CAST_DOWN(unsigned int, user_stack);
+	}
 }
 
 /*
@@ -829,10 +799,23 @@ thread_adjuserstack(
 	thread_t	thread,
 	int		adjust)
 {
-	struct i386_saved_state *ss = get_user_regs(thread);
+        if (thread_is_64bit(thread)) {
+	        x86_saved_state64_t	*iss64;
 
-	ss->uesp += adjust;
-	return CAST_USER_ADDR_T(ss->uesp);
+		iss64 = USER_REGS64(thread);
+
+		iss64->isf.rsp += adjust;
+
+		return iss64->isf.rsp;
+	} else {
+	        x86_saved_state32_t	*iss32;
+		
+		iss32 = USER_REGS32(thread);
+
+		iss32->uesp += adjust;
+
+		return CAST_USER_ADDR_T(iss32->uesp);
+	}
 }
 
 /*
@@ -842,12 +825,55 @@ thread_adjuserstack(
  * dependent thread state info.
  */
 void
-thread_setentrypoint(
-	thread_t	thread,
-	mach_vm_address_t	entry)
+thread_setentrypoint(thread_t thread, mach_vm_address_t entry)
 {
-	struct i386_saved_state *ss = get_user_regs(thread);
+        if (thread_is_64bit(thread)) {
+	        x86_saved_state64_t	*iss64;
 
- 	ss->eip = CAST_DOWN(unsigned int,entry);
-}    
+		iss64 = USER_REGS64(thread);
+
+		iss64->isf.rip = (uint64_t)entry;
+	} else {
+	        x86_saved_state32_t	*iss32;
+		
+		iss32 = USER_REGS32(thread);
+
+		iss32->eip = CAST_DOWN(unsigned int, entry);
+	}
+}
+
+
+void
+thread_setsinglestep(thread_t thread, int on)
+{
+        if (thread_is_64bit(thread)) {
+	        x86_saved_state64_t	*iss64;
+
+		iss64 = USER_REGS64(thread);
+
+		if (on)
+		        iss64->isf.rflags |= EFL_TF;
+		else
+		        iss64->isf.rflags &= ~EFL_TF;
+	} else {
+	        x86_saved_state32_t	*iss32;
+		
+		iss32 = USER_REGS32(thread);
+
+		if (on)
+		        iss32->efl |= EFL_TF;
+		else
+		        iss32->efl &= ~EFL_TF;
+	}
+}
+
+
+
+/* XXX this should be a struct savearea so that CHUD will work better on x86 */
+void *
+find_user_regs(
+	thread_t        thread)
+{
+	return USER_STATE(thread);
+}
 
diff --git a/osfmk/i386/commpage/atomic.s b/osfmk/i386/commpage/atomic.s
index 71a02d058..82e85b78c 100644
--- a/osfmk/i386/commpage/atomic.s
+++ b/osfmk/i386/commpage/atomic.s
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -99,14 +99,14 @@ Lcompare_and_swap64_up:
 Lbit_test_and_set_mp:
 .long	_COMM_PAGE_BTS+4
 	lock
-	bts %eax, (%edx)
+	btsl %eax, (%edx)
 	ret
 
     COMMPAGE_DESCRIPTOR(bit_test_and_set_mp,_COMM_PAGE_BTS,0,kUP)
 
 Lbit_test_and_set_up:
 .long	_COMM_PAGE_BTS+4
-	bts %eax, (%edx)
+	btsl %eax, (%edx)
 	ret
 
     COMMPAGE_DESCRIPTOR(bit_test_and_set_up,_COMM_PAGE_BTS,kUP,0)
@@ -120,14 +120,14 @@ Lbit_test_and_set_up:
 Lbit_test_and_clear_mp:
 .long	_COMM_PAGE_BTC+4
 	lock
-	btc %eax, (%edx)
+	btrl %eax, (%edx)
 	ret
 
     COMMPAGE_DESCRIPTOR(bit_test_and_clear_mp,_COMM_PAGE_BTC,0,kUP)
 
 Lbit_test_and_clear_up:
 .long	_COMM_PAGE_BTC+4
-	btc %eax, (%edx)
+	btrl %eax, (%edx)
 	ret
 
     COMMPAGE_DESCRIPTOR(bit_test_and_clear_up,_COMM_PAGE_BTC,kUP,0)
@@ -153,3 +153,153 @@ Latomic_add32_up:
 	ret
 
     COMMPAGE_DESCRIPTOR(atomic_add32_up,_COMM_PAGE_ATOMIC_ADD32,kUP,0)
+
+
+/************************* x86_64 versions follow **************************/
+
+
+// This is a subroutine used by:
+
+// bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value);
+// int32_t OSAtomicAnd32( int32_t mask, int32_t *value);
+// int32_t OSAtomicOr32( int32_t mask, int32_t *value);
+// int32_t OSAtomicXor32( int32_t mask, int32_t *value);
+
+// It assumes: old -> %rdi  (ie, it follows the ABI parameter conventions)
+//             new -> %rsi
+//             value -> %rdx
+// on success: returns with ZF set
+// on failure: returns with *value in %eax, ZF clear
+
+	.code64
+Lcompare_and_swap32_mp_64:
+	movl	%edi,%eax			// put old value where "cmpxchg" wants it
+	lock
+	cmpxchgl  %esi, (%rdx)
+	ret
+
+    COMMPAGE_DESCRIPTOR(compare_and_swap32_mp_64,_COMM_PAGE_COMPARE_AND_SWAP32,0,kUP)
+
+	.code64
+Lcompare_and_swap32_up_64:
+	movl	%edi,%eax			// put old value where "cmpxchg" wants it
+	cmpxchgl  %esi, (%rdx)
+	ret
+
+    COMMPAGE_DESCRIPTOR(compare_and_swap32_up_64,_COMM_PAGE_COMPARE_AND_SWAP32,kUP,0)
+
+// This is a subroutine used by:
+// bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value);
+
+// It assumes: old -> %rdi  (ie, it follows the ABI parameter conventions)
+//             new -> %rsi
+//             value -> %rdx
+// on success: returns with ZF set
+// on failure: returns with *value in %rax, ZF clear
+
+	.code64
+Lcompare_and_swap64_mp_64:
+	movq	%rdi,%rax			// put old value where "cmpxchg" wants it
+	lock
+	cmpxchgq  %rsi, (%rdx)
+	ret
+
+    COMMPAGE_DESCRIPTOR(compare_and_swap64_mp_64,_COMM_PAGE_COMPARE_AND_SWAP64,0,kUP)
+
+	.code64
+Lcompare_and_swap64_up_64:
+	movq	%rdi,%rax			// put old value where "cmpxchg" wants it
+	cmpxchgq  %rsi, (%rdx)
+	ret
+
+    COMMPAGE_DESCRIPTOR(compare_and_swap64_up_64,_COMM_PAGE_COMPARE_AND_SWAP64,kUP,0)
+
+// This is a subroutine used by:
+// bool OSAtomicTestAndSet( uint32_t n, void *value );
+// It is called with standard register conventions:
+//			n = %rdi
+//			value = %rsi
+// Returns: old value of bit in CF
+
+	.code64
+Lbit_test_and_set_mp_64:
+	lock
+	btsl %edi, (%rsi)
+	ret
+
+    COMMPAGE_DESCRIPTOR(bit_test_and_set_mp_64,_COMM_PAGE_BTS,0,kUP)
+
+	.code64
+Lbit_test_and_set_up_64:
+	btsl %edi, (%rsi)
+	ret
+
+    COMMPAGE_DESCRIPTOR(bit_test_and_set_up_64,_COMM_PAGE_BTS,kUP,0)
+
+// This is a subroutine used by:
+// bool OSAtomicTestAndClear( uint32_t n, void *value );
+// It is called with standard register conventions:
+//			n = %rdi
+//			value = %rsi
+// Returns: old value of bit in CF
+
+	.code64
+Lbit_test_and_clear_mp_64:
+	lock
+	btrl %edi, (%rsi)
+	ret
+
+    COMMPAGE_DESCRIPTOR(bit_test_and_clear_mp_64,_COMM_PAGE_BTC,0,kUP)
+
+	.code64
+Lbit_test_and_clear_up_64:
+	btrl %edi, (%rsi)
+	ret
+
+    COMMPAGE_DESCRIPTOR(bit_test_and_clear_up_64,_COMM_PAGE_BTC,kUP,0)
+
+// This is a subroutine used by:
+// int32_t OSAtomicAdd32( int32_t amt, int32_t *value );
+// It is called with standard register conventions:
+//			amt = %rdi
+//			value = %rsi
+// Returns: old value in %edi
+// NB: OSAtomicAdd32 returns the new value,  so clients will add amt to %edi 
+
+	.code64
+Latomic_add32_mp_64:
+	lock
+	xaddl	%edi, (%rsi)
+	ret
+		
+    COMMPAGE_DESCRIPTOR(atomic_add32_mp_64,_COMM_PAGE_ATOMIC_ADD32,0,kUP)
+
+	.code64
+Latomic_add32_up_64:
+	xaddl	%edi, (%rsi)
+	ret
+
+    COMMPAGE_DESCRIPTOR(atomic_add32_up_64,_COMM_PAGE_ATOMIC_ADD32,kUP,0)
+
+// This is a subroutine used by:
+// int64_t OSAtomicAdd64( int64_t amt, int64_t *value );
+// It is called with standard register conventions:
+//			amt = %rdi
+//			value = %rsi
+// Returns: old value in %rdi
+// NB: OSAtomicAdd64 returns the new value,  so clients will add amt to %rdi 
+
+	.code64
+Latomic_add64_mp_64:
+	lock
+	xaddq	%rdi, (%rsi)
+	ret
+		
+    COMMPAGE_DESCRIPTOR(atomic_add64_mp_64,_COMM_PAGE_ATOMIC_ADD64,0,kUP)
+
+	.code64
+Latomic_add64_up_64:
+	xaddq	%rdi, (%rsi)
+	ret
+
+    COMMPAGE_DESCRIPTOR(atomic_add64_up_64,_COMM_PAGE_ATOMIC_ADD64,kUP,0)
diff --git a/osfmk/i386/commpage/bcopy_scalar.s b/osfmk/i386/commpage/bcopy_scalar.s
index 9c078efea..f35773b70 100644
--- a/osfmk/i386/commpage/bcopy_scalar.s
+++ b/osfmk/i386/commpage/bcopy_scalar.s
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -74,10 +74,12 @@
 .text
 .align 5, 0x90
 Lbcopy_scalar:
+	pushl	%ebp		/* set up a frame for backtraces */
+	movl	%esp,%ebp
         pushl   %esi
         pushl   %edi
-        movl    12(%esp),%esi
-        movl    16(%esp),%edi
+        movl    8(%ebp),%esi
+        movl    12(%ebp),%edi
 	jmp	1f
 /*
 ** These need to be 32 bytes from Lbcopy_scalar
@@ -85,13 +87,15 @@ Lbcopy_scalar:
 .align 5, 0x90
 Lmemcpy_scalar:
 Lmemmove_scalar:
+	pushl	%ebp		/* set up a frame for backtraces */
+	movl	%esp,%ebp
         pushl   %esi
         pushl   %edi
-        movl    12(%esp),%edi
-        movl    16(%esp),%esi
+        movl    8(%ebp),%edi
+        movl    12(%ebp),%esi
         movl    %edi,%eax
 1:
-        movl    20(%esp),%ecx
+        movl    16(%ebp),%ecx
         movl    %edi,%edx
         subl    %esi,%edx
         cmpl    %ecx,%edx       /* overlapping? */
@@ -107,6 +111,7 @@ Lmemmove_scalar:
         movsb
         popl    %edi
         popl    %esi
+	popl	%ebp
         ret
 2:
         addl    %ecx,%edi       /* copy backwards. */
@@ -126,7 +131,8 @@ Lmemmove_scalar:
         movsl
         popl    %edi
         popl    %esi
+	popl	%ebp
         cld
         ret
 
-	COMMPAGE_DESCRIPTOR(bcopy_scalar,_COMM_PAGE_BCOPY,0,0)
+	COMMPAGE_DESCRIPTOR(bcopy_scalar,_COMM_PAGE_BCOPY,0,kHasSSE2+kHasSupplementalSSE3)
diff --git a/osfmk/i386/commpage/bcopy_sse3.s b/osfmk/i386/commpage/bcopy_sse3.s
new file mode 100644
index 000000000..4d8ba9c53
--- /dev/null
+++ b/osfmk/i386/commpage/bcopy_sse3.s
@@ -0,0 +1,476 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+        
+#include <machine/cpu_capabilities.h>
+#include <machine/commpage.h>
+
+/*
+ * The bcopy/memcpy loops, tuned for Pentium-M class processors with SSE3
+ * and 64-byte cache lines, such as Core and Core 2.
+ *
+ * The following #defines are tightly coupled to the u-architecture:
+ */
+
+#define kShort  80			// too short to bother with SSE (must be >=80)
+#define kVeryLong   (500*1024)          // large enough for non-temporal stores (must be >= 8192)
+#define kBigChunk   (256*1024)          // outer loop chunk size for kVeryLong sized operands
+#define kFastUCode  (16*1024)		// cutoff for microcode fastpath for "rep/movsl"
+
+
+// void bcopy(const void *src, void *dst, size_t len);
+ 
+        .text
+        .align 5, 0x90
+Lbcopy_sse3:				// void bcopy(const void *src, void *dst, size_t len)
+	pushl	%ebp			// set up a frame for backtraces
+	movl	%esp,%ebp
+        pushl   %esi
+        pushl   %edi
+        movl    8(%ebp),%esi		// get source ptr
+        movl    12(%ebp),%edi           // get dest ptr
+        jmp     Ljoin 
+
+//
+// void *memcpy(void *dst, const void *src, size_t len);
+// void *memmove(void *dst, const void *src, size_t len);
+//
+// NB: These need to be 32 bytes from bcopy():
+//
+
+        .align	5, 0x90
+Lmemcpy:				// void *memcpy(void *dst, const void *src, size_t len)
+Lmemmove:				// void *memmove(void *dst, const void *src, size_t len)
+	pushl	%ebp			// set up a frame for backtraces
+	movl	%esp,%ebp
+        pushl   %esi
+        pushl   %edi
+        movl    8(%ebp),%edi		// get dest ptr
+        movl    12(%ebp),%esi           // get source ptr
+        
+Ljoin:                                  // here from bcopy() with esi and edi loaded
+        movl    16(%ebp),%ecx           // get length
+        movl    %edi,%edx
+        subl    %esi,%edx               // (dest - source)
+        cmpl    %ecx,%edx               // must move in reverse if (dest - source) < length
+        jb      LReverseIsland
+Lrejoin:                                // here from very-long-operand copies
+        cmpl    $(kShort),%ecx          // long enough to bother with SSE?
+        ja      LNotShort               // yes
+        
+// Handle short forward copies.  As the most common case, this is the fall-through path.
+//      ecx = length (<= kShort)
+//      esi = source ptr
+//      edi = dest ptr
+
+Lshort:
+	movl    %ecx,%edx		// copy length
+	shrl	$2,%ecx			// get #doublewords
+	jz	LLeftovers
+2:					// loop copying doublewords
+	movl	(%esi),%eax
+	addl	$4,%esi
+	movl	%eax,(%edi)
+	addl	$4,%edi
+	dec	%ecx
+	jnz	2b
+LLeftovers:				// handle leftover bytes (0..3) in last word
+	andl	$3,%edx			// any leftover bytes?
+	jz	5f
+4:					// loop copying bytes
+	movb	(%esi),%al
+	inc	%esi
+	movb	%al,(%edi)
+	inc	%edi
+	dec	%edx
+	jnz	4b
+5:
+        movl    8(%ebp),%eax		// get return value (dst ptr) for memcpy/memmove
+        popl    %edi
+        popl    %esi
+	popl	%ebp
+        ret
+
+
+LReverseIsland:				// keep the "jb" above a short branch...
+	jmp	LReverse		// ...because reverse moves are uncommon
+
+
+// Handle forward moves that are long enough to justify use of SSE3.
+// First, 16-byte align the destination.
+//      ecx = length (> kShort)
+//      esi = source ptr
+//      edi = dest ptr
+
+LNotShort:
+        cmpl    $(kVeryLong),%ecx       // long enough to justify heavyweight loops?
+        movl    %edi,%edx               // copy destination
+        jae     LVeryLong		// use very-long-operand path
+        negl    %edx
+        andl    $15,%edx                // get #bytes to align destination
+	jz	LDestAligned		// already aligned
+        subl    %edx,%ecx               // decrement length
+1:					// loop copying 1..15 bytes
+	movb	(%esi),%al
+	inc	%esi
+	movb	%al,(%edi)
+	inc	%edi
+	dec	%edx
+	jnz	1b
+        
+// Destination is now aligned.  Prepare for forward loops over 64-byte chunks.
+// Since kShort>=80 and we've moved at most 15 bytes already, there is at least one chunk.
+
+LDestAligned:
+        movl    %ecx,%edx               // copy length
+	movl	%ecx,%eax		// twice
+        andl    $63,%ecx                // get remaining bytes for Lshort
+        andl    $-64,%edx               // get number of bytes we will copy in inner loop
+        addl    %edx,%esi               // point to 1st byte not copied
+        addl    %edx,%edi
+        negl    %edx                    // now generate offset to 1st byte to be copied
+	testl	$15,%esi		// is source aligned too?
+	jnz	LUnalignedLoop		// no
+
+
+	cmpl	$(kFastUCode),%eax	// long enough for the fastpath in microcode?
+	jb	LAlignedLoop		// no, use SSE
+	cld				// we'll move forward
+	movl	%eax,%ecx		// copy length again
+	shrl	$2,%ecx			// compute #words to move
+        addl    %edx,%esi               // restore ptrs to 1st byte of source and dest
+        addl    %edx,%edi
+	rep				// the u-code will optimize this
+	movsl
+	movl	%eax,%edx		// original length
+	jmp	LLeftovers		// handle 0..3 leftover bytes
+
+
+// Forward aligned loop for medium length operands (kShort < n < kVeryLong).
+
+	.align	4,0x90			// 16-byte align inner loops
+LAlignedLoop:                           // loop over 64-byte chunks
+        movdqa  (%esi,%edx),%xmm0
+        movdqa  16(%esi,%edx),%xmm1
+        movdqa  32(%esi,%edx),%xmm2
+        movdqa  48(%esi,%edx),%xmm3
+
+        movdqa  %xmm0,(%edi,%edx)
+        movdqa  %xmm1,16(%edi,%edx)
+        movdqa  %xmm2,32(%edi,%edx)
+        movdqa  %xmm3,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     LAlignedLoop
+        
+        jmp     Lshort                  // copy remaining 0..15 bytes and done
+            
+
+// Forward unaligned loop for medium length operands (kShort < n < kVeryLong).
+// Note that LDDQU==MOVDQU on these machines, ie we don't care when we cross
+// source cache lines.
+
+	.align	4,0x90			// 16-byte align inner loops
+LUnalignedLoop:                         // loop over 64-byte chunks
+        movdqu  (%esi,%edx),%xmm0	// the loads are unaligned
+        movdqu  16(%esi,%edx),%xmm1
+        movdqu  32(%esi,%edx),%xmm2
+        movdqu  48(%esi,%edx),%xmm3
+        
+        movdqa  %xmm0,(%edi,%edx)	// we can use aligned stores
+        movdqa  %xmm1,16(%edi,%edx)
+        movdqa  %xmm2,32(%edi,%edx)
+        movdqa  %xmm3,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     LUnalignedLoop
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+        
+
+// Very long forward moves.  These are at least several pages, so we loop over big
+// chunks of memory (kBigChunk in size.)  We first prefetch the chunk, and then copy
+// it using non-temporal stores.  Hopefully all the reads occur in the prefetch loop,
+// so the copy loop reads from L2 and writes directly to memory (with write combining.)
+// This minimizes bus turnaround and maintains good DRAM page locality.
+// Note that for this scheme to work, kVeryLong must be a large fraction of L2 cache
+// size.  Otherwise, it is counter-productive to bypass L2 on the stores.
+//      ecx = length (>= kVeryLong bytes)
+//      edi = dest (aligned)
+//      esi = source
+
+LVeryLong:
+	pushl	%ebx			// we'll need to use this
+	movl	%edi,%ebx		// copy dest ptr
+	negl	%ebx
+	andl	$63,%ebx		// get #bytes to cache line align destination
+	jz	LBigChunkLoop		// already aligned
+	
+// Cache line align destination, so temporal stores in copy loops work right.
+
+	pushl	%ecx			// save total length remaining
+	pushl	%ebx			// arg3 - #bytes to align destination (1..63)
+	pushl	%esi			// arg2 - source
+	pushl	%edi			// arg1 - dest
+	call	Lmemcpy			// align the destination
+	movl	12(%esp),%ecx		// recover total length
+	addl	$16,%esp
+	addl	%ebx,%esi		// adjust ptrs and lengths past copy
+	addl	%ebx,%edi
+	subl	%ebx,%ecx
+	
+// Loop over big chunks.
+//      ecx = length remaining (>= 4096)
+//      edi = dest (64-byte aligned)
+//      esi = source (may be unaligned)
+	
+LBigChunkLoop:
+        movl    $(kBigChunk),%edx       // assume we can do a full chunk
+	cmpl	%edx,%ecx		// do we have a full chunk left to do?
+	cmovbl	%ecx,%edx		// if not, only move what we have left
+	andl	$-4096,%edx		// we work in page multiples
+	xor	%eax,%eax		// initialize chunk offset
+	jmp	LTouchLoop
+
+// Because the source may be unaligned, we use byte loads to touch.
+//      ecx = length remaining (including this chunk)
+//      edi = ptr to start of dest chunk
+//      esi = ptr to start of source chunk
+//	edx = chunk length (multiples of pages)
+//	ebx = scratch reg used to read a byte of each cache line
+//	eax = chunk offset
+
+	.align	4,0x90			// 16-byte align inner loops
+LTouchLoop:
+	movzb	(%esi,%eax),%ebx	// touch line 0, 2, 4, or 6 of page
+	movzb	1*64(%esi,%eax),%ebx	// touch line 1, 3, 5, or 7
+	movzb	8*64(%esi,%eax),%ebx	// touch line 8, 10, 12, or 14
+	movzb	9*64(%esi,%eax),%ebx	// etc
+	
+	movzb	16*64(%esi,%eax),%ebx
+	movzb	17*64(%esi,%eax),%ebx
+	movzb	24*64(%esi,%eax),%ebx
+	movzb	25*64(%esi,%eax),%ebx
+
+	movzb	32*64(%esi,%eax),%ebx
+	movzb	33*64(%esi,%eax),%ebx
+	movzb	40*64(%esi,%eax),%ebx
+	movzb	41*64(%esi,%eax),%ebx
+
+	movzb	48*64(%esi,%eax),%ebx
+	movzb	49*64(%esi,%eax),%ebx
+	movzb	56*64(%esi,%eax),%ebx
+	movzb	57*64(%esi,%eax),%ebx
+	
+	subl	$-128,%eax		// next slice of page (adding 128 w 8-bit immediate)
+	testl	$512,%eax		// done with this page?
+	jz	LTouchLoop		// no, next of four slices
+	addl	$(4096-512),%eax	// move on to next page
+	cmpl	%eax,%edx		// done with this chunk?
+	jnz	LTouchLoop		// no, do next page
+	
+// The chunk has been pre-fetched, now copy it using non-temporal stores.
+// There are two copy loops, depending on whether the source is 16-byte aligned
+// or not.
+
+	addl	%edx,%esi		// increment ptrs by chunk length
+	addl	%edx,%edi
+	subl	%edx,%ecx		// adjust remaining length
+	negl	%edx			// prepare loop index (counts up to 0)
+	testl	$15,%esi		// is source 16-byte aligned?
+	jnz	LVeryLongUnaligned	// source is not aligned
+	jmp	LVeryLongAligned
+
+	.align	4,0x90			// 16-byte align inner loops
+LVeryLongAligned:			// aligned loop over 128-bytes
+        movdqa  (%esi,%edx),%xmm0
+        movdqa  16(%esi,%edx),%xmm1
+        movdqa  32(%esi,%edx),%xmm2
+        movdqa  48(%esi,%edx),%xmm3
+        movdqa  64(%esi,%edx),%xmm4
+        movdqa  80(%esi,%edx),%xmm5
+        movdqa  96(%esi,%edx),%xmm6
+        movdqa  112(%esi,%edx),%xmm7
+
+        movntdq %xmm0,(%edi,%edx)
+        movntdq %xmm1,16(%edi,%edx)
+        movntdq %xmm2,32(%edi,%edx)
+        movntdq %xmm3,48(%edi,%edx)
+        movntdq %xmm4,64(%edi,%edx)
+        movntdq %xmm5,80(%edi,%edx)
+        movntdq %xmm6,96(%edi,%edx)
+        movntdq %xmm7,112(%edi,%edx)
+        
+        subl    $-128,%edx		// add 128 with an 8-bit immediate
+        jnz	LVeryLongAligned
+	jmp	LVeryLongChunkEnd
+
+	.align	4,0x90			// 16-byte align inner loops
+LVeryLongUnaligned:			// unaligned loop over 128-bytes
+        movdqu  (%esi,%edx),%xmm0
+        movdqu  16(%esi,%edx),%xmm1
+        movdqu  32(%esi,%edx),%xmm2
+        movdqu  48(%esi,%edx),%xmm3
+        movdqu  64(%esi,%edx),%xmm4
+        movdqu  80(%esi,%edx),%xmm5
+        movdqu  96(%esi,%edx),%xmm6
+        movdqu  112(%esi,%edx),%xmm7
+
+        movntdq %xmm0,(%edi,%edx)
+        movntdq %xmm1,16(%edi,%edx)
+        movntdq %xmm2,32(%edi,%edx)
+        movntdq %xmm3,48(%edi,%edx)
+        movntdq %xmm4,64(%edi,%edx)
+        movntdq %xmm5,80(%edi,%edx)
+        movntdq %xmm6,96(%edi,%edx)
+        movntdq %xmm7,112(%edi,%edx)
+        
+        subl    $-128,%edx		// add 128 with an 8-bit immediate
+        jnz     LVeryLongUnaligned
+
+LVeryLongChunkEnd:
+	cmpl	$4096,%ecx		// at least another page to go?
+	jae	LBigChunkLoop		// yes
+	
+	sfence				// required by non-temporal stores
+	popl	%ebx
+        jmp     Lrejoin                 // handle remaining (0..4095) bytes
+
+
+// Reverse moves.
+//      ecx = length
+//      esi = source ptr
+//      edi = dest ptr
+
+LReverse:
+        addl    %ecx,%esi               // point to end of strings
+        addl    %ecx,%edi
+        cmpl    $(kShort),%ecx          // long enough to bother with SSE?
+        ja      LReverseNotShort        // yes
+
+// Handle reverse short copies.
+//      ecx = length
+//      esi = one byte past end of source
+//      edi = one byte past end of dest
+
+LReverseShort:
+	movl    %ecx,%edx		// copy length
+	shrl	$2,%ecx			// #words
+	jz	3f
+1:
+	subl	$4,%esi
+	movl	(%esi),%eax
+	subl	$4,%edi
+	movl	%eax,(%edi)
+	dec	%ecx
+	jnz	1b
+3:
+	andl	$3,%edx			// bytes?
+	jz	5f
+4:
+	dec	%esi
+	movb	(%esi),%al
+	dec	%edi
+	movb	%al,(%edi)
+	dec	%edx
+	jnz	4b
+5:
+        movl    8(%ebp),%eax		// get return value (dst ptr) for memcpy/memmove
+        popl    %edi
+        popl    %esi
+	popl	%ebp
+        ret
+
+// Handle a reverse move long enough to justify using SSE.
+//      ecx = length
+//      esi = one byte past end of source
+//      edi = one byte past end of dest
+
+LReverseNotShort:
+        movl    %edi,%edx               // copy destination
+        andl    $15,%edx                // get #bytes to align destination
+        je      LReverseDestAligned     // already aligned
+        subl	%edx,%ecx		// adjust length
+1:					// loop copying 1..15 bytes
+	dec	%esi
+	movb	(%esi),%al
+	dec	%edi
+	movb	%al,(%edi)
+	dec	%edx
+	jnz	1b
+        
+// Destination is now aligned.  Prepare for reverse loops.
+
+LReverseDestAligned:
+        movl    %ecx,%edx               // copy length
+        andl    $63,%ecx                // get remaining bytes for Lshort
+        andl    $-64,%edx               // get number of bytes we will copy in inner loop
+        subl    %edx,%esi               // point to endpoint of copy
+        subl    %edx,%edi
+	testl	$15,%esi		// is source aligned too?
+        jnz     LReverseUnalignedLoop   // no
+	jmp	LReverseAlignedLoop	// use aligned loop
+
+	.align	4,0x90			// 16-byte align inner loops
+LReverseAlignedLoop:                    // loop over 64-byte chunks
+        movdqa  -16(%esi,%edx),%xmm0
+        movdqa  -32(%esi,%edx),%xmm1
+        movdqa  -48(%esi,%edx),%xmm2
+        movdqa  -64(%esi,%edx),%xmm3
+
+        movdqa  %xmm0,-16(%edi,%edx)
+        movdqa  %xmm1,-32(%edi,%edx)
+        movdqa  %xmm2,-48(%edi,%edx)
+        movdqa  %xmm3,-64(%edi,%edx)
+        
+        subl    $64,%edx
+        jne     LReverseAlignedLoop
+        
+        jmp     LReverseShort           // copy remaining 0..63 bytes and done
+
+    
+// Reverse, unaligned loop.  LDDQU==MOVDQU on these machines.
+        
+	.align	4,0x90			// 16-byte align inner loops
+LReverseUnalignedLoop:                  // loop over 64-byte chunks
+        movdqu  -16(%esi,%edx),%xmm0
+        movdqu  -32(%esi,%edx),%xmm1
+        movdqu  -48(%esi,%edx),%xmm2
+        movdqu  -64(%esi,%edx),%xmm3
+        
+        movdqa  %xmm0,-16(%edi,%edx)
+        movdqa  %xmm1,-32(%edi,%edx)
+        movdqa  %xmm2,-48(%edi,%edx)
+        movdqa  %xmm3,-64(%edi,%edx)
+        
+        subl    $64,%edx
+        jne     LReverseUnalignedLoop
+        
+        jmp     LReverseShort           // copy remaining 0..63 bytes and done
+
+
+	COMMPAGE_DESCRIPTOR(bcopy_sse3,_COMM_PAGE_BCOPY,kHasSSE2+kCache64,kHasSupplementalSSE3)
diff --git a/osfmk/i386/commpage/bcopy_sse4.s b/osfmk/i386/commpage/bcopy_sse4.s
new file mode 100644
index 000000000..b7ba42383
--- /dev/null
+++ b/osfmk/i386/commpage/bcopy_sse4.s
@@ -0,0 +1,805 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+        
+#include <machine/cpu_capabilities.h>
+#include <machine/commpage.h>
+
+/*
+ * The bcopy/memcpy loops, tuned for Pentium-M class processors with SSE4
+ * and 64-byte cache lines.
+ *
+ * The following #defines are tightly coupled to the u-architecture:
+ */
+
+#define kShort  80			// too short to bother with SSE (must be >=80)
+#define kVeryLong   (500*1024)          // large enough for non-temporal stores (must be >= 8192)
+#define kFastUCode  ((16*1024)-15)	// cutoff for microcode fastpath for "rep/movsl"
+
+
+// void bcopy(const void *src, void *dst, size_t len);
+ 
+        .text
+        .align	5, 0x90
+LZero:
+Lbcopy_sse4:				// void bcopy(const void *src, void *dst, size_t len)
+	pushl	%ebp			// set up a frame for backtraces
+	movl	%esp,%ebp
+        pushl   %esi
+        pushl   %edi
+        movl    8(%ebp),%esi		// get source ptr
+        movl    12(%ebp),%edi           // get dest ptr
+        movl    16(%ebp),%ecx           // get length
+        movl    %edi,%edx
+        subl    %esi,%edx               // (dest - source)
+        cmpl    %ecx,%edx               // must move in reverse if (dest - source) < length
+        jb      LReverseIsland
+        cmpl    $(kShort),%ecx          // long enough to bother with SSE?
+        jbe     Lshort			// no
+	jmp	LNotShort
+
+//
+// void *memcpy(void *dst, const void *src, size_t len);
+// void *memmove(void *dst, const void *src, size_t len);
+//
+// NB: These need to be 32 bytes from bcopy():
+//
+
+        .align	5, 0x90
+Lmemcpy:				// void *memcpy(void *dst, const void *src, size_t len)
+Lmemmove:				// void *memmove(void *dst, const void *src, size_t len)
+	pushl	%ebp			// set up a frame for backtraces
+	movl	%esp,%ebp
+        pushl   %esi
+        pushl   %edi
+        movl    8(%ebp),%edi		// get dest ptr
+        movl    12(%ebp),%esi           // get source ptr
+        movl    16(%ebp),%ecx           // get length
+        movl    %edi,%edx
+        subl    %esi,%edx               // (dest - source)
+        cmpl    %ecx,%edx               // must move in reverse if (dest - source) < length
+        jb      LReverseIsland
+        cmpl    $(kShort),%ecx          // long enough to bother with SSE?
+        ja      LNotShort               // yes
+        
+// Handle short forward copies.  As the most common case, this is the fall-through path.
+//      ecx = length (<= kShort)
+//      esi = source ptr
+//      edi = dest ptr
+
+Lshort:
+	movl    %ecx,%edx		// copy length
+	shrl	$2,%ecx			// get #doublewords
+	jz	LLeftovers
+2:					// loop copying doublewords
+	movl	(%esi),%eax
+	addl	$4,%esi
+	movl	%eax,(%edi)
+	addl	$4,%edi
+	dec	%ecx
+	jnz	2b
+LLeftovers:				// handle leftover bytes (0..3) in last word
+	andl	$3,%edx			// any leftover bytes?
+	jz	Lexit
+4:					// loop copying bytes
+	movb	(%esi),%al
+	inc	%esi
+	movb	%al,(%edi)
+	inc	%edi
+	dec	%edx
+	jnz	4b
+Lexit:
+        movl    8(%ebp),%eax		// get return value (dst ptr) for memcpy/memmove
+        popl    %edi
+        popl    %esi
+	popl	%ebp
+        ret
+
+
+LReverseIsland:				// keep the "jb" above a short branch...
+	jmp	LReverse		// ...because reverse moves are uncommon
+
+
+// Handle forward moves that are long enough to justify use of SSE3.
+// First, 16-byte align the destination.
+//      ecx = length (> kShort)
+//      esi = source ptr
+//      edi = dest ptr
+
+LNotShort:
+        cmpl    $(kVeryLong),%ecx       // long enough to justify heavyweight loops?
+        movl    %edi,%edx               // copy destination
+        jae     LVeryLong		// use very-long-operand path
+        negl    %edx
+        andl    $15,%edx                // get #bytes to align destination
+	jz	LDestAligned		// already aligned
+        subl    %edx,%ecx               // decrement length
+1:					// loop copying 1..15 bytes
+	movb	(%esi),%al
+	inc	%esi
+	movb	%al,(%edi)
+	inc	%edi
+	dec	%edx
+	jnz	1b
+        
+// Destination is now aligned.  Dispatch to one of sixteen loops over 64-byte chunks,
+// based on the alignment of the source.  All vector loads and stores are aligned.
+// Even though this means we have to shift and repack vectors, doing so is much faster
+// than unaligned loads.  Since kShort>=80 and we've moved at most 15 bytes already,
+// there is at least one chunk.  When we enter the copy loops, the following registers
+// are set up:
+//      ecx = residual length (0..63)
+//	edx = -(length to move), a multiple of 64
+//      esi = ptr to 1st source byte not to move (unaligned)
+//      edi = ptr to 1st dest byte not to move (aligned)
+
+LDestAligned:
+        movl    %ecx,%edx               // copy length
+	movl	%esi,%eax		// copy source address
+        andl    $63,%ecx                // get remaining bytes for Lshort
+        andl    $-64,%edx               // get number of bytes we will copy in inner loop
+	andl	$15,%eax		// mask to low 4 bits of source address
+        addl    %edx,%esi               // point to 1st byte not copied
+        addl    %edx,%edi
+        negl    %edx                    // now generate offset to 1st byte to be copied
+	movl	(_COMM_PAGE_BCOPY+LTable-LZero)(,%eax,4),%eax
+	jmp	*%eax
+	
+	.align	2
+LTable:					// table of copy loop addresses
+	.long	LMod0 + _COMM_PAGE_BCOPY - LZero
+	.long	LMod1 + _COMM_PAGE_BCOPY - LZero
+	.long	LMod2 + _COMM_PAGE_BCOPY - LZero
+	.long	LMod3 + _COMM_PAGE_BCOPY - LZero
+	.long	LMod4 + _COMM_PAGE_BCOPY - LZero
+	.long	LMod5 + _COMM_PAGE_BCOPY - LZero
+	.long	LMod6 + _COMM_PAGE_BCOPY - LZero
+	.long	LMod7 + _COMM_PAGE_BCOPY - LZero
+	.long	LMod8 + _COMM_PAGE_BCOPY - LZero
+	.long	LMod9 + _COMM_PAGE_BCOPY - LZero
+	.long	LMod10 + _COMM_PAGE_BCOPY - LZero
+	.long	LMod11 + _COMM_PAGE_BCOPY - LZero
+	.long	LMod12 + _COMM_PAGE_BCOPY - LZero
+	.long	LMod13 + _COMM_PAGE_BCOPY - LZero
+	.long	LMod14 + _COMM_PAGE_BCOPY - LZero
+	.long	LMod15 + _COMM_PAGE_BCOPY - LZero
+
+
+// Very long forward moves.  These are at least several pages.  They are special cased
+// and aggressively optimized, not so much because they are common or useful, but
+// because they are subject to benchmark.  There isn't enough room for them in the
+// area reserved on the commpage for bcopy, so we put them elsewhere.  We call
+// the longcopy routine using the normal ABI.
+
+LVeryLong:
+	pushl	%ecx			// length (>= kVeryLong)
+	pushl	%esi			// source ptr
+	pushl	%edi			// dest ptr
+	movl	$(_COMM_PAGE_LONGCOPY),%eax
+	call	*%eax			// do the long copy
+	addl	$12,%esp		// pop off our parameters
+	jmp	Lexit
+
+
+// On Pentium-M, the microcode for "rep/movsl" is faster than SSE for 8-byte
+// aligned operands from about 32KB up to kVeryLong for the hot cache case, and from
+// about 256 bytes up to kVeryLong for cold caches.  This is because the microcode
+// avoids having to read destination cache lines that will be completely overwritten.
+// The cutoff we use (ie, kFastUCode) must somehow balance the two cases, since
+// we do not know if the destination is in cache or not.
+
+Lfastpath:
+        addl    %edx,%esi               // restore ptrs to 1st byte of source and dest
+        addl    %edx,%edi
+	negl	%edx			// make length positive
+	orl	%edx,%ecx		// restore total #bytes remaining to move
+	cld				// we'll move forward
+	movl	%ecx,%edx		// copy total length to move
+	shrl	$2,%ecx			// compute #words to move
+	rep				// the u-code will optimize this
+	movsl
+	jmp	LLeftovers		// handle 0..3 leftover bytes
+
+
+// Forward loop for medium length operands in which low four bits of %esi == 0000
+
+LMod0:
+	cmpl	$(-kFastUCode),%edx	// %edx == -length, where (length < kVeryLong)
+	jle	Lfastpath		// long enough for fastpath in microcode
+	jmp	1f
+	.align	4,0x90			// 16-byte align inner loops
+1:					// loop over 64-byte chunks
+        movdqa  (%esi,%edx),%xmm0
+        movdqa  16(%esi,%edx),%xmm1
+        movdqa  32(%esi,%edx),%xmm2
+        movdqa  48(%esi,%edx),%xmm3
+
+        movdqa  %xmm0,(%edi,%edx)
+        movdqa  %xmm1,16(%edi,%edx)
+        movdqa  %xmm2,32(%edi,%edx)
+        movdqa  %xmm3,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %esi == 0001
+
+LMod1:
+	movdqa	-1(%esi,%edx),%xmm0	// prime the loop by loading 1st quadword
+1:					// loop over 64-byte chunks
+        movdqa  15(%esi,%edx),%xmm1
+        movdqa  31(%esi,%edx),%xmm2
+        movdqa  47(%esi,%edx),%xmm3
+        movdqa  63(%esi,%edx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$1,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$1,%xmm2,%xmm3
+	palignr	$1,%xmm1,%xmm2
+	palignr	$1,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%edi,%edx)
+        movdqa  %xmm2,16(%edi,%edx)
+        movdqa  %xmm3,32(%edi,%edx)
+        movdqa  %xmm4,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %esi == 0010
+
+LMod2:
+	movdqa	-2(%esi,%edx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  14(%esi,%edx),%xmm1
+        movdqa  30(%esi,%edx),%xmm2
+        movdqa  46(%esi,%edx),%xmm3
+        movdqa  62(%esi,%edx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$2,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$2,%xmm2,%xmm3
+	palignr	$2,%xmm1,%xmm2
+	palignr	$2,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%edi,%edx)
+        movdqa  %xmm2,16(%edi,%edx)
+        movdqa  %xmm3,32(%edi,%edx)
+        movdqa  %xmm4,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %esi == 0011
+
+LMod3:
+	movdqa	-3(%esi,%edx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  13(%esi,%edx),%xmm1
+        movdqa  29(%esi,%edx),%xmm2
+        movdqa  45(%esi,%edx),%xmm3
+        movdqa  61(%esi,%edx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$3,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$3,%xmm2,%xmm3
+	palignr	$3,%xmm1,%xmm2
+	palignr	$3,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%edi,%edx)
+        movdqa  %xmm2,16(%edi,%edx)
+        movdqa  %xmm3,32(%edi,%edx)
+        movdqa  %xmm4,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+	
+	
+// Forward loop for medium length operands in which low four bits of %esi == 0100
+// We use the float single data type in order to use "movss" to merge vectors.
+
+LMod4:
+	movaps	-4(%esi,%edx),%xmm0	// 4-byte aligned: prime the loop
+	jmp	1f
+	.align	4,0x90
+1:					// loop over 64-byte chunks
+        movaps  12(%esi,%edx),%xmm1
+        movaps  28(%esi,%edx),%xmm2
+	movss	%xmm1,%xmm0		// copy low 4 bytes of source into destination
+	pshufd	$(0x39),%xmm0,%xmm0	// rotate right 4 bytes (mask -- 00 11 10 01)
+        movaps  44(%esi,%edx),%xmm3
+	movss	%xmm2,%xmm1
+	pshufd	$(0x39),%xmm1,%xmm1
+	movaps	60(%esi,%edx),%xmm4
+	movss	%xmm3,%xmm2
+	pshufd	$(0x39),%xmm2,%xmm2
+
+        movaps  %xmm0,(%edi,%edx)
+	movss	%xmm4,%xmm3
+	pshufd	$(0x39),%xmm3,%xmm3
+        movaps  %xmm1,16(%edi,%edx)
+        movaps  %xmm2,32(%edi,%edx)
+	movaps	%xmm4,%xmm0
+        movaps  %xmm3,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %esi == 0101
+
+LMod5:
+	movdqa	-5(%esi,%edx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  11(%esi,%edx),%xmm1
+        movdqa  27(%esi,%edx),%xmm2
+        movdqa  43(%esi,%edx),%xmm3
+        movdqa  59(%esi,%edx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$5,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$5,%xmm2,%xmm3
+	palignr	$5,%xmm1,%xmm2
+	palignr	$5,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%edi,%edx)
+        movdqa  %xmm2,16(%edi,%edx)
+        movdqa  %xmm3,32(%edi,%edx)
+        movdqa  %xmm4,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %esi == 0110
+
+LMod6:
+	movdqa	-6(%esi,%edx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  10(%esi,%edx),%xmm1
+        movdqa  26(%esi,%edx),%xmm2
+        movdqa  42(%esi,%edx),%xmm3
+        movdqa  58(%esi,%edx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$6,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$6,%xmm2,%xmm3
+	palignr	$6,%xmm1,%xmm2
+	palignr	$6,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%edi,%edx)
+        movdqa  %xmm2,16(%edi,%edx)
+        movdqa  %xmm3,32(%edi,%edx)
+        movdqa  %xmm4,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %esi == 0111
+
+LMod7:
+	movdqa	-7(%esi,%edx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  9(%esi,%edx),%xmm1
+        movdqa  25(%esi,%edx),%xmm2
+        movdqa  41(%esi,%edx),%xmm3
+        movdqa  57(%esi,%edx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$7,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$7,%xmm2,%xmm3
+	palignr	$7,%xmm1,%xmm2
+	palignr	$7,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%edi,%edx)
+        movdqa  %xmm2,16(%edi,%edx)
+        movdqa  %xmm3,32(%edi,%edx)
+        movdqa  %xmm4,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+	
+	
+// Forward loop for medium length operands in which low four bits of %esi == 1000
+// We use the float double data type in order to use "shufpd" to shift by 8 bytes.
+
+LMod8:
+	cmpl	$(-kFastUCode),%edx	// %edx == -length, where (length < kVeryLong)
+	jle	Lfastpath		// long enough for fastpath in microcode
+	movapd	-8(%esi,%edx),%xmm0	// 8-byte aligned: prime the loop
+	jmp	1f
+	.align	4,0x90
+1:					// loop over 64-byte chunks
+        movapd  8(%esi,%edx),%xmm1
+        movapd  24(%esi,%edx),%xmm2
+	shufpd	$01,%xmm1,%xmm0		// %xmm0 <- shr( %xmm0 || %xmm1, 8 bytes)
+        movapd  40(%esi,%edx),%xmm3
+	shufpd	$01,%xmm2,%xmm1
+	movapd	56(%esi,%edx),%xmm4
+	shufpd	$01,%xmm3,%xmm2
+
+        movapd  %xmm0,(%edi,%edx)
+	shufpd	$01,%xmm4,%xmm3
+        movapd  %xmm1,16(%edi,%edx)
+        movapd  %xmm2,32(%edi,%edx)
+	movapd	%xmm4,%xmm0
+        movapd  %xmm3,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %esi == 1001
+
+LMod9:
+	movdqa	-9(%esi,%edx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  7(%esi,%edx),%xmm1
+        movdqa  23(%esi,%edx),%xmm2
+        movdqa  39(%esi,%edx),%xmm3
+        movdqa  55(%esi,%edx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$9,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$9,%xmm2,%xmm3
+	palignr	$9,%xmm1,%xmm2
+	palignr	$9,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%edi,%edx)
+        movdqa  %xmm2,16(%edi,%edx)
+        movdqa  %xmm3,32(%edi,%edx)
+        movdqa  %xmm4,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %esi == 1010
+
+LMod10:
+	movdqa	-10(%esi,%edx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  6(%esi,%edx),%xmm1
+        movdqa  22(%esi,%edx),%xmm2
+        movdqa  38(%esi,%edx),%xmm3
+        movdqa  54(%esi,%edx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$10,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$10,%xmm2,%xmm3
+	palignr	$10,%xmm1,%xmm2
+	palignr	$10,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%edi,%edx)
+        movdqa  %xmm2,16(%edi,%edx)
+        movdqa  %xmm3,32(%edi,%edx)
+        movdqa  %xmm4,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %esi == 1011
+
+LMod11:
+	movdqa	-11(%esi,%edx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  5(%esi,%edx),%xmm1
+        movdqa  21(%esi,%edx),%xmm2
+        movdqa  37(%esi,%edx),%xmm3
+        movdqa  53(%esi,%edx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$11,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$11,%xmm2,%xmm3
+	palignr	$11,%xmm1,%xmm2
+	palignr	$11,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%edi,%edx)
+        movdqa  %xmm2,16(%edi,%edx)
+        movdqa  %xmm3,32(%edi,%edx)
+        movdqa  %xmm4,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+	
+
+// Forward loop for medium length operands in which low four bits of %esi == 1100
+// We use the float single data type in order to use "movss" to merge vectors.
+
+LMod12:
+	movss	(%esi,%edx),%xmm0	// prefetch 1st four bytes of source, right justified
+	jmp	1f
+	.align	4,0x90
+1:					// loop over 64-byte chunks
+	pshufd	$(0x93),4(%esi,%edx),%xmm1 // load and rotate right 12 bytes (mask -- 10 01 00 11)
+	pshufd	$(0x93),20(%esi,%edx),%xmm2
+	pshufd	$(0x93),36(%esi,%edx),%xmm3
+	pshufd	$(0x93),52(%esi,%edx),%xmm4
+	
+	movaps	%xmm4,%xmm5
+	movss	%xmm3,%xmm4		// copy low 4 bytes of source into destination
+	movss	%xmm2,%xmm3
+	movss	%xmm1,%xmm2
+	movss	%xmm0,%xmm1
+	
+        movaps  %xmm1,(%edi,%edx)
+        movaps  %xmm2,16(%edi,%edx)
+	movaps	%xmm5,%xmm0
+        movaps  %xmm3,32(%edi,%edx)
+        movaps  %xmm4,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %esi == 1101
+
+LMod13:
+	movdqa	-13(%esi,%edx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  3(%esi,%edx),%xmm1
+        movdqa  19(%esi,%edx),%xmm2
+        movdqa  35(%esi,%edx),%xmm3
+        movdqa  51(%esi,%edx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$13,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$13,%xmm2,%xmm3
+	palignr	$13,%xmm1,%xmm2
+	palignr	$13,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%edi,%edx)
+        movdqa  %xmm2,16(%edi,%edx)
+        movdqa  %xmm3,32(%edi,%edx)
+        movdqa  %xmm4,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %esi == 1110
+
+LMod14:
+	movdqa	-14(%esi,%edx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  2(%esi,%edx),%xmm1
+        movdqa  18(%esi,%edx),%xmm2
+        movdqa  34(%esi,%edx),%xmm3
+        movdqa  50(%esi,%edx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$14,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$14,%xmm2,%xmm3
+	palignr	$14,%xmm1,%xmm2
+	palignr	$14,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%edi,%edx)
+        movdqa  %xmm2,16(%edi,%edx)
+        movdqa  %xmm3,32(%edi,%edx)
+        movdqa  %xmm4,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %esi == 1111
+
+LMod15:
+	movdqa	-15(%esi,%edx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  1(%esi,%edx),%xmm1
+        movdqa  17(%esi,%edx),%xmm2
+        movdqa  33(%esi,%edx),%xmm3
+        movdqa  49(%esi,%edx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$15,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$15,%xmm2,%xmm3
+	palignr	$15,%xmm1,%xmm2
+	palignr	$15,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%edi,%edx)
+        movdqa  %xmm2,16(%edi,%edx)
+        movdqa  %xmm3,32(%edi,%edx)
+        movdqa  %xmm4,48(%edi,%edx)
+        
+        addl    $64,%edx
+        jnz     1b
+        
+        jmp     Lshort                  // copy remaining 0..63 bytes and done
+        
+
+// Reverse moves.  These are not optimized as aggressively as their forward
+// counterparts, as they are only used with destructive overlap.
+//      ecx = length
+//      esi = source ptr
+//      edi = dest ptr
+
+LReverse:
+        addl    %ecx,%esi               // point to end of strings
+        addl    %ecx,%edi
+        cmpl    $(kShort),%ecx          // long enough to bother with SSE?
+        ja      LReverseNotShort        // yes
+
+// Handle reverse short copies.
+//      ecx = length
+//      esi = one byte past end of source
+//      edi = one byte past end of dest
+
+LReverseShort:
+	movl    %ecx,%edx		// copy length
+	shrl	$2,%ecx			// #words
+	jz	3f
+1:
+	subl	$4,%esi
+	movl	(%esi),%eax
+	subl	$4,%edi
+	movl	%eax,(%edi)
+	dec	%ecx
+	jnz	1b
+3:
+	andl	$3,%edx			// bytes?
+	jz	5f
+4:
+	dec	%esi
+	movb	(%esi),%al
+	dec	%edi
+	movb	%al,(%edi)
+	dec	%edx
+	jnz	4b
+5:
+        movl    8(%ebp),%eax		// get return value (dst ptr) for memcpy/memmove
+        popl    %edi
+        popl    %esi
+	popl	%ebp
+        ret
+
+// Handle a reverse move long enough to justify using SSE.
+//      ecx = length
+//      esi = one byte past end of source
+//      edi = one byte past end of dest
+
+LReverseNotShort:
+        movl    %edi,%edx               // copy destination
+        andl    $15,%edx                // get #bytes to align destination
+        je      LReverseDestAligned     // already aligned
+        subl	%edx,%ecx		// adjust length
+1:					// loop copying 1..15 bytes
+	dec	%esi
+	movb	(%esi),%al
+	dec	%edi
+	movb	%al,(%edi)
+	dec	%edx
+	jnz	1b
+        
+// Destination is now aligned.  Prepare for reverse loops.
+
+LReverseDestAligned:
+        movl    %ecx,%edx               // copy length
+        andl    $63,%ecx                // get remaining bytes for Lshort
+        andl    $-64,%edx               // get number of bytes we will copy in inner loop
+        subl    %edx,%esi               // point to endpoint of copy
+        subl    %edx,%edi
+	testl	$15,%esi		// is source aligned too?
+        jnz     LReverseUnalignedLoop   // no
+
+LReverseAlignedLoop:                    // loop over 64-byte chunks
+        movdqa  -16(%esi,%edx),%xmm0
+        movdqa  -32(%esi,%edx),%xmm1
+        movdqa  -48(%esi,%edx),%xmm2
+        movdqa  -64(%esi,%edx),%xmm3
+
+        movdqa  %xmm0,-16(%edi,%edx)
+        movdqa  %xmm1,-32(%edi,%edx)
+        movdqa  %xmm2,-48(%edi,%edx)
+        movdqa  %xmm3,-64(%edi,%edx)
+        
+        subl    $64,%edx
+        jne     LReverseAlignedLoop
+        
+        jmp     LReverseShort           // copy remaining 0..63 bytes and done
+
+    
+// Reverse, unaligned loop.  LDDQU==MOVDQU on these machines.
+        
+LReverseUnalignedLoop:                  // loop over 64-byte chunks
+        movdqu  -16(%esi,%edx),%xmm0
+        movdqu  -32(%esi,%edx),%xmm1
+        movdqu  -48(%esi,%edx),%xmm2
+        movdqu  -64(%esi,%edx),%xmm3
+        
+        movdqa  %xmm0,-16(%edi,%edx)
+        movdqa  %xmm1,-32(%edi,%edx)
+        movdqa  %xmm2,-48(%edi,%edx)
+        movdqa  %xmm3,-64(%edi,%edx)
+        
+        subl    $64,%edx
+        jne     LReverseUnalignedLoop
+        
+        jmp     LReverseShort           // copy remaining 0..63 bytes and done
+
+
+	COMMPAGE_DESCRIPTOR(bcopy_sse4,_COMM_PAGE_BCOPY,kHasSSE3+kHasSupplementalSSE3+kCache64,0)
diff --git a/osfmk/i386/commpage/bcopy_sse4_64.s b/osfmk/i386/commpage/bcopy_sse4_64.s
new file mode 100644
index 000000000..cd4747dc3
--- /dev/null
+++ b/osfmk/i386/commpage/bcopy_sse4_64.s
@@ -0,0 +1,797 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+        
+#include <machine/cpu_capabilities.h>
+#include <machine/commpage.h>
+
+/*
+ * The bcopy/memcpy loops, tuned for 64-bit Pentium-M class processors with 
+ * SSE4 and 64-byte cache lines.  This is the 64-bit version.
+ *
+ * The following #defines are tightly coupled to the u-architecture:
+ */
+
+#define kShort  80			// too short to bother with SSE (must be >=80)
+#define kVeryLong   (500*1024)          // large enough for non-temporal stores (>=8192 and <2GB)
+#define kFastUCode  ((16*1024)-15)	// cutoff for microcode fastpath for "rep/movsl"
+
+
+// void bcopy(const void *src, void *dst, size_t len);
+ 
+        .text
+	.code64
+        .align 5, 0x90
+LZero:
+Lbcopy_sse4_64:				// void bcopy(const void *src, void *dst, size_t len)
+	pushq	%rbp			// set up a frame for backtraces
+	movq	%rsp,%rbp
+	movq	%rsi,%rax		// copy dest ptr
+	movq	%rdi,%rsi		// xchange source and dest ptrs
+	movq	%rax,%rdi
+        subq    %rsi,%rax               // (dest - source)
+        cmpq    %rdx,%rax               // must move in reverse if (dest - source) < length
+        jb      LReverseIsland
+        cmpq    $(kShort),%rdx          // long enough to bother with SSE?
+        jbe     LShort			// no
+	jmp	LNotShort
+
+//
+// void *memcpy(void *dst, const void *src, size_t len);
+// void *memmove(void *dst, const void *src, size_t len);
+//
+// NB: These need to be 32 bytes from bcopy():
+//
+
+        .align	5, 0x90
+Lmemcpy:				// void *memcpy(void *dst, const void *src, size_t len)
+Lmemmove:				// void *memmove(void *dst, const void *src, size_t len)
+	pushq	%rbp			// set up a frame for backtraces
+	movq	%rsp,%rbp
+	movq	%rdi,%r11		// save return value here        
+        movq    %rdi,%rax
+        subq    %rsi,%rax               // (dest - source)
+        cmpq    %rdx,%rax               // must move in reverse if (dest - source) < length
+        jb      LReverseIsland
+        cmpq    $(kShort),%rdx          // long enough to bother with SSE?
+        ja      LNotShort               // yes
+        
+// Handle short forward copies.  As the most common case, this is the fall-through path.
+//      rdx = length (<= kShort)
+//      rsi = source ptr
+//      rdi = dest ptr
+
+LShort:
+	movl    %edx,%ecx		// copy length using 32-bit operation
+	shrl	$2,%ecx			// get #doublewords
+	jz	LLeftovers
+2:					// loop copying doublewords
+	movl	(%rsi),%eax
+	addq	$4,%rsi
+	movl	%eax,(%rdi)
+	addq	$4,%rdi
+	decl	%ecx
+	jnz	2b
+LLeftovers:				// handle leftover bytes (0..3) in last word
+	andl	$3,%edx			// any leftover bytes?
+	jz	5f
+4:					// loop copying bytes
+	movb	(%rsi),%al
+	incq	%rsi
+	movb	%al,(%rdi)
+	incq	%rdi
+	decl	%edx
+	jnz	4b
+5:
+        movq	%r11,%rax		// get return value (dst ptr) for memcpy/memmove
+	popq	%rbp
+        ret
+
+
+LReverseIsland:				// keep the "jb" above a short branch...
+	jmp	LReverse		// ...because reverse moves are uncommon
+
+
+// Handle forward moves that are long enough to justify use of SSE.
+// First, 16-byte align the destination.
+//      rdx = length (> kShort)
+//      rsi = source ptr
+//      rdi = dest ptr
+
+LNotShort:
+        cmpq    $(kVeryLong),%rdx       // long enough to justify heavyweight loops?
+        jae     LVeryLong		// use very-long-operand path
+        movl    %edi,%ecx               // copy low half of destination ptr
+        negl    %ecx
+        andl    $15,%ecx                // get #bytes to align destination
+	jz	LDestAligned		// already aligned
+        subl    %ecx,%edx               // decrement length
+	rep				// align destination
+	movsb
+
+        
+// Destination is now aligned.  Dispatch to one of sixteen loops over 64-byte chunks,
+// based on the alignment of the source.  All vector loads and stores are aligned.
+// Even though this means we have to shift and repack vectors, doing so is much faster
+// than unaligned loads.  Since kShort>=80 and we've moved at most 15 bytes already,
+// there is at least one chunk.  When we enter the copy loops, the following registers
+// are set up:
+//      rdx = residual length (0..63)
+//	rcx = -(length to move), a multiple of 64 less than 2GB
+//      rsi = ptr to 1st source byte not to move (unaligned)
+//      rdi = ptr to 1st dest byte not to move (aligned)
+
+LDestAligned:
+        movl    %edx,%ecx               // copy length
+	movl	%esi,%eax		// copy low half of source address
+        andl    $63,%edx                // get remaining bytes for LShort
+	andl	$15,%eax		// mask to low 4 bits of source address
+        andl    $-64,%ecx               // get number of bytes we will copy in inner loop
+// We'd like to use lea with rip-relative addressing, but cannot in a .code64 block.
+//	lea	LTable(%rip),%r8	// point to dispatch table
+	movq	$(_COMM_PAGE_32_TO_64(_COMM_PAGE_BCOPY)),%r8 // work around 4586528
+	addq	$(LTable-LZero),%r8	// work around 4586528
+        addq    %rcx,%rsi               // point to 1st byte not copied
+        addq    %rcx,%rdi
+	movl	(%r8,%rax,4),%eax	// get offset of routine
+        negq    %rcx                    // now generate offset to 1st byte to be copied
+	addq	%r8,%rax		// generate address of copy loop
+	jmp	*%rax			// enter copy loop, selected by source alignment
+
+	.align	2
+LTable:					// table of copy loop addresses
+	.long	(LMod0 - LTable)
+	.long	(LMod1 - LTable)
+	.long	(LMod2 - LTable)
+	.long	(LMod3 - LTable)
+	.long	(LMod4 - LTable)
+	.long	(LMod5 - LTable)
+	.long	(LMod6 - LTable)
+	.long	(LMod7 - LTable)
+	.long	(LMod8 - LTable)
+	.long	(LMod9 - LTable)
+	.long	(LMod10 - LTable)
+	.long	(LMod11 - LTable)
+	.long	(LMod12 - LTable)
+	.long	(LMod13 - LTable)
+	.long	(LMod14 - LTable)
+	.long	(LMod15 - LTable)
+
+
+// Very long forward moves.  These are at least several pages.  They are special cased
+// and aggressively optimized, not so much because they are common or useful, but
+// because they are subject to benchmark.  There isn't enough room for them in the
+// area reserved on the commpage for bcopy, so we put them elsewhere.  We call
+// the longcopy routine using the normal ABI:
+//      rdi = dest
+//      rsi = source
+//      rdx = length (>= kVeryLong bytes)
+
+LVeryLong:
+	pushq	%r11			// save return value
+	movq	$_COMM_PAGE_32_TO_64(_COMM_PAGE_LONGCOPY),%rax
+	call	*%rax			// call very long operand routine
+	popq	%rax			// pop return value
+	popq	%rbp
+	ret	
+
+
+// On Pentium-M, the microcode for "rep/movsl" is faster than SSE for 16-byte
+// aligned operands from about 32KB up to kVeryLong for the hot cache case, and from
+// about 256 bytes up to kVeryLong for cold caches.  This is because the microcode
+// avoids having to read destination cache lines that will be completely overwritten.
+// The cutoff we use (ie, kFastUCode) must somehow balance the two cases, since
+// we do not know if the destination is in cache or not.
+
+Lfastpath:
+        addq    %rcx,%rsi               // restore ptrs to 1st byte of source and dest
+        addq    %rcx,%rdi
+	negl	%ecx			// make length positive (known to be < 2GB)
+	orl	%edx,%ecx		// restore total #bytes remaining to move
+	cld				// we'll move forward
+	shrl	$2,%ecx			// compute #words to move
+	rep				// the u-code will optimize this
+	movsl
+	jmp	LLeftovers		// handle 0..3 leftover bytes
+
+
+// Forward loop for medium length operands in which low four bits of %rsi == 0000
+
+LMod0:
+	cmpl	$(-kFastUCode),%ecx	// %rcx == -length, where (length < kVeryLong)
+	jle	Lfastpath		// long enough for fastpath in microcode
+	jmp	1f
+	.align	4,0x90			// 16-byte align inner loops
+1:					// loop over 64-byte chunks
+        movdqa  (%rsi,%rcx),%xmm0
+        movdqa  16(%rsi,%rcx),%xmm1
+        movdqa  32(%rsi,%rcx),%xmm2
+        movdqa  48(%rsi,%rcx),%xmm3
+
+        movdqa  %xmm0,(%rdi,%rcx)
+        movdqa  %xmm1,16(%rdi,%rcx)
+        movdqa  %xmm2,32(%rdi,%rcx)
+        movdqa  %xmm3,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %rsi == 0001
+
+LMod1:
+	movdqa	-1(%rsi,%rcx),%xmm0	// prime the loop by loading 1st quadword
+1:					// loop over 64-byte chunks
+        movdqa  15(%rsi,%rcx),%xmm1
+        movdqa  31(%rsi,%rcx),%xmm2
+        movdqa  47(%rsi,%rcx),%xmm3
+        movdqa  63(%rsi,%rcx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$1,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$1,%xmm2,%xmm3
+	palignr	$1,%xmm1,%xmm2
+	palignr	$1,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%rdi,%rcx)
+        movdqa  %xmm2,16(%rdi,%rcx)
+        movdqa  %xmm3,32(%rdi,%rcx)
+        movdqa  %xmm4,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %rsi == 0010
+
+LMod2:
+	movdqa	-2(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  14(%rsi,%rcx),%xmm1
+        movdqa  30(%rsi,%rcx),%xmm2
+        movdqa  46(%rsi,%rcx),%xmm3
+        movdqa  62(%rsi,%rcx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$2,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$2,%xmm2,%xmm3
+	palignr	$2,%xmm1,%xmm2
+	palignr	$2,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%rdi,%rcx)
+        movdqa  %xmm2,16(%rdi,%rcx)
+        movdqa  %xmm3,32(%rdi,%rcx)
+        movdqa  %xmm4,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %rsi == 0011
+
+LMod3:
+	movdqa	-3(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  13(%rsi,%rcx),%xmm1
+        movdqa  29(%rsi,%rcx),%xmm2
+        movdqa  45(%rsi,%rcx),%xmm3
+        movdqa  61(%rsi,%rcx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$3,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$3,%xmm2,%xmm3
+	palignr	$3,%xmm1,%xmm2
+	palignr	$3,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%rdi,%rcx)
+        movdqa  %xmm2,16(%rdi,%rcx)
+        movdqa  %xmm3,32(%rdi,%rcx)
+        movdqa  %xmm4,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+	
+	
+// Forward loop for medium length operands in which low four bits of %rsi == 0100
+// We use the float single data type in order to use "movss" to merge vectors.
+
+LMod4:
+	movaps	-4(%rsi,%rcx),%xmm0	// 4-byte aligned: prime the loop
+	jmp	1f
+	.align	4,0x90
+1:					// loop over 64-byte chunks
+        movaps  12(%rsi,%rcx),%xmm1
+        movaps  28(%rsi,%rcx),%xmm2
+	movss	%xmm1,%xmm0		// copy low 4 bytes of source into destination
+	pshufd	$(0x39),%xmm0,%xmm0	// rotate right 4 bytes (mask -- 00 11 10 01)
+        movaps  44(%rsi,%rcx),%xmm3
+	movss	%xmm2,%xmm1
+	pshufd	$(0x39),%xmm1,%xmm1
+	movaps	60(%rsi,%rcx),%xmm4
+	movss	%xmm3,%xmm2
+	pshufd	$(0x39),%xmm2,%xmm2
+
+        movaps  %xmm0,(%rdi,%rcx)
+	movss	%xmm4,%xmm3
+	pshufd	$(0x39),%xmm3,%xmm3
+        movaps  %xmm1,16(%rdi,%rcx)
+        movaps  %xmm2,32(%rdi,%rcx)
+	movaps	%xmm4,%xmm0
+        movaps  %xmm3,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %rsi == 0101
+
+LMod5:
+	movdqa	-5(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  11(%rsi,%rcx),%xmm1
+        movdqa  27(%rsi,%rcx),%xmm2
+        movdqa  43(%rsi,%rcx),%xmm3
+        movdqa  59(%rsi,%rcx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$5,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$5,%xmm2,%xmm3
+	palignr	$5,%xmm1,%xmm2
+	palignr	$5,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%rdi,%rcx)
+        movdqa  %xmm2,16(%rdi,%rcx)
+        movdqa  %xmm3,32(%rdi,%rcx)
+        movdqa  %xmm4,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %rsi == 0110
+
+LMod6:
+	movdqa	-6(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  10(%rsi,%rcx),%xmm1
+        movdqa  26(%rsi,%rcx),%xmm2
+        movdqa  42(%rsi,%rcx),%xmm3
+        movdqa  58(%rsi,%rcx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$6,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$6,%xmm2,%xmm3
+	palignr	$6,%xmm1,%xmm2
+	palignr	$6,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%rdi,%rcx)
+        movdqa  %xmm2,16(%rdi,%rcx)
+        movdqa  %xmm3,32(%rdi,%rcx)
+        movdqa  %xmm4,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %rsi == 0111
+
+LMod7:
+	movdqa	-7(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  9(%rsi,%rcx),%xmm1
+        movdqa  25(%rsi,%rcx),%xmm2
+        movdqa  41(%rsi,%rcx),%xmm3
+        movdqa  57(%rsi,%rcx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$7,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$7,%xmm2,%xmm3
+	palignr	$7,%xmm1,%xmm2
+	palignr	$7,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%rdi,%rcx)
+        movdqa  %xmm2,16(%rdi,%rcx)
+        movdqa  %xmm3,32(%rdi,%rcx)
+        movdqa  %xmm4,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+	
+	
+// Forward loop for medium length operands in which low four bits of %rsi == 1000
+// We use the float double data type in order to use "shufpd" to shift by 8 bytes.
+
+LMod8:
+	cmpl	$(-kFastUCode),%ecx	// %rcx == -length, where (length < kVeryLong)
+	jle	Lfastpath		// long enough for fastpath in microcode
+	movapd	-8(%rsi,%rcx),%xmm0	// 8-byte aligned: prime the loop
+	jmp	1f
+	.align	4,0x90
+1:					// loop over 64-byte chunks
+        movapd  8(%rsi,%rcx),%xmm1
+        movapd  24(%rsi,%rcx),%xmm2
+	shufpd	$01,%xmm1,%xmm0		// %xmm0 <- shr( %xmm0 || %xmm1, 8 bytes)
+        movapd  40(%rsi,%rcx),%xmm3
+	shufpd	$01,%xmm2,%xmm1
+	movapd	56(%rsi,%rcx),%xmm4
+	shufpd	$01,%xmm3,%xmm2
+
+        movapd  %xmm0,(%rdi,%rcx)
+	shufpd	$01,%xmm4,%xmm3
+        movapd  %xmm1,16(%rdi,%rcx)
+        movapd  %xmm2,32(%rdi,%rcx)
+	movapd	%xmm4,%xmm0
+        movapd  %xmm3,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %rsi == 1001
+
+LMod9:
+	movdqa	-9(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  7(%rsi,%rcx),%xmm1
+        movdqa  23(%rsi,%rcx),%xmm2
+        movdqa  39(%rsi,%rcx),%xmm3
+        movdqa  55(%rsi,%rcx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$9,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$9,%xmm2,%xmm3
+	palignr	$9,%xmm1,%xmm2
+	palignr	$9,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%rdi,%rcx)
+        movdqa  %xmm2,16(%rdi,%rcx)
+        movdqa  %xmm3,32(%rdi,%rcx)
+        movdqa  %xmm4,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %rsi == 1010
+
+LMod10:
+	movdqa	-10(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  6(%rsi,%rcx),%xmm1
+        movdqa  22(%rsi,%rcx),%xmm2
+        movdqa  38(%rsi,%rcx),%xmm3
+        movdqa  54(%rsi,%rcx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$10,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$10,%xmm2,%xmm3
+	palignr	$10,%xmm1,%xmm2
+	palignr	$10,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%rdi,%rcx)
+        movdqa  %xmm2,16(%rdi,%rcx)
+        movdqa  %xmm3,32(%rdi,%rcx)
+        movdqa  %xmm4,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %rsi == 1011
+
+LMod11:
+	movdqa	-11(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  5(%rsi,%rcx),%xmm1
+        movdqa  21(%rsi,%rcx),%xmm2
+        movdqa  37(%rsi,%rcx),%xmm3
+        movdqa  53(%rsi,%rcx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$11,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$11,%xmm2,%xmm3
+	palignr	$11,%xmm1,%xmm2
+	palignr	$11,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%rdi,%rcx)
+        movdqa  %xmm2,16(%rdi,%rcx)
+        movdqa  %xmm3,32(%rdi,%rcx)
+        movdqa  %xmm4,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+	
+
+// Forward loop for medium length operands in which low four bits of %rsi == 1100
+// We use the float single data type in order to use "movss" to merge vectors.
+
+LMod12:
+	movss	(%rsi,%rcx),%xmm0	// prefetch 1st four bytes of source, right justified
+	jmp	1f
+	.align	4,0x90
+1:					// loop over 64-byte chunks
+	pshufd	$(0x93),4(%rsi,%rcx),%xmm1 // load and rotate right 12 bytes (mask -- 10 01 00 11)
+	pshufd	$(0x93),20(%rsi,%rcx),%xmm2
+	pshufd	$(0x93),36(%rsi,%rcx),%xmm3
+	pshufd	$(0x93),52(%rsi,%rcx),%xmm4
+	
+	movaps	%xmm4,%xmm5
+	movss	%xmm3,%xmm4		// copy low 4 bytes of source into destination
+	movss	%xmm2,%xmm3
+	movss	%xmm1,%xmm2
+	movss	%xmm0,%xmm1
+	
+        movaps  %xmm1,(%rdi,%rcx)
+        movaps  %xmm2,16(%rdi,%rcx)
+	movaps	%xmm5,%xmm0
+        movaps  %xmm3,32(%rdi,%rcx)
+        movaps  %xmm4,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %rsi == 1101
+
+LMod13:
+	movdqa	-13(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  3(%rsi,%rcx),%xmm1
+        movdqa  19(%rsi,%rcx),%xmm2
+        movdqa  35(%rsi,%rcx),%xmm3
+        movdqa  51(%rsi,%rcx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$13,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$13,%xmm2,%xmm3
+	palignr	$13,%xmm1,%xmm2
+	palignr	$13,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%rdi,%rcx)
+        movdqa  %xmm2,16(%rdi,%rcx)
+        movdqa  %xmm3,32(%rdi,%rcx)
+        movdqa  %xmm4,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %rsi == 1110
+
+LMod14:
+	movdqa	-14(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  2(%rsi,%rcx),%xmm1
+        movdqa  18(%rsi,%rcx),%xmm2
+        movdqa  34(%rsi,%rcx),%xmm3
+        movdqa  50(%rsi,%rcx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$14,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$14,%xmm2,%xmm3
+	palignr	$14,%xmm1,%xmm2
+	palignr	$14,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%rdi,%rcx)
+        movdqa  %xmm2,16(%rdi,%rcx)
+        movdqa  %xmm3,32(%rdi,%rcx)
+        movdqa  %xmm4,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+
+
+// Forward loop for medium length operands in which low four bits of %rsi == 1111
+
+LMod15:
+	movdqa	-15(%rsi,%rcx),%xmm0	// prime the loop by loading 1st source dq
+1:					// loop over 64-byte chunks
+        movdqa  1(%rsi,%rcx),%xmm1
+        movdqa  17(%rsi,%rcx),%xmm2
+        movdqa  33(%rsi,%rcx),%xmm3
+        movdqa  49(%rsi,%rcx),%xmm4
+	
+	movdqa	%xmm0,%xmm5
+	movdqa	%xmm4,%xmm0
+
+	palignr	$15,%xmm3,%xmm4		// dest <- shr( dest || source, imm*8 )
+	palignr	$15,%xmm2,%xmm3
+	palignr	$15,%xmm1,%xmm2
+	palignr	$15,%xmm5,%xmm1
+	
+        movdqa  %xmm1,(%rdi,%rcx)
+        movdqa  %xmm2,16(%rdi,%rcx)
+        movdqa  %xmm3,32(%rdi,%rcx)
+        movdqa  %xmm4,48(%rdi,%rcx)
+        
+        addq    $64,%rcx
+        jnz     1b
+        
+        jmp     LShort                  // copy remaining 0..63 bytes and done
+	
+
+// Reverse moves.  These are not optimized as aggressively as their forward
+// counterparts, as they are only used with destructive overlap.
+//      rdx = length
+//      rsi = source ptr
+//      rdi = dest ptr
+
+LReverse:
+        addq    %rdx,%rsi               // point to end of strings
+        addq    %rdx,%rdi
+        cmpq    $(kShort),%rdx          // long enough to bother with SSE?
+        ja      LReverseNotShort        // yes
+
+// Handle reverse short copies.
+//      edx = length (<= kShort)
+//      rsi = one byte past end of source
+//      rdi = one byte past end of dest
+
+LReverseShort:
+	movl    %edx,%ecx		// copy length
+	shrl	$3,%ecx			// #quadwords
+	jz	3f
+1:
+	subq	$8,%rsi
+	movq	(%rsi),%rax
+	subq	$8,%rdi
+	movq	%rax,(%rdi)
+	decl	%ecx
+	jnz	1b
+3:
+	andl	$7,%edx			// bytes?
+	jz	5f
+4:
+	decq	%rsi
+	movb	(%rsi),%al
+	decq	%rdi
+	movb	%al,(%rdi)
+	decl	%edx
+	jnz	4b
+5:
+        movq	%r11,%rax		// get return value (dst ptr) for memcpy/memmove
+	popq	%rbp
+        ret
+
+// Handle a reverse move long enough to justify using SSE.
+//      rdx = length (> kShort)
+//      rsi = one byte past end of source
+//      rdi = one byte past end of dest
+
+LReverseNotShort:
+        movl    %edi,%ecx               // copy destination
+        andl    $15,%ecx                // get #bytes to align destination
+        je      LReverseDestAligned     // already aligned
+        subq	%rcx,%rdx		// adjust length
+1:					// loop copying 1..15 bytes
+	decq	%rsi
+	movb	(%rsi),%al
+	decq	%rdi
+	movb	%al,(%rdi)
+	decl	%ecx
+	jnz	1b
+        
+// Destination is now aligned.  Prepare for reverse loops.
+
+LReverseDestAligned:
+        movq    %rdx,%rcx               // copy length
+        andl    $63,%edx                // get remaining bytes for LReverseShort
+        andq    $-64,%rcx               // get number of bytes we will copy in inner loop
+        subq    %rcx,%rsi               // point to endpoint of copy
+        subq    %rcx,%rdi
+	testl	$15,%esi		// is source aligned too?
+        jnz     LReverseUnalignedLoop   // no
+
+LReverseAlignedLoop:                    // loop over 64-byte chunks
+        movdqa  -16(%rsi,%rcx),%xmm0
+        movdqa  -32(%rsi,%rcx),%xmm1
+        movdqa  -48(%rsi,%rcx),%xmm2
+        movdqa  -64(%rsi,%rcx),%xmm3
+
+        movdqa  %xmm0,-16(%rdi,%rcx)
+        movdqa  %xmm1,-32(%rdi,%rcx)
+        movdqa  %xmm2,-48(%rdi,%rcx)
+        movdqa  %xmm3,-64(%rdi,%rcx)
+        
+        subq    $64,%rcx
+        jne     LReverseAlignedLoop
+        
+        jmp     LReverseShort           // copy remaining 0..63 bytes and done
+
+    
+// Reverse, unaligned loop.  LDDQU==MOVDQU on these machines.
+        
+LReverseUnalignedLoop:                  // loop over 64-byte chunks
+        movdqu  -16(%rsi,%rcx),%xmm0
+        movdqu  -32(%rsi,%rcx),%xmm1
+        movdqu  -48(%rsi,%rcx),%xmm2
+        movdqu  -64(%rsi,%rcx),%xmm3
+        
+        movdqa  %xmm0,-16(%rdi,%rcx)
+        movdqa  %xmm1,-32(%rdi,%rcx)
+        movdqa  %xmm2,-48(%rdi,%rcx)
+        movdqa  %xmm3,-64(%rdi,%rcx)
+        
+        subq    $64,%rcx
+        jne     LReverseUnalignedLoop
+        
+        jmp     LReverseShort           // copy remaining 0..63 bytes and done
+
+
+	COMMPAGE_DESCRIPTOR(bcopy_sse4_64,_COMM_PAGE_BCOPY,kHasSSE3+kHasSupplementalSSE3+kCache64,0)
diff --git a/osfmk/i386/commpage/bzero_scalar.s b/osfmk/i386/commpage/bzero_scalar.s
index c55405696..8d33c61b1 100644
--- a/osfmk/i386/commpage/bzero_scalar.s
+++ b/osfmk/i386/commpage/bzero_scalar.s
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -71,10 +71,12 @@
 .text
 .align 5, 0x90
 Lbzero_scalar:
+	pushl	%ebp			/* set up a frame for backtraces */
+	movl	%esp,%ebp
         pushl   %edi
         pushl   %ebx
-        movl    12(%esp),%edi
-        movl    16(%esp),%ecx
+        movl    8(%ebp),%edi
+        movl    12(%ebp),%ecx
 
         cld                             /* set fill direction forward */
         xorl    %eax,%eax               /* set fill data to 0 */
@@ -85,7 +87,7 @@ Lbzero_scalar:
          * unaligned set.
          */
         cmpl    $0x0f,%ecx
-        jle     L1
+        jbe     L1
 
         movl    %edi,%edx               /* compute misalignment */
         negl    %edx
@@ -109,6 +111,7 @@ L1:     rep
 
         popl    %ebx
         popl    %edi
+	popl	%ebp
 	ret
 
-	COMMPAGE_DESCRIPTOR(bzero_scalar,_COMM_PAGE_BZERO,0,0)
+	COMMPAGE_DESCRIPTOR(bzero_scalar,_COMM_PAGE_BZERO,0,kHasSSE2)
diff --git a/osfmk/i386/commpage/bzero_sse3.s b/osfmk/i386/commpage/bzero_sse3.s
new file mode 100644
index 000000000..9b801c7f9
--- /dev/null
+++ b/osfmk/i386/commpage/bzero_sse3.s
@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <machine/cpu_capabilities.h>
+#include <machine/commpage.h>
+
+/*
+ * Bzero, tuned for Pentium-M class processors with SSE3
+ * and 64-byte cache lines.
+ *
+ * This routine is also used for memset(p,0,n), which is a common case
+ * since gcc sometimes silently maps bzero() into memset().  As a result,
+ * we always load the original ptr into %eax before returning.
+ */
+
+#define kShort		80		// too short to bother with SSE (must be >=80)
+#define	kVeryLong	(1024*1024)
+
+
+        .text
+        .align  5, 0x90
+Lbzero_sse3:                            // void	bzero(void *b, size_t len);
+	pushl	%ebp			// set up a frame for backtraces
+	movl	%esp,%ebp
+        pushl   %edi
+        movl    8(%ebp),%edi            // get ptr
+        movl    12(%ebp),%edx           // get length
+
+        xorl    %eax,%eax               // set fill data to 0
+        cmpl    $(kShort),%edx          // long enough for SSE?
+        jg	LNotShort               // yes
+        
+// Here for short operands or the end of long ones.
+//      %edx = length
+//      %edi = ptr
+//      %eax = zero
+
+Lshort:
+	cmpl	$16,%edx		// long enough to word align?
+	jge	3f			// yes
+	test	%edx,%edx		// length==0?
+	jz	6f
+1:
+	movb	%al,(%edi)		// zero a byte
+	inc	%edi
+	dec	%edx
+	jnz	1b
+	jmp	6f
+2:
+	movb	%al,(%edi)		// zero a byte
+	inc	%edi
+	dec	%edx
+3:
+	test	$3,%edi			// is ptr doubleword aligned?
+	jnz	2b			// no
+	movl	%edx,%ecx		// copy length
+	shrl	$2,%edx			// #doublewords to store
+4:      
+	movl	%eax,(%edi)		// zero an aligned doubleword
+	addl	$4,%edi
+	dec	%edx
+	jnz	4b
+	andl	$3,%ecx			// mask down to #bytes at end (0..3)
+	jz	6f			// none
+5:
+	movb	%al,(%edi)		// zero a byte
+	inc	%edi
+	dec	%ecx
+	jnz	5b
+6:
+	movl	8(%ebp),%eax		// get return value in case this was a call of memset()
+        popl    %edi
+	popl	%ebp
+        ret
+
+        
+// We will be using SSE, so align ptr.
+
+LNotShort:
+        movl    %edi,%ecx
+        negl    %ecx
+        andl    $15,%ecx                // mask down to #bytes to 16-byte align
+	jz	LDestAligned		// already aligned
+        subl    %ecx,%edx               // decrement length
+0:					// loop storing bytes to align the ptr
+	movb	%al,(%edi)		// pack in a byte
+	inc	%edi
+	dec	%ecx
+	jnz	0b
+	
+// Destination is now 16-byte aligned.  Prepare to loop over 64-byte chunks.
+//      %edx = length
+//      %edi = ptr
+//      %eax = zero
+
+LDestAligned:
+        movl    %edx,%ecx
+        andl    $63,%edx                // mask down to residual length (0..63)
+        andl    $-64,%ecx               // get #bytes we will zero in this loop
+        pxor    %xmm0,%xmm0             // zero an SSE register
+        addl    %ecx,%edi               // increment ptr by length to move
+	cmpl	$(kVeryLong),%ecx	// long enough to justify non-temporal stores?
+	jae	LVeryLong		// yes
+        negl    %ecx			// negate length to move
+	jmp	1f
+	
+// Loop over 64-byte chunks, storing into cache.
+
+	.align	4,0x90			// keep inner loops 16-byte aligned
+1:
+        movdqa  %xmm0,(%edi,%ecx)
+        movdqa  %xmm0,16(%edi,%ecx)
+        movdqa  %xmm0,32(%edi,%ecx)
+        movdqa  %xmm0,48(%edi,%ecx)
+        addl    $64,%ecx
+        jne     1b
+	
+	jmp	Lshort
+	
+// Very long operands: use non-temporal stores to bypass cache.
+
+LVeryLong:
+        negl    %ecx			// negate length to move
+	jmp	1f
+	
+	.align	4,0x90			// keep inner loops 16-byte aligned
+1:
+        movntdq %xmm0,(%edi,%ecx)
+        movntdq %xmm0,16(%edi,%ecx)
+        movntdq %xmm0,32(%edi,%ecx)
+        movntdq %xmm0,48(%edi,%ecx)
+        addl    $64,%ecx
+        jne     1b
+	
+        sfence                          // required by non-temporal stores
+	jmp	Lshort
+
+
+	COMMPAGE_DESCRIPTOR(bzero_sse3,_COMM_PAGE_BZERO,kHasSSE2,0)
diff --git a/osfmk/i386/commpage/bzero_sse3_64.s b/osfmk/i386/commpage/bzero_sse3_64.s
new file mode 100644
index 000000000..ee23dc001
--- /dev/null
+++ b/osfmk/i386/commpage/bzero_sse3_64.s
@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <machine/cpu_capabilities.h>
+#include <machine/commpage.h>
+
+/*
+ * Bzero, tuned for Pentium-M class processors with SSE3
+ * and 64-byte cache lines.  This is the 64-bit version.
+ *
+ * This routine is also used for memset(p,0,n), which is a common case
+ * since gcc sometimes silently maps bzero() into memset().  As a result,
+ * we always load the original ptr into %eax before returning.
+ */
+
+#define kShort		80		// too short to bother with SSE (must be >=80)
+#define	kVeryLong	(1024*1024)
+
+
+        .text
+	.code64
+        .align  5, 0x90
+Lbzero_sse3_64:                         // void	bzero(void *b, size_t len);
+	pushq	%rbp			// set up a frame for backtraces
+	movq	%rsp,%rbp
+        xorl    %eax,%eax               // set fill data to 0
+	movq	%rdi,%r11		// save original ptr as return value
+        cmpq    $(kShort),%rsi          // long enough for SSE?
+        jg	LNotShort               // yes
+        
+// Here for short operands or the end of long ones.
+//      %esi = length (<= kShort)
+//      %rdi = ptr
+//      %eax = zero
+
+Lshort:
+	cmpl	$16,%esi		// long enough to word align?
+	jge	3f			// yes
+	test	%esi,%esi		// length==0?
+	jz	6f
+1:
+	movb	%al,(%rdi)		// zero a byte
+	incq	%rdi
+	decl	%esi
+	jnz	1b
+	jmp	6f
+2:
+	movb	%al,(%rdi)		// zero a byte
+	incq	%rdi
+	decl	%esi
+3:
+	testl	$3,%edi			// is ptr doubleword aligned?
+	jnz	2b			// no
+	movl	%esi,%ecx		// copy length
+	shrl	$2,%esi			// #doublewords to store
+4:      
+	movl	%eax,(%rdi)		// zero an aligned doubleword
+	addq	$4,%rdi
+	decl	%esi
+	jnz	4b
+	andl	$3,%ecx			// mask down to #bytes at end (0..3)
+	jz	6f			// none
+5:
+	movb	%al,(%rdi)		// zero a byte
+	incq	%rdi
+	decl	%ecx
+	jnz	5b
+6:
+	movq	%r11,%rax		// set return value in case this was a call of memset()
+	popq	%rbp
+        ret
+		
+        
+// We will be using SSE, so align ptr.
+//      %rsi = length (> kShort)
+//      %rdi = ptr
+//      %eax = zero
+
+LNotShort:
+        movl    %edi,%ecx		// get #bytes to 16-byte align ptr
+        negl    %ecx
+        andl    $15,%ecx
+	jz	LDestAligned		// already aligned
+        subq    %rcx,%rsi               // decrement length
+0:					// loop storing bytes to align the ptr
+	movb	%al,(%rdi)		// pack in a byte
+	incq	%rdi
+	decl	%ecx
+	jnz	0b
+	
+// Destination is now 16-byte aligned.  Prepare to loop over 64-byte chunks.
+//      %rsi = length (> (kShort-15))
+//      %rdi = ptr (aligned)
+//      %eax = zero
+
+LDestAligned:
+        movq    %rsi,%rcx
+        andl    $63,%esi                // mask down to residual length (0..63)
+        andq    $-64,%rcx               // get #bytes we will zero in this loop
+        pxor    %xmm0,%xmm0             // zero an SSE register
+        addq    %rcx,%rdi               // increment ptr by length to move
+	cmpq	$(kVeryLong),%rcx	// long enough to justify non-temporal stores?
+	jae	LVeryLong		// yes
+        negq    %rcx			// negate length to move
+	jmp	1f
+	
+// Loop over 64-byte chunks, storing into cache.
+
+	.align	4,0x90			// keep inner loops 16-byte aligned
+1:
+        movdqa  %xmm0,(%rdi,%rcx)
+        movdqa  %xmm0,16(%rdi,%rcx)
+        movdqa  %xmm0,32(%rdi,%rcx)
+        movdqa  %xmm0,48(%rdi,%rcx)
+        addq    $64,%rcx
+        jne     1b
+	
+	jmp	Lshort
+	
+// Very long operands: use non-temporal stores to bypass cache.
+
+LVeryLong:
+        negq    %rcx			// negate length to move
+	jmp	1f
+	
+	.align	4,0x90			// keep inner loops 16-byte aligned
+1:
+        movntdq %xmm0,(%rdi,%rcx)
+        movntdq %xmm0,16(%rdi,%rcx)
+        movntdq %xmm0,32(%rdi,%rcx)
+        movntdq %xmm0,48(%rdi,%rcx)
+        addq    $64,%rcx
+        jne     1b
+	
+        sfence                          // required by non-temporal stores
+	jmp	Lshort
+
+
+	COMMPAGE_DESCRIPTOR(bzero_sse3_64,_COMM_PAGE_BZERO,kHasSSE3,0)
diff --git a/osfmk/i386/commpage/cacheflush.s b/osfmk/i386/commpage/cacheflush.s
index 2e12d0746..768d95966 100644
--- a/osfmk/i386/commpage/cacheflush.s
+++ b/osfmk/i386/commpage/cacheflush.s
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -26,19 +26,59 @@
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-#include <sys/appleapiopts.h>
 #include <machine/cpu_capabilities.h>
 #include <machine/commpage.h>
 
         .text
         .align  2, 0x90
 
+// void sysFlushDcache( void *p, size_t len );
+// 32-bit version
+
 Lsys_flush_dcache:
+	movl	4(%esp),%ecx		// get length
+	movl	8(%esp),%edx		// get ptr
+	testl	%ecx,%ecx		// length 0?
+	jz	2f			// yes
+	mfence				// ensure previous stores make it to memory
+1:
+	clflush	(%edx)			// flush a line
+	addl	$64,%edx
+	subl	$64,%ecx
+	jnc	1b
+	mfence				// make sure memory is updated before we return
+2:
+	ret
+
+	COMMPAGE_DESCRIPTOR(sys_flush_dcache,_COMM_PAGE_FLUSH_DCACHE,kCache64,0)
+
+
+// void sysFlushDcache( void *p, size_t len );
+// 64-bit version
+	.code64
+Lsys_flush_dcache_64:			// %rdi = ptr,  %rsi = length
+	testq	%rsi,%rsi		// length 0?
+	jz	2f			// yes
+	mfence				// ensure previous stores make it to memory
+1:
+	clflush	(%rdi)			// flush a line
+	addq	$64,%rdi
+	subq	$64,%rsi
+	jnc	1b
+	mfence				// make sure memory is updated before we return
+2:
 	ret
+	.code32
+	COMMPAGE_DESCRIPTOR(sys_flush_dcache_64,_COMM_PAGE_FLUSH_DCACHE,kCache64,0)
+
 
-	COMMPAGE_DESCRIPTOR(sys_flush_dcache,_COMM_PAGE_FLUSH_DCACHE,0,0)
+// void sysIcacheInvalidate( void *p, size_t len );
 
 Lsys_icache_invalidate:
+	// This is a NOP on intel processors, since the intent of the API
+	// is to make data executable, and Intel L1Is are coherent with L1D.
+	// We can use same routine both in 32 and 64-bit mode, since it is
+	// just a RET instruction.
 	ret
 
 	COMMPAGE_DESCRIPTOR(sys_icache_invalidate,_COMM_PAGE_FLUSH_ICACHE,0,0)
diff --git a/osfmk/i386/commpage/commpage.c b/osfmk/i386/commpage/commpage.c
index 1eef9af6c..328e095ab 100644
--- a/osfmk/i386/commpage/commpage.c
+++ b/osfmk/i386/commpage/commpage.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -29,18 +29,18 @@
 /*
  *	Here's what to do if you want to add a new routine to the comm page:
  *
- *		1. Add a definition for it's address in osfmk/ppc/cpu_capabilities.h,
+ *		1. Add a definition for it's address in osfmk/i386/cpu_capabilities.h,
  *		   being careful to reserve room for future expansion.
  *
  *		2. Write one or more versions of the routine, each with it's own
  *		   commpage_descriptor.  The tricky part is getting the "special",
  *		   "musthave", and "canthave" fields right, so that exactly one
  *		   version of the routine is selected for every machine.
- *		   The source files should be in osfmk/ppc/commpage/.
+ *		   The source files should be in osfmk/i386/commpage/.
  *
  *		3. Add a ptr to your new commpage_descriptor(s) in the "routines"
- *		   array in commpage_populate().  Of course, you'll also have to
- *		   declare them "extern" in commpage_populate().
+ *		   array in osfmk/i386/commpage/commpage_asm.s.  There are two
+ *		   arrays, one for the 32-bit and one for the 64-bit commpage.
  *
  *		4. Write the code in Libc to use the new routine.
  */
@@ -49,6 +49,7 @@
 #include <mach/machine.h>
 #include <mach/vm_map.h>
 #include <i386/machine_routines.h>
+#include <i386/misc_protos.h>
 #include <machine/cpu_capabilities.h>
 #include <machine/commpage.h>
 #include <machine/pmap.h>
@@ -56,16 +57,34 @@
 #include <vm/vm_map.h>
 #include <ipc/ipc_port.h>
 
+#include <kern/page_decrypt.h>
 
-extern vm_map_t	com_region_map32;	// the shared submap, set up in vm init
+/* the lists of commpage routines are in commpage_asm.s  */
+extern	commpage_descriptor*	commpage_32_routines[];
+extern	commpage_descriptor*	commpage_64_routines[];
 
-static uintptr_t next = 0;		// next available byte in comm page
-static int     	cur_routine = 0;	// comm page address of "current" routine
-static int     	matched;		// true if we've found a match for "current" routine
+/* translated commpage descriptors from commpage_sigs.c  */
+extern	commpage_descriptor sigdata_descriptor;
+extern	commpage_descriptor *ba_descriptors[];
+
+extern vm_map_t	com_region_map32;	// the shared submap, set up in vm init
+extern vm_map_t	com_region_map64;	// the shared submap, set up in vm init
 
+char	*commPagePtr32 = NULL;		// virtual addr in kernel map of 32-bit commpage
+char	*commPagePtr64 = NULL;		// ...and of 64-bit commpage
 int     _cpu_capabilities = 0;          // define the capability vector
 
-char    *commPagePtr = NULL;            // virtual address of comm page in kernel map
+int	noVMX = 0;		/* if true, do not set kHasAltivec in ppc _cpu_capabilities */
+
+void*	dsmos_blobs[3];         /* ptrs to the system integrity data in each commpage */
+int	dsmos_blob_count = 0;
+
+static uintptr_t next;			// next available byte in comm page
+static int     	cur_routine;		// comm page address of "current" routine
+static int     	matched;		// true if we've found a match for "current" routine
+
+static char    *commPagePtr;		// virtual addr in kernel map of commpage we are working on
+static size_t	commPageBaseOffset;	// add to 32-bit runtime address to get offset in commpage
 
 /* Allocate the commpage and add to the shared submap created by vm:
  * 	1. allocate a page in the kernel map (RW)
@@ -75,59 +94,61 @@ char    *commPagePtr = NULL;            // virtual address of comm page in kerne
  */
 
 static  void*
-commpage_allocate( void )
+commpage_allocate( 
+	vm_map_t	submap,			// com_region_map32 or com_region_map64
+	size_t		area_used )		// _COMM_PAGE32_AREA_USED or _COMM_PAGE64_AREA_USED
 {
-    vm_offset_t         kernel_addr;                // address of commpage in kernel map
-    vm_offset_t         zero = 0;
-    vm_size_t           size = _COMM_PAGE_AREA_LENGTH;
-    vm_map_entry_t	entry;
-    ipc_port_t          handle;
-
-    if (com_region_map32 == NULL)
-        panic("commpage map is null");
-
-    if (vm_allocate(kernel_map,&kernel_addr,_COMM_PAGE_AREA_LENGTH,VM_FLAGS_ANYWHERE))
-        panic("cannot allocate commpage");
-
-    if (vm_map_wire(kernel_map,kernel_addr,kernel_addr+_COMM_PAGE_AREA_LENGTH,VM_PROT_DEFAULT,FALSE))
-        panic("cannot wire commpage");
-
-    /* 
-     * Now that the object is created and wired into the kernel map, mark it so that no delay
-     * copy-on-write will ever be performed on it as a result of mapping it into user-space.
-     * If such a delayed copy ever occurred, we could remove the kernel's wired mapping - and
-     * that would be a real disaster.
-     *
-     * JMM - What we really need is a way to create it like this in the first place.
-     */
-    if (!vm_map_lookup_entry( kernel_map, vm_map_trunc_page(kernel_addr), &entry) || entry->is_sub_map)
-	panic("cannot find commpage entry");
-    entry->object.vm_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
-
-    if (mach_make_memory_entry( kernel_map,         // target map
-                                &size,              // size 
-                                kernel_addr,        // offset (address in kernel map)
-                                VM_PROT_DEFAULT,    // map it RW
-                                &handle,            // this is the object handle we get
-                                NULL ))             // parent_entry (what is this?)
-        panic("cannot make entry for commpage");
-
-    if (vm_map_64(  com_region_map32,               // target map (shared submap)
-                    &zero,                          // address (map into 1st page in submap)
-                    _COMM_PAGE_AREA_LENGTH,         // size
-                    0,                              // mask
-                    VM_FLAGS_FIXED,                 // flags (it must be 1st page in submap)
-                    handle,                         // port is the memory entry we just made
-                    0,                              // offset (map 1st page in memory entry)
-                    FALSE,                          // copy
-                    VM_PROT_READ,                   // cur_protection (R-only in user map)
-                    VM_PROT_READ,                   // max_protection
-                    VM_INHERIT_SHARE ))             // inheritance
-        panic("cannot map commpage");
-
-    ipc_port_release(handle);
-
-    return (void*) kernel_addr;                     // return address in kernel map
+	vm_offset_t	kernel_addr;		// address of commpage in kernel map
+	vm_offset_t	zero = 0;
+	vm_size_t	size = area_used;	// size actually populated
+	vm_map_entry_t	entry;
+	ipc_port_t	handle;
+
+	if (submap == NULL)
+		panic("commpage submap is null");
+
+	if (vm_allocate(kernel_map,&kernel_addr,area_used,VM_FLAGS_ANYWHERE))
+		panic("cannot allocate commpage");
+
+	if (vm_map_wire(kernel_map,kernel_addr,kernel_addr+area_used,VM_PROT_DEFAULT,FALSE))
+		panic("cannot wire commpage");
+
+	/* 
+	 * Now that the object is created and wired into the kernel map, mark it so that no delay
+	 * copy-on-write will ever be performed on it as a result of mapping it into user-space.
+	 * If such a delayed copy ever occurred, we could remove the kernel's wired mapping - and
+	 * that would be a real disaster.
+	 *
+	 * JMM - What we really need is a way to create it like this in the first place.
+	 */
+	if (!vm_map_lookup_entry( kernel_map, vm_map_trunc_page(kernel_addr), &entry) || entry->is_sub_map)
+		panic("cannot find commpage entry");
+	entry->object.vm_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
+
+	if (mach_make_memory_entry( kernel_map,		// target map
+				    &size,		// size 
+				    kernel_addr,	// offset (address in kernel map)
+				    VM_PROT_DEFAULT,	// map it RW
+				    &handle,		// this is the object handle we get
+				    NULL ))		// parent_entry (what is this?)
+		panic("cannot make entry for commpage");
+
+	if (vm_map_64(	submap,				// target map (shared submap)
+			&zero,				// address (map into 1st page in submap)
+			area_used,			// size
+			0,				// mask
+			VM_FLAGS_FIXED,			// flags (it must be 1st page in submap)
+			handle,				// port is the memory entry we just made
+			0,                              // offset (map 1st page in memory entry)
+			FALSE,                          // copy
+			VM_PROT_READ,                   // cur_protection (R-only in user map)
+			VM_PROT_READ,                   // max_protection
+			VM_INHERIT_SHARE ))             // inheritance
+		panic("cannot map commpage");
+
+	ipc_port_release(handle);
+
+	return (void*) kernel_addr;                     // return address in kernel map
 }
 
 /* Get address (in kernel map) of a commpage field. */
@@ -136,7 +157,7 @@ static void*
 commpage_addr_of(
     int     addr_at_runtime )
 {
-    return  (void*) ((uintptr_t)commPagePtr + addr_at_runtime - _COMM_PAGE_BASE_ADDRESS);
+    return  (void*) ((uintptr_t)commPagePtr + addr_at_runtime - commPageBaseOffset);
 }
 
 /* Determine number of CPUs on this system.  We cannot rely on
@@ -170,6 +191,9 @@ commpage_init_cpu_capabilities( void )
 	ml_cpu_get_info(&cpu_info);
 	
 	switch (cpu_info.vector_unit) {
+		case 6:
+			bits |= kHasSupplementalSSE3;
+			/* fall thru */
 		case 5:
 			bits |= kHasSSE3;
 			/* fall thru */
@@ -206,15 +230,24 @@ commpage_init_cpu_capabilities( void )
 
 	bits |= kFastThreadLocalStorage;	// we use %gs for TLS
 
+	if (cpu_mode_is64bit())			// k64Bit means processor is 64-bit capable
+		bits |= k64Bit;
+
 	_cpu_capabilities = bits;		// set kernel version for use by drivers etc
 }
 
+int
+_get_cpu_capabilities()
+{
+	return _cpu_capabilities;
+}
+
 /* Copy data into commpage. */
 
 static void
 commpage_stuff(
     int 	address,
-    void 	*source,
+    const void 	*source,
     int 	length	)
 {    
     void	*dest = commpage_addr_of(address);
@@ -227,13 +260,38 @@ commpage_stuff(
     next = ((uintptr_t)dest + length);
 }
 
+static void
+commpage_stuff_swap(
+	int	address,
+	void	*source,
+	int	length,
+	int	legacy )
+{
+	if ( legacy ) {
+		void *dest = commpage_addr_of(address);
+		dest = (void *)((uintptr_t) dest + _COMM_PAGE_SIGS_OFFSET);
+		switch (length) {
+			case 2:
+				OSWriteSwapInt16(dest, 0, *(uint16_t *)source);
+				break;
+			case 4:
+				OSWriteSwapInt32(dest, 0, *(uint32_t *)source);
+				break;
+			case 8:
+				OSWriteSwapInt64(dest, 0, *(uint64_t *)source);
+				break;
+		}
+	}
+}
 
 static void
 commpage_stuff2(
-	int address,
-	void *source,
-	int length )
+	int	address,
+	void	*source,
+	int	length,
+	int	legacy )
 {
+	commpage_stuff_swap(address, source, length, legacy);
 	commpage_stuff(address, source, length);
 }
 
@@ -247,7 +305,7 @@ commpage_stuff_routine(
     
     if (rd->commpage_address != cur_routine) {
         if ((cur_routine!=0) && (matched==0))
-            panic("commpage no match");
+            panic("commpage no match for last, next address %08x", rd->commpage_address);
         cur_routine = rd->commpage_address;
         matched = 0;
     }
@@ -257,100 +315,60 @@ commpage_stuff_routine(
     
     if ((must == rd->musthave) && (cant == 0)) {
         if (matched)
-            panic("commpage duplicate matches");
+            panic("commpage multiple matches for address %08x", rd->commpage_address);
         matched = 1;
         
         commpage_stuff(rd->commpage_address,rd->code_address,rd->code_length);
 	}
 }
 
- 
-#define COMMPAGE_DESC(name)	commpage_ ## name
-#define EXTERN_COMMPAGE_DESC(name)				\
-	extern commpage_descriptor COMMPAGE_DESC(name)
-
-EXTERN_COMMPAGE_DESC(compare_and_swap32_mp);
-EXTERN_COMMPAGE_DESC(compare_and_swap32_up);
-EXTERN_COMMPAGE_DESC(compare_and_swap64_mp);
-EXTERN_COMMPAGE_DESC(compare_and_swap64_up);
-EXTERN_COMMPAGE_DESC(atomic_add32_mp);
-EXTERN_COMMPAGE_DESC(atomic_add32_up);
-EXTERN_COMMPAGE_DESC(mach_absolute_time);
-EXTERN_COMMPAGE_DESC(spin_lock_try_mp);
-EXTERN_COMMPAGE_DESC(spin_lock_try_up);
-EXTERN_COMMPAGE_DESC(spin_lock_mp);
-EXTERN_COMMPAGE_DESC(spin_lock_up);
-EXTERN_COMMPAGE_DESC(spin_unlock);
-EXTERN_COMMPAGE_DESC(pthread_getspecific);
-EXTERN_COMMPAGE_DESC(gettimeofday);
-EXTERN_COMMPAGE_DESC(sys_flush_dcache);
-EXTERN_COMMPAGE_DESC(sys_icache_invalidate);
-EXTERN_COMMPAGE_DESC(pthread_self);
-EXTERN_COMMPAGE_DESC(relinquish);
-EXTERN_COMMPAGE_DESC(bit_test_and_set_mp);
-EXTERN_COMMPAGE_DESC(bit_test_and_set_up);
-EXTERN_COMMPAGE_DESC(bit_test_and_clear_mp);
-EXTERN_COMMPAGE_DESC(bit_test_and_clear_up);
-EXTERN_COMMPAGE_DESC(bzero_scalar);
-EXTERN_COMMPAGE_DESC(bcopy_scalar);
-EXTERN_COMMPAGE_DESC(nanotime);
-
-static  commpage_descriptor *routines[] = {
-	&COMMPAGE_DESC(compare_and_swap32_mp),
-	&COMMPAGE_DESC(compare_and_swap32_up),
-	&COMMPAGE_DESC(compare_and_swap64_mp),
-	&COMMPAGE_DESC(compare_and_swap64_up),
-	&COMMPAGE_DESC(atomic_add32_mp),
-	&COMMPAGE_DESC(atomic_add32_up),
-	&COMMPAGE_DESC(mach_absolute_time),
-	&COMMPAGE_DESC(spin_lock_try_mp),
-	&COMMPAGE_DESC(spin_lock_try_up),
-	&COMMPAGE_DESC(spin_lock_mp),
-	&COMMPAGE_DESC(spin_lock_up),
-	&COMMPAGE_DESC(spin_unlock),
-	&COMMPAGE_DESC(pthread_getspecific),
-	&COMMPAGE_DESC(gettimeofday),
-	&COMMPAGE_DESC(sys_flush_dcache),
-	&COMMPAGE_DESC(sys_icache_invalidate),
-	&COMMPAGE_DESC(pthread_self),
-	&COMMPAGE_DESC(relinquish),
-	&COMMPAGE_DESC(bit_test_and_set_mp),
-	&COMMPAGE_DESC(bit_test_and_set_up),
-	&COMMPAGE_DESC(bit_test_and_clear_mp),
-	&COMMPAGE_DESC(bit_test_and_clear_up),
-	&COMMPAGE_DESC(bzero_scalar),
-	&COMMPAGE_DESC(bcopy_scalar),
-	&COMMPAGE_DESC(nanotime),
-	NULL
-};
-
-
-/* Fill in commpage: called once, during kernel initialization, from the
- * startup thread before user-mode code is running.
- * See the top of this file for a list of what you have to do to add
- * a new routine to the commpage.
+/* Fill in the 32- or 64-bit commpage.  Called once for each.
+ * The 32-bit ("legacy") commpage has a bunch of stuff added to it
+ * for translated processes, some of which is byte-swapped.
  */
 
-void
-commpage_populate( void )
+static void
+commpage_populate_one( 
+	vm_map_t	submap,		// com_region_map32 or com_region_map64
+	char **		kernAddressPtr,	// &commPagePtr32 or &commPagePtr64
+	size_t		area_used,	// _COMM_PAGE32_AREA_USED or _COMM_PAGE64_AREA_USED
+	size_t		base_offset,	// will become commPageBaseOffset
+	commpage_descriptor** commpage_routines, // list of routine ptrs for this commpage
+	boolean_t	legacy,		// true if 32-bit commpage
+	const char*	signature )	// "commpage 32-bit" or "commpage 64-bit"
 {
    	short   c2;
 	static double   two52 = 1048576.0 * 1048576.0 * 4096.0; // 2**52
 	static double   ten6 = 1000000.0;                       // 10**6
 	commpage_descriptor **rd;
 	short   version = _COMM_PAGE_THIS_VERSION;
+	int		swapcaps;
 
-	commPagePtr = (char *)commpage_allocate();
-
-	commpage_init_cpu_capabilities();
+	next = (uintptr_t) NULL;
+	cur_routine = 0;
+	commPagePtr = (char *)commpage_allocate( submap, (vm_size_t) area_used );
+	*kernAddressPtr = commPagePtr;				// save address either in commPagePtr32 or 64
+	commPageBaseOffset = base_offset;
 
 	/* Stuff in the constants.  We move things into the comm page in strictly
 	* ascending order, so we can check for overlap and panic if so.
 	*/
-
-	commpage_stuff2(_COMM_PAGE_VERSION,&version,sizeof(short));
-	commpage_stuff(_COMM_PAGE_CPU_CAPABILITIES,&_cpu_capabilities,
-		sizeof(int));
+	commpage_stuff(_COMM_PAGE_SIGNATURE,signature,strlen(signature));
+	commpage_stuff2(_COMM_PAGE_VERSION,&version,sizeof(short),legacy);
+	commpage_stuff(_COMM_PAGE_CPU_CAPABILITIES,&_cpu_capabilities,sizeof(int));
+
+	/* excuse our magic constants, we cannot include ppc/cpu_capabilities.h */
+	/* always set kCache32 and kDcbaAvailable */
+	swapcaps =  0x44;
+	if ( _cpu_capabilities & kUP )
+		swapcaps |= (kUP + (1 << kNumCPUsShift));
+	else
+		swapcaps |= 2 << kNumCPUsShift;	/* limit #cpus to 2 */
+	if ( ! noVMX )		/* if rosetta will be emulating altivec... */
+		swapcaps |= 0x101;	/* ...then set kHasAltivec and kDataStreamsAvailable too */
+	commpage_stuff_swap(_COMM_PAGE_CPU_CAPABILITIES, &swapcaps, sizeof(int), legacy);
+	c2 = 32;
+	commpage_stuff_swap(_COMM_PAGE_CACHE_LINESIZE,&c2,2,legacy);
 
 	if (_cpu_capabilities & kCache32)
 		c2 = 32;
@@ -360,13 +378,12 @@ commpage_populate( void )
 		c2 = 128;
 	commpage_stuff(_COMM_PAGE_CACHE_LINESIZE,&c2,2);
 
-	c2 = 32;
-
-	commpage_stuff2(_COMM_PAGE_2_TO_52,&two52,8);
-
-	commpage_stuff2(_COMM_PAGE_10_TO_6,&ten6,8);
+	if ( legacy ) {
+		commpage_stuff2(_COMM_PAGE_2_TO_52,&two52,8,legacy);
+		commpage_stuff2(_COMM_PAGE_10_TO_6,&ten6,8,legacy);
+	}
 
-	for( rd = routines; *rd != NULL ; rd++ )
+	for( rd = commpage_routines; *rd != NULL ; rd++ )
 		commpage_stuff_routine(*rd);
 
 	if (!matched)
@@ -375,35 +392,54 @@ commpage_populate( void )
 	if (next > (uintptr_t)_COMM_PAGE_END)
 		panic("commpage overflow: next = 0x%08x, commPagePtr = 0x%08x", next, (uintptr_t)commPagePtr);
 
+	if ( legacy ) {
+		next = (uintptr_t) NULL;
+		for( rd = ba_descriptors; *rd != NULL ; rd++ )
+			commpage_stuff_routine(*rd);
+
+		next = (uintptr_t) NULL;
+		commpage_stuff_routine(&sigdata_descriptor);
+	}	
 
-	pmap_commpage_init((vm_offset_t) commPagePtr, _COMM_PAGE_BASE_ADDRESS, 
-			   _COMM_PAGE_AREA_LENGTH/INTEL_PGBYTES);
+	/* salt away a ptr to the system integrity data in this commpage */
+	dsmos_blobs[dsmos_blob_count++] = 
+		commpage_addr_of( _COMM_PAGE_SYSTEM_INTEGRITY );
 }
 
-/*
- * This macro prevents compiler instruction scheduling:
- */
-#define NO_REORDERING	asm volatile("" : : : "memory")
+
+/* Fill in commpages: called once, during kernel initialization, from the
+ * startup thread before user-mode code is running.
+ *
+ * See the top of this file for a list of what you have to do to add
+ * a new routine to the commpage.
+ */  
 
 void
-commpage_set_nanotime(commpage_nanotime_t *newp)
+commpage_populate( void )
 {
-	commpage_nanotime_t	*cnp;
-
-	/* Nop if commpage not set up yet */
-	if (commPagePtr == NULL)
-		return;
-
-	cnp = (commpage_nanotime_t *)commpage_addr_of(_COMM_PAGE_NANOTIME_INFO);
+	commpage_init_cpu_capabilities();
+	
+	commpage_populate_one(	com_region_map32, 
+				&commPagePtr32,
+				_COMM_PAGE32_AREA_USED,
+				_COMM_PAGE32_BASE_ADDRESS,
+				commpage_32_routines, 
+				TRUE,			/* legacy (32-bit) commpage */
+				"commpage 32-bit");
+	pmap_commpage32_init((vm_offset_t) commPagePtr32, _COMM_PAGE32_BASE_ADDRESS, 
+			   _COMM_PAGE32_AREA_USED/INTEL_PGBYTES);
+
+	if (_cpu_capabilities & k64Bit) {
+		commpage_populate_one(	com_region_map64, 
+					&commPagePtr64,
+					_COMM_PAGE64_AREA_USED,
+					_COMM_PAGE32_START_ADDRESS, /* because kernel is built 32-bit */
+					commpage_64_routines, 
+					FALSE,		/* not a legacy commpage */
+					"commpage 64-bit");
+		pmap_commpage64_init((vm_offset_t) commPagePtr64, _COMM_PAGE64_BASE_ADDRESS, 
+				   _COMM_PAGE64_AREA_USED/INTEL_PGBYTES);
+	}
 
-	/*
-	 * Update in reverse order:
-	 * check_tsc first - it's read and compared with base_tsc last.
-	 */
-	cnp->nt_check_tsc = newp->nt_base_tsc;	NO_REORDERING;
-	cnp->nt_shift     = newp->nt_shift;	NO_REORDERING;
-	cnp->nt_scale     = newp->nt_scale;	NO_REORDERING;
-	cnp->nt_base_ns   = newp->nt_base_ns;	NO_REORDERING;
-	cnp->nt_base_tsc  = newp->nt_base_tsc;
+	rtc_nanotime_init_commpage();
 }
-
diff --git a/osfmk/i386/commpage/commpage.h b/osfmk/i386/commpage/commpage.h
index caa6cba68..dd63874cf 100644
--- a/osfmk/i386/commpage/commpage.h
+++ b/osfmk/i386/commpage/commpage.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -33,8 +33,15 @@
 #include <stdint.h>
 #endif /* __ASSEMBLER__ */
 
+/* The following macro is used to generate the 64-bit commpage address for a given
+ * routine, based on its 32-bit address.  This is used in the kernel to compile
+ * the 64-bit commpage.  Since the kernel is a 32-bit object, cpu_capabilities.h
+ * only defines the 32-bit address.
+ */
+#define	_COMM_PAGE_32_TO_64( ADDRESS )	( ADDRESS + _COMM_PAGE64_START_ADDRESS - _COMM_PAGE32_START_ADDRESS )
+
+
 #ifdef	__ASSEMBLER__
-#include <machine/asm.h>
 
 #define	COMMPAGE_DESCRIPTOR(label,address,must,cant)	\
 L ## label ## _end:					;\
@@ -65,18 +72,15 @@ typedef	struct	commpage_descriptor	{
 } commpage_descriptor;
 
 
-extern	char	*commPagePtr;				// virt address of commpage in kernel map
+extern	char	*commPagePtr32;				// virt address of 32-bit commpage in kernel map
+extern	char	*commPagePtr64;				// ...and of 64-bit commpage
+
+extern	void	_commpage_set_timestamp(uint64_t abstime, uint64_t secs);
+#define commpage_set_timestamp(x, y, z)		_commpage_set_timestamp((x), (y))
 
-extern	void	commpage_set_timestamp(uint64_t tbr,uint32_t secs,uint32_t usecs,uint32_t ticks_per_sec);
+extern  void	commpage_set_nanotime(uint64_t tsc_base, uint64_t ns_base, uint32_t scale, uint32_t shift);
 
-typedef struct {
-	uint64_t	nt_base_tsc;
-	uint64_t	nt_base_ns; 
-	uint32_t	nt_scale;
-	uint32_t	nt_shift;
-	uint64_t	nt_check_tsc;
-} commpage_nanotime_t;
-extern  void	commpage_set_nanotime(commpage_nanotime_t *new_nanotime);
+#include <kern/page_decrypt.h>
 
 #endif	/* __ASSEMBLER__ */
 
diff --git a/osfmk/i386/commpage/commpage_asm.s b/osfmk/i386/commpage/commpage_asm.s
new file mode 100644
index 000000000..a7afb72ee
--- /dev/null
+++ b/osfmk/i386/commpage/commpage_asm.s
@@ -0,0 +1,196 @@
+/*
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <machine/cpu_capabilities.h>
+
+        .text
+        .align  2, 0x90
+	.globl	__commpage_set_timestamp
+/* extern void	_commpage_set_timestamp(uint64_t abstime, uint64_t secs); */
+__commpage_set_timestamp:
+	push	%ebp
+	mov	%esp,%ebp
+
+	mov	_commPagePtr32,%ecx
+	sub	$ _COMM_PAGE32_BASE_ADDRESS,%ecx
+	mov	_commPagePtr64,%edx			/* point to 64-bit commpage too */
+	mov	%edx,%eax
+	sub	$ _COMM_PAGE32_START_ADDRESS,%edx	/* because kernel is built 32-bit */
+	test	%eax,%eax
+	cmovz	%ecx,%edx				/* if no 64-bit commpage, point to 32 with both */
+
+	movl	$0,_COMM_PAGE_TIMEENABLE(%ecx)
+	movl	$0,_COMM_PAGE_TIMEENABLE(%edx)
+
+	mov	8(%ebp),%eax
+	or	12(%ebp),%eax
+	je	1f
+
+	mov	8(%ebp),%eax
+	mov	%eax,_COMM_PAGE_TIMEBASE(%ecx)
+	mov	%eax,_COMM_PAGE_TIMEBASE(%edx)
+	mov	12(%ebp),%eax
+	mov	%eax,_COMM_PAGE_TIMEBASE+4(%ecx)
+	mov	%eax,_COMM_PAGE_TIMEBASE+4(%edx)
+
+	mov	16(%ebp),%eax
+	mov	%eax,_COMM_PAGE_TIMESTAMP(%ecx)
+	mov	%eax,_COMM_PAGE_TIMESTAMP(%edx)
+	mov	20(%ebp),%eax
+	mov	%eax,_COMM_PAGE_TIMESTAMP+4(%ecx)
+	mov	%eax,_COMM_PAGE_TIMESTAMP+4(%edx)
+
+	movl	$1,_COMM_PAGE_TIMEENABLE(%ecx)
+	movl	$1,_COMM_PAGE_TIMEENABLE(%edx)
+1:
+	pop	%ebp
+	ret
+
+        .text
+        .align  2, 0x90
+	.globl	_commpage_set_nanotime
+/* extern void	commpage_set_nanotime(uint64_t tsc_base, uint64_t ns_base, uint32_t scale, uint32_t shift); */
+_commpage_set_nanotime:
+	push	%ebp
+	mov	%esp,%ebp
+
+	mov	_commPagePtr32,%ecx
+	testl	%ecx,%ecx
+	je	1f
+
+	sub	$(_COMM_PAGE_BASE_ADDRESS),%ecx
+	mov	_commPagePtr64,%edx			/* point to 64-bit commpage too */
+	mov	%edx,%eax
+	sub	$ _COMM_PAGE32_START_ADDRESS,%edx	/* because kernel is built 32-bit */
+	test	%eax,%eax
+	cmovz	%ecx,%edx				/* if no 64-bit commpage, point to 32 with both */
+
+	mov	8(%ebp),%eax
+	mov	%eax,_COMM_PAGE_NT_TSC_BASE(%ecx)
+	mov	%eax,_COMM_PAGE_NT_TSC_BASE(%edx)
+	mov	12(%ebp),%eax
+	mov	%eax,_COMM_PAGE_NT_TSC_BASE+4(%ecx)
+	mov	%eax,_COMM_PAGE_NT_TSC_BASE+4(%edx)
+
+	mov	24(%ebp),%eax
+	mov	%eax,_COMM_PAGE_NT_SCALE(%ecx)
+	mov	%eax,_COMM_PAGE_NT_SCALE(%edx)
+
+	mov	28(%ebp),%eax
+	mov	%eax,_COMM_PAGE_NT_SHIFT(%ecx)
+	mov	%eax,_COMM_PAGE_NT_SHIFT(%edx)
+
+	mov	16(%ebp),%eax
+	mov	%eax,_COMM_PAGE_NT_NS_BASE(%ecx)
+	mov	%eax,_COMM_PAGE_NT_NS_BASE(%edx)
+	mov	20(%ebp),%eax
+	mov	%eax,_COMM_PAGE_NT_NS_BASE+4(%ecx)
+	mov	%eax,_COMM_PAGE_NT_NS_BASE+4(%edx)
+1:
+	pop	%ebp
+	ret
+
+#define	CPN(routine)	_commpage_ ## routine
+
+/* pointers to the 32-bit commpage routine descriptors */
+/* WARNING: these must be sorted by commpage address! */
+	.const_data
+	.align	2
+	.globl	_commpage_32_routines
+_commpage_32_routines:
+	.long	CPN(compare_and_swap32_mp)
+	.long	CPN(compare_and_swap32_up)
+	.long	CPN(compare_and_swap64_mp)
+	.long	CPN(compare_and_swap64_up)
+	.long	CPN(atomic_add32_mp)
+	.long	CPN(atomic_add32_up)
+	.long	CPN(mach_absolute_time)
+	.long	CPN(spin_lock_try_mp)
+	.long	CPN(spin_lock_try_up)
+	.long	CPN(spin_lock_mp)
+	.long	CPN(spin_lock_up)
+	.long	CPN(spin_unlock)
+	.long	CPN(pthread_getspecific)
+	.long	CPN(gettimeofday)
+	.long	CPN(sys_flush_dcache)
+	.long	CPN(sys_icache_invalidate)
+	.long	CPN(pthread_self)
+//	.long	CPN(relinquish)
+	.long	CPN(bit_test_and_set_mp)
+	.long	CPN(bit_test_and_set_up)
+	.long	CPN(bit_test_and_clear_mp)
+	.long	CPN(bit_test_and_clear_up)
+	.long	CPN(bzero_scalar)
+	.long	CPN(bzero_sse3)
+	.long	CPN(bcopy_scalar)
+	.long	CPN(bcopy_sse3)
+	.long	CPN(bcopy_sse4)
+	.long	CPN(old_nanotime)
+	.long	CPN(memset_pattern_sse3)
+	.long	CPN(longcopy_sse4)
+	.long	CPN(nanotime)
+	.long	0
+
+
+/* pointers to the 64-bit commpage routine descriptors */
+/* WARNING: these must be sorted by commpage address! */
+	.const_data
+	.align	2
+	.globl	_commpage_64_routines
+_commpage_64_routines:
+	.long	CPN(compare_and_swap32_mp_64)
+	.long	CPN(compare_and_swap32_up_64)
+	.long	CPN(compare_and_swap64_mp_64)
+	.long	CPN(compare_and_swap64_up_64)
+	.long	CPN(atomic_add32_mp_64)
+	.long	CPN(atomic_add32_up_64)
+	.long	CPN(atomic_add64_mp_64)
+	.long	CPN(atomic_add64_up_64)
+	.long	CPN(mach_absolute_time)
+	.long	CPN(spin_lock_try_mp_64)
+	.long	CPN(spin_lock_try_up_64)
+	.long	CPN(spin_lock_mp_64)
+	.long	CPN(spin_lock_up_64)
+	.long	CPN(spin_unlock_64)
+	.long	CPN(pthread_getspecific_64)
+	.long	CPN(gettimeofday_64)
+	.long	CPN(sys_flush_dcache_64)
+	.long	CPN(sys_icache_invalidate)	/* same routine as 32-bit version, just a "ret" */
+	.long	CPN(pthread_self_64)
+	.long	CPN(bit_test_and_set_mp_64)
+	.long	CPN(bit_test_and_set_up_64)
+	.long	CPN(bit_test_and_clear_mp_64)
+	.long	CPN(bit_test_and_clear_up_64)
+	.long	CPN(bzero_sse3_64)
+	.long	CPN(bcopy_sse4_64)
+	.long	CPN(old_nanotime_64)
+	.long	CPN(memset_pattern_sse3_64)
+	.long	CPN(longcopy_sse4_64)
+	.long	CPN(nanotime_64)
+	.long	0
+
diff --git a/osfmk/i386/commpage/commpage_gettimeofday.s b/osfmk/i386/commpage/commpage_gettimeofday.s
index dd7ea884c..34c4f9bee 100644
--- a/osfmk/i386/commpage/commpage_gettimeofday.s
+++ b/osfmk/i386/commpage/commpage_gettimeofday.s
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -30,11 +30,106 @@
 #include <machine/cpu_capabilities.h>
 #include <machine/commpage.h>
 
+#define NSEC_PER_SEC	1000*1000*1000
+#define NSEC_PER_USEC	1000
+
         .text
         .align  2, 0x90
 
 Lgettimeofday:
-	int	$0x3
+	push	%ebp
+	mov	%esp,%ebp
+	push	%esi
+	push	%edi
+	push	%ebx
+
+0:
+	cmp	$0,_COMM_PAGE_TIMEENABLE
+	je	4f
+	mov	_COMM_PAGE_TIMEBASE,%esi
+	mov	_COMM_PAGE_TIMEBASE+4,%edi
+	mov	_COMM_PAGE_TIMESTAMP,%ebx
+
+	mov	$ _COMM_PAGE_NANOTIME,%eax
+	call	*%eax		/* get ns in %edx:%eax */
+
+	cmp	_COMM_PAGE_TIMEBASE,%esi
+	jne	0b
+	cmp	_COMM_PAGE_TIMEBASE+4,%edi
+	jne	0b
+	cmp	$0,_COMM_PAGE_TIMEENABLE
+	je	4f
+	
+	mov	$ NSEC_PER_SEC,%ecx
+	sub	%esi,%eax
+	sbb	%edi,%edx
+	div	%ecx
+	add	%eax,%ebx
+
+	mov	$ NSEC_PER_USEC,%ecx
+	mov	%edx,%eax
+	xor	%edx,%edx
+	div	%ecx
+
+	mov	8(%ebp),%ecx
+	mov	%ebx,(%ecx)
+	mov	%eax,4(%ecx)
+	xor	%eax,%eax
+
+3:
+	pop	%ebx
+	pop	%edi
+	pop	%esi
+	pop	%ebp
 	ret
+4:				/* fail */
+	movl	$1,%eax
+	jmp	3b
 
 	COMMPAGE_DESCRIPTOR(gettimeofday,_COMM_PAGE_GETTIMEOFDAY,0,0)
+
+
+	.code64
+        .text
+        .align  2, 0x90
+
+Lgettimeofday_64:			// %rdi = ptr to timeval
+	pushq	%rbp			// set up a frame for backtraces
+	movq	%rsp,%rbp
+	movq	%rdi,%r9		// save ptr to timeval
+	movq	$_COMM_PAGE_32_TO_64(_COMM_PAGE_TIMEBASE),%r10
+0:
+	cmpl	$0,_TIMEENABLE(%r10)	// is data valid? (test _COMM_PAGE_TIMEENABLE)
+	jz	4f			// no
+	movq	_TIMEBASE(%r10),%r11	// get _COMM_PAGE_TIMEBASE
+	movq	$_COMM_PAGE_32_TO_64(_COMM_PAGE_NANOTIME),%rax
+	call	*%rax			// get %rax <- nanotime(), preserving %r9, %r10 and %r11
+	movl	_TIMESTAMP(%r10),%r8d	// get _COMM_PAGE_TIMESTAMP
+	cmpq	_TIMEBASE(%r10),%r11	// has _COMM_PAGE_TIMEBASE changed?
+	jne	0b			// loop until we have consistent data
+	cmpl	$0,_TIMEENABLE(%r10)	// is data valid? (test _COMM_PAGE_TIMEENABLE)
+	jz	4f			// no
+	
+	movl	$ NSEC_PER_SEC,%ecx
+	subq	%r11,%rax		// generate nanoseconds since timestamp
+	movq	%rax,%rdx
+	shrq	$32,%rdx		// get high half of delta in %edx
+	divl	%ecx			// %eax <- seconds since timestamp, %edx <- nanoseconds
+	addl	%eax,%r8d		// add seconds elapsed to timestamp seconds
+
+	movl	$ NSEC_PER_USEC,%ecx
+	movl	%edx,%eax
+	xorl	%edx,%edx
+	divl	%ecx			// divide residual ns by 1000 to get residual us in %eax
+	
+	movq	%r8,(%r9)		// store 64-bit seconds into timeval
+	movl	%eax,8(%r9)		// store 32-bit useconds into timeval
+	xorl	%eax,%eax		// return 0 for success
+3:
+	popq	%rbp
+	ret
+4:					// fail
+	movl	$1,%eax
+	jmp	3b
+
+	COMMPAGE_DESCRIPTOR(gettimeofday_64,_COMM_PAGE_GETTIMEOFDAY,0,0)
diff --git a/osfmk/i386/commpage/commpage_mach_absolute_time.s b/osfmk/i386/commpage/commpage_mach_absolute_time.s
index a3c1fee8c..1cacdb5b2 100644
--- a/osfmk/i386/commpage/commpage_mach_absolute_time.s
+++ b/osfmk/i386/commpage/commpage_mach_absolute_time.s
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -37,89 +37,87 @@
         .align  2, 0x90
 
 Lmach_absolute_time:
-
 	int	$0x3
 	ret
 
-	COMMPAGE_DESCRIPTOR(mach_absolute_time,_COMM_PAGE_ABSOLUTE_TIME,1,0)
-
+	COMMPAGE_DESCRIPTOR(mach_absolute_time,_COMM_PAGE_ABSOLUTE_TIME,0,0)
 
+/* Nanotime is being moved out of the way of bcopy in the commpage.
+ * First we put it in both places, old and new.  Then, when all the build
+ * trains have rebuilt libSystem, we can remove the deprecated instance.
+ */
+ 
+/* return nanotime in %edx:%eax */
+Lold_nanotime:
 Lnanotime:
+	push	%ebp
+	mov	%esp,%ebp
+	push	%esi
+	push	%edi
+	push	%ebx
+
+0:
+	mov	_COMM_PAGE_NT_TSC_BASE,%esi
+	mov	_COMM_PAGE_NT_TSC_BASE+4,%edi
+
+	rdtsc
+	sub	%esi,%eax
+	sbb	%edi,%edx
+
+	mov	_COMM_PAGE_NT_SCALE,%ecx
+
+	mov	%edx,%ebx
+	mull	%ecx
+	mov	%ebx,%eax
+	mov	%edx,%ebx
+	mull	%ecx
+	add	%ebx,%eax
+	adc	$0,%edx
+
+	add	_COMM_PAGE_NT_NS_BASE,%eax
+	adc	_COMM_PAGE_NT_NS_BASE+4,%edx
+
+	cmp	_COMM_PAGE_NT_TSC_BASE,%esi
+	jne	0b
+	cmp	_COMM_PAGE_NT_TSC_BASE+4,%edi
+	jne	0b
+
+	pop	%ebx
+	pop	%edi
+	pop	%esi
+	pop	%ebp
+	ret
 
-	pushl	%ebx
-	pushl	%esi
-	pushl	%edi
-	pushl	%ebp
-	movl	$(_COMM_PAGE_NANOTIME_INFO), %esi
-
-	/*
-	 * The nanotime info consists of:
-	 *	- base_tsc	64-bit timestamp register value
-	 *	- base_ns	64-bit corresponding nanosecond uptime value
-	 *	- scale		32-bit current scale multiplier
-	 *	- shift		32-bit current shift divider
-	 *	- check_tsc	64-bit timestamp check value
-	 *
-	 * This enables an timestamp register's value, tsc, to be converted
-	 * into a nanosecond nanotime value, ns:
-	 *
-	 * 	ns = base_ns + ((tsc - base_tsc) * scale >> shift)
-	 *
-	 * The kernel updates this every tick or whenever a performance
-	 * speed-step changes the scaling. To avoid locking, a duplicated
-	 * sequence counting scheme is used. The base_tsc value is updated
-	 * whenever the info starts to be changed, and check_tsc is updated
-	 * to the same value at the end of the update. The regularity of
-	 * update ensures that (tsc - base_tsc) is a 32-bit quantity.
-	 * When a conversion is performed, we read base_tsc before we start
-	 * and check_tsc at the end -- if there's a mis-match we repeat.
-	 * It's sufficient to compare only the low-order 32-bits. 
-	 */
+	COMMPAGE_DESCRIPTOR(nanotime,_COMM_PAGE_NANOTIME,0,0)
+	COMMPAGE_DESCRIPTOR(old_nanotime,_COMM_PAGE_OLD_NANOTIME,0,0)
 
+
+/* The 64-bit version.  We return the 64-bit nanotime in %rax,
+ * and by convention we must preserve %r9, %r10, and %r11.
+ */
+	.text
+	.align	2
+	.code64
+Lold_nanotime_64:
+Lnanotime_64:				// NB: must preserve r9, r10, and r11
+	pushq	%rbp			// set up a frame for backtraces
+	movq	%rsp,%rbp
+	movq	$_COMM_PAGE_32_TO_64(_COMM_PAGE_NT_TSC_BASE),%rsi
 1:
-	//
-	//  Read nanotime info and stash in registers.
-	//
-	movl	NANOTIME_BASE_TSC(%esi), %ebx	// ebx := lo(base_tsc)
-	movl	NANOTIME_BASE_NS(%esi), %ebp
-	movl	NANOTIME_BASE_NS+4(%esi), %edi	// edi:ebp := base_ns
-	movl	NANOTIME_SHIFT(%esi), %ecx	// ecx := shift
-	//
-	// Read timestamp register (tsc) and calculate delta.
-	//
-	rdtsc					// edx:eax := tsc
-	subl	%ebx, %eax			// eax := (tsc - base_tsc)
-	movl	NANOTIME_SCALE(%esi), %edx	// edx := shift
-	//
-	// Check for consistency and re-read if necessary.
-	//
-	cmpl	NANOTIME_CHECK_TSC(%esi), %ebx
+	movq	_NT_TSC_BASE(%rsi),%r8	// r8  := base_tsc
+	rdtsc				// edx:eax := tsc
+	shlq	$32,%rdx		// rax := ((edx << 32) | eax), ie 64-bit tsc
+	orq	%rdx,%rax
+	subq	%r8, %rax		// rax := (tsc - base_tsc)
+	movl	_NT_SCALE(%rsi),%ecx
+	mulq	%rcx			// rdx:rax := (tsc - base_tsc) * scale
+	shrdq	$32,%rdx,%rax		// _COMM_PAGE_NT_SHIFT is always 32
+	addq	_NT_NS_BASE(%rsi),%rax	// (((tsc - base_tsc) * scale) >> 32) + ns_base
+	
+	cmpq	_NT_TSC_BASE(%rsi),%r8	// did the data change during computation?
 	jne	1b
-
-	//
-	// edx:eax := ((tsc - base_tsc) * scale)
-	//
-	mull	%edx
-
-	//
-	// eax := ((tsc - base_tsc) * scale >> shift)
-	//
-	shrdl	%cl, %edx, %eax
-	andb	$32, %cl
-	cmovnel	%edx, %eax		// %eax := %edx if shift == 32
-	xorl	%edx, %edx
-
-	//
-	// Add base_ns: 
-	// edx:eax = (base_ns + ((tsc - base_tsc) * scale >> shift))
-	//
-	addl	%ebp, %eax
-	adcl	%edi, %edx
-
-	popl	%ebp
-	popl	%edi
-	popl	%esi
-	popl	%ebx
+	popq	%rbp
 	ret
 
-	COMMPAGE_DESCRIPTOR(nanotime,_COMM_PAGE_NANOTIME,1,0)
+	COMMPAGE_DESCRIPTOR(nanotime_64,_COMM_PAGE_NANOTIME,0,0)
+	COMMPAGE_DESCRIPTOR(old_nanotime_64,_COMM_PAGE_OLD_NANOTIME,0,0)
diff --git a/osfmk/i386/commpage/commpage_sigs.c b/osfmk/i386/commpage/commpage_sigs.c
new file mode 100644
index 000000000..8b7e7d404
--- /dev/null
+++ b/osfmk/i386/commpage/commpage_sigs.c
@@ -0,0 +1,189 @@
+#include "commpage.h"
+
+static unsigned int sigdata[] =
+{
+	0x06004018, 0x01000000, 0x6d6f635f, 0x65726170, 0x646e615f, 0x6177735f, 
+	0x00323370, 0x06004018, 0x2828e07c, 0x4018077c, 0x14008240, 0x2d29807c, 
+	0xf0ffa240, 0x01006038, 0x2000804e, 0x00006038, 0x2000804e, 0x06004018, 
+	0x00000000, 0x6d6f635f, 0x65726170, 0x646e615f, 0x6177735f, 0x00323370, 
+	0x06004018, 0x06004018, 0x01000000, 0x6d6f635f, 0x65726170, 0x646e615f, 
+	0x6177735f, 0x62323370, 0x00000000, 0x06004018, 0xac06007c, 0x2828e07c, 
+	0x4018077c, 0x18008240, 0x2d29807c, 0xf0ffa240, 0x2c01004c, 0x01006038, 
+	0x2000804e, 0x00006038, 0x2000804e, 0x2000804e, 0x06004018, 0x00000000, 
+	0x6d6f635f, 0x65726170, 0x646e615f, 0x6177735f, 0x62323370, 0x00000000, 
+	0x06004018, 0x06004018, 0x01000000, 0x6d6f635f, 0x65726170, 0x646e615f, 
+	0x6177735f, 0x62343670, 0x00000000, 0x06004018, 0xac04207c, 0xa828e07c, 
+	0x4018277c, 0x1800c240, 0xad29807c, 0xf0ffc240, 0x2c01004c, 0x01006038, 
+	0x2000804e, 0xf8ff0039, 0x00006038, 0xad09887c, 0x2000804e, 0x2000804e, 
+	0x06004018, 0x00000000, 0x6d6f635f, 0x65726170, 0x646e615f, 0x6177735f, 
+	0x62343670, 0x00000000, 0x06004018, 0x06004018, 0x01000000, 0x6d656d5f, 
+	0x5f746573, 0x74746170, 0x006e7265, 0x06004018, 0x10000071, 0x000009c8, 
+	0x080029c8, 0x14008241, 0x000008d8, 0x080028d8, 0xf0ff8438, 0x10000839, 
+	0xbfd18054, 0x0000807f, 0xbe068454, 0x70008241, 0xec47007c, 0xa603097c, 
+	0x2000c038, 0x3400be41, 0x000008d8, 0x080028d8, 0x100008d8, 0x180028d8, 
+	0x200008d8, 0x280028d8, 0x300008d8, 0x380028d8, 0x40000839, 0xdcff0042, 
+	0x34000048, 0xec47007c, 0xec47067c, 0x000008d8, 0x080028d8, 0x100008d8, 
+	0x180028d8, 0x200008d8, 0x280028d8, 0x300008d8, 0x380028d8, 0x40000839, 
+	0xd4ff0042, 0x3fe18054, 0x3e078454, 0x2000824d, 0xa603097c, 0x000008d8, 
+	0x080028d8, 0x10000839, 0xf4ff0042, 0x2000804e, 0x2000804e, 0x06004018, 
+	0x00000000, 0x6d656d5f, 0x5f746573, 0x74746170, 0x006e7265, 0x06004018, 
+	0x06004018, 0x01000000, 0x6f74615f, 0x5f63696d, 0x75716e65, 0x00657565, 
+	0x06004018, 0x2818c07c, 0x2e29c47c, 0x00000060, 0x2d19807c, 0x2000e24d, 
+	0xecffff4b, 0x2000804e, 0x06004018, 0x00000000, 0x6f74615f, 0x5f63696d, 
+	0x75716e65, 0x00657565, 0x06004018, 0x06004018, 0x01000000, 0x6f74615f, 
+	0x5f63696d, 0x75716564, 0x00657565, 0x06004018, 0x781b657c, 0x2828607c, 
+	0x0000032c, 0x2000824d, 0x2e20c37c, 0x2d29c07c, 0xecffc240, 0x00000060, 
+	0x2000804e, 0x06004018, 0x00000000, 0x6f74615f, 0x5f63696d, 0x75716564, 
+	0x00657565, 0x06004018, 0x06004018, 0x01000000, 0x6f74615f, 0x5f63696d, 
+	0x72726162, 0x00726569, 0x06004018, 0x2000804e, 0x06004018, 0x00000000, 
+	0x6f74615f, 0x5f63696d, 0x72726162, 0x00726569, 0x06004018, 0x06004018, 
+	0x01000000, 0x6f74615f, 0x5f63696d, 0x5f646461, 0x00003233, 0x06004018, 
+	0x2820a07c, 0x142ac37c, 0x2d21c07c, 0xf4ffc240, 0x7833c37c, 0x2000804e, 
+	0x06004018, 0x00000000, 0x6f74615f, 0x5f63696d, 0x5f646461, 0x00003233, 
+	0x06004018, 0x06004018, 0x01000000, 0x63616d5f, 0x62615f68, 0x756c6f73, 
+	0x745f6574, 0x00656d69, 0x06004018, 0x00004018, 0x04006000, 0x00000000, 
+	0x00000000, 0x63616d5f, 0x62615f68, 0x756c6f73, 0x745f6574, 0x5f656d69, 
+	0x68676968, 0x00000000, 0x00004018, 0x00004018, 0x04008000, 0x00000000, 
+	0x00000000, 0x63616d5f, 0x62615f68, 0x756c6f73, 0x745f6574, 0x5f656d69, 
+	0x00776f6c, 0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x63616d5f, 
+	0x62615f68, 0x756c6f73, 0x745f6574, 0x00656d69, 0x06004018, 0x06004018, 
+	0x01000000, 0x6970735f, 0x6f6c5f6e, 0x745f6b63, 0x00007972, 0x06004018, 
+	0x05004018, 0x04006000, 0x05004018, 0x00004018, 0x04006000, 0x00000000, 
+	0x00000000, 0x6970735f, 0x6f6c5f6e, 0x745f6b63, 0x775f7972, 0x70706172, 
+	0x00007265, 0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x6970735f, 
+	0x6f6c5f6e, 0x745f6b63, 0x00007972, 0x06004018, 0x06004018, 0x01000000, 
+	0x6970735f, 0x6f6c5f6e, 0x00006b63, 0x06004018, 0x05004018, 0x04006000, 
+	0x05004018, 0x00004018, 0x00000000, 0x00000000, 0x00000000, 0x6970735f, 
+	0x6f6c5f6e, 0x775f6b63, 0x70706172, 0x00007265, 0x00004018, 0x2000804e, 
+	0x06004018, 0x00000000, 0x6970735f, 0x6f6c5f6e, 0x00006b63, 0x06004018, 
+	0x06004018, 0x01000000, 0x6970735f, 0x6e755f6e, 0x6b636f6c, 0x00000000, 
+	0x06004018, 0x05004018, 0x04006000, 0x05004018, 0x00004018, 0x00000000, 
+	0x00000000, 0x00000000, 0x6970735f, 0x6e755f6e, 0x6b636f6c, 0x00000000, 
+	0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x6970735f, 0x6e755f6e, 
+	0x6b636f6c, 0x00000000, 0x06004018, 0x06004018, 0x01000000, 0x6874705f, 
+	0x64616572, 0x7465675f, 0x63657073, 0x63696669, 0x00000000, 0x06004018, 
+	0x02004018, 0xc082ffff, 0x02004018, 0x2000804e, 0x06004018, 0x00000000, 
+	0x6874705f, 0x64616572, 0x7465675f, 0x63657073, 0x63696669, 0x00000000, 
+	0x06004018, 0x06004018, 0x01000000, 0x7465675f, 0x656d6974, 0x6164666f, 
+	0x00000079, 0x06004018, 0x05004018, 0x04006000, 0x05004018, 0x00004018, 
+	0x04006000, 0x00000000, 0x00000000, 0x7465675f, 0x656d6974, 0x6164666f, 
+	0x72775f79, 0x65707061, 0x00000072, 0x00004018, 0x2000804e, 0x06004018, 
+	0x00000000, 0x7465675f, 0x656d6974, 0x6164666f, 0x00000079, 0x06004018, 
+	0x06004018, 0x01000000, 0x7379735f, 0x6163645f, 0x5f656863, 0x73756c66, 
+	0x00000068, 0x06004018, 0x05004018, 0x04006000, 0x05004018, 0x05004018, 
+	0x04008000, 0x05004018, 0x00004018, 0x00000000, 0x00000000, 0x00000000, 
+	0x7379735f, 0x6163645f, 0x5f656863, 0x73756c66, 0x00000068, 0x00004018, 
+	0x2000804e, 0x06004018, 0x00000000, 0x7379735f, 0x6163645f, 0x5f656863, 
+	0x73756c66, 0x00000068, 0x06004018, 0x06004018, 0x01000000, 0x7379735f, 
+	0x6163695f, 0x5f656863, 0x61766e69, 0x6164696c, 0x00006574, 0x06004018, 
+	0x05004018, 0x04006000, 0x05004018, 0x05004018, 0x04008000, 0x05004018, 
+	0x00004018, 0x00000000, 0x00000000, 0x00000000, 0x7379735f, 0x6163695f, 
+	0x5f656863, 0x61766e69, 0x6164696c, 0x775f6574, 0x70706172, 0x00007265, 
+	0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x7379735f, 0x6163695f, 
+	0x5f656863, 0x61766e69, 0x6164696c, 0x00006574, 0x06004018, 0x06004018, 
+	0x01000000, 0x6874705f, 0x64616572, 0x6c65735f, 0x00000066, 0x06004018, 
+	0x02004018, 0x8085ffff, 0x02004018, 0x2000804e, 0x06004018, 0x00000000, 
+	0x6874705f, 0x64616572, 0x6c65735f, 0x00000066, 0x06004018, 0x06004018, 
+	0x01000000, 0x657a625f, 0x00006f72, 0x06004018, 0x05004018, 0x04006000, 
+	0x05004018, 0x05004018, 0x04008000, 0x05004018, 0x00004018, 0x00000000, 
+	0x00000000, 0x00000000, 0x657a625f, 0x00006f72, 0x00004018, 0x2000804e, 
+	0x06004018, 0x00000000, 0x657a625f, 0x00006f72, 0x06004018, 0x06004018, 
+	0x01000000, 0x6f63625f, 0x00007970, 0x06004018, 0x05004018, 0x04006000, 
+	0x05004018, 0x05004018, 0x04008000, 0x05004018, 0x05004018, 0x0400a000, 
+	0x05004018, 0x00004018, 0x00000000, 0x00000000, 0x00000000, 0x6f63625f, 
+	0x00007970, 0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x6f63625f, 
+	0x00007970, 0x06004018, 0x06004018, 0x01000000, 0x6d656d5f, 0x65766f6d, 
+	0x00000000, 0x06004018, 0x05004018, 0x04006000, 0x05004018, 0x05004018, 
+	0x04008000, 0x05004018, 0x05004018, 0x0400a000, 0x05004018, 0x00004018, 
+	0x00000000, 0x00000000, 0x00000000, 0x6d656d5f, 0x65766f6d, 0x00000000, 
+	0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x6d656d5f, 0x65766f6d, 
+	0x00000000, 0x06004018, 0x06004018, 0x01000000, 0x6e616e5f, 0x6d69746f, 
+	0x00000065, 0x06004018, 0x00004018, 0x04006000, 0x00000000, 0x00000000, 
+	0x6e616e5f, 0x6d69746f, 0x69685f65, 0x00006867, 0x00004018, 0x00004018, 
+	0x04008000, 0x00000000, 0x00000000, 0x6e616e5f, 0x6d69746f, 0x6f6c5f65, 
+	0x00000077, 0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x6e616e5f, 
+	0x6d69746f, 0x00000065, 0x06004018, 0x06004018, 0x01000000, 0x6a626f5f, 
+	0x736d5f63, 0x6e655367, 0x00000064, 0x06004018, 0x00004018, 0x00000000, 
+	0x00000400, 0x00000000, 0x6a626f5f, 0x736d5f63, 0x6e655367, 0x00000064, 
+	0x00004018, 0x06004018, 0x00000000, 0x6a626f5f, 0x736d5f63, 0x6e655367, 
+	0x00000064, 0x06004018, 0x06004018, 0x01000000, 0x6a626f5f, 0x73615f63, 
+	0x6e676973, 0x6176695f, 0x00000072, 0x06004018, 0x00004018, 0x00000000, 
+	0x00000400, 0x00000000, 0x6a626f5f, 0x73615f63, 0x6e676973, 0x6176695f, 
+	0x65675f72, 0x6972656e, 0x00000063, 0x00004018, 0x06004018, 0x00000000, 
+	0x6a626f5f, 0x73615f63, 0x6e676973, 0x6176695f, 0x00000072, 0x06004018, 
+	0x06004018, 0x01000000, 0x6a626f5f, 0x73615f63, 0x6e676973, 0x6f6c675f, 
+	0x006c6162, 0x06004018, 0x00004018, 0x00000000, 0x00000400, 0x00000000, 
+	0x6a626f5f, 0x73615f63, 0x6e676973, 0x6f6c675f, 0x5f6c6162, 0x656e6567, 
+	0x00636972, 0x00004018, 0x06004018, 0x00000000, 0x6a626f5f, 0x73615f63, 
+	0x6e676973, 0x6f6c675f, 0x006c6162, 0x06004018, 0x06004018, 0x01000000, 
+	0x6a626f5f, 0x73615f63, 0x6e676973, 0x7274735f, 0x43676e6f, 0x00747361, 
+	0x06004018, 0x00004018, 0x00000000, 0x00000400, 0x00000000, 0x6a626f5f, 
+	0x73615f63, 0x6e676973, 0x7274735f, 0x43676e6f, 0x5f747361, 0x656e6567, 
+	0x00636972, 0x00004018, 0x06004018, 0x00000000, 0x6a626f5f, 0x73615f63, 
+	0x6e676973, 0x7274735f, 0x43676e6f, 0x00747361, 0x06004018, 
+};
+commpage_descriptor sigdata_descriptor =
+{ sigdata, sizeof(sigdata), 0xffff3000, 0, 0 };
+
+static unsigned int badata[] =
+{
+	0xaa3aff4b, 0x3a3aff4b, 0xca39ff4b, 0x6e39ff4b, 0x0230ff4b, 0x5a32ff4b, 
+	0xae32ff4b, 0x0a33ff4b, 0x4633ff4b, 0x9633ff4b, 0x3634ff4b, 0xaa34ff4b, 
+	0x1235ff4b, 0x7e35ff4b, 0xd635ff4b, 0x4a36ff4b, 0xce36ff4b, 0x6637ff4b, 
+	0xae37ff4b, 0x0e38ff4b, 0x7a38ff4b, 0x6630ff4b, 0xde30ff4b, 0x5e31ff4b, 
+};
+static commpage_descriptor badata_descriptor_ary[] =
+{
+	{ &badata[ 0], 4, 0xfffefea0, 0, 0 },
+	{ &badata[ 1], 4, 0xfffefeb0, 0, 0 },
+	{ &badata[ 2], 4, 0xfffefec0, 0, 0 },
+	{ &badata[ 3], 4, 0xfffeff00, 0, 0 },
+	{ &badata[ 4], 4, 0xffff8080, 0, 0 },
+	{ &badata[ 5], 4, 0xffff8100, 0, 0 },
+	{ &badata[ 6], 4, 0xffff8140, 0, 0 },
+	{ &badata[ 7], 4, 0xffff8180, 0, 0 },
+	{ &badata[ 8], 4, 0xffff81a0, 0, 0 },
+	{ &badata[ 9], 4, 0xffff8200, 0, 0 },
+	{ &badata[10], 4, 0xffff8220, 0, 0 },
+	{ &badata[11], 4, 0xffff8260, 0, 0 },
+	{ &badata[12], 4, 0xffff82a0, 0, 0 },
+	{ &badata[13], 4, 0xffff82c0, 0, 0 },
+	{ &badata[14], 4, 0xffff82e0, 0, 0 },
+	{ &badata[15], 4, 0xffff84e0, 0, 0 },
+	{ &badata[16], 4, 0xffff8520, 0, 0 },
+	{ &badata[17], 4, 0xffff8580, 0, 0 },
+	{ &badata[18], 4, 0xffff8600, 0, 0 },
+	{ &badata[19], 4, 0xffff8780, 0, 0 },
+	{ &badata[20], 4, 0xffff87a0, 0, 0 },
+	{ &badata[21], 4, 0xffff8f80, 0, 0 },
+	{ &badata[22], 4, 0xffff8fc0, 0, 0 },
+	{ &badata[23], 4, 0xffff9000, 0, 0 },
+};
+commpage_descriptor *ba_descriptors[] =
+{
+	&badata_descriptor_ary[ 0],
+	&badata_descriptor_ary[ 1],
+	&badata_descriptor_ary[ 2],
+	&badata_descriptor_ary[ 3],
+	&badata_descriptor_ary[ 4],
+	&badata_descriptor_ary[ 5],
+	&badata_descriptor_ary[ 6],
+	&badata_descriptor_ary[ 7],
+	&badata_descriptor_ary[ 8],
+	&badata_descriptor_ary[ 9],
+	&badata_descriptor_ary[10],
+	&badata_descriptor_ary[11],
+	&badata_descriptor_ary[12],
+	&badata_descriptor_ary[13],
+	&badata_descriptor_ary[14],
+	&badata_descriptor_ary[15],
+	&badata_descriptor_ary[16],
+	&badata_descriptor_ary[17],
+	&badata_descriptor_ary[18],
+	&badata_descriptor_ary[19],
+	&badata_descriptor_ary[20],
+	&badata_descriptor_ary[21],
+	&badata_descriptor_ary[22],
+	&badata_descriptor_ary[23],
+	0
+};
diff --git a/osfmk/i386/commpage/commpage_sigs.h b/osfmk/i386/commpage/commpage_sigs.h
deleted file mode 100644
index e69de29bb..000000000
diff --git a/osfmk/i386/commpage/commpage_sigs.s b/osfmk/i386/commpage/commpage_sigs.s
deleted file mode 100644
index e69de29bb..000000000
diff --git a/osfmk/i386/commpage/longcopy_sse4.s b/osfmk/i386/commpage/longcopy_sse4.s
new file mode 100644
index 000000000..0abff2916
--- /dev/null
+++ b/osfmk/i386/commpage/longcopy_sse4.s
@@ -0,0 +1,221 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+         
+#include <machine/cpu_capabilities.h>
+#include <machine/commpage.h>
+       
+
+/*
+ * The bcopy/memcpy loops for very long operands, tuned for Pentium-M
+ * class processors with SSE4 and 64-byte cache lines.
+ *
+ * The following #defines are tightly coupled to the u-architecture:
+ */
+
+#define kBigChunk   (256*1024)          // outer loop chunk size for kVeryLong sized operands
+
+
+// Very long forward moves.  These are at least several pages, so we loop over big
+// chunks of memory (kBigChunk in size.)  We first prefetch the chunk, and then copy
+// it using non-temporal stores.  Hopefully all the reads occur in the prefetch loop,
+// so the copy loop reads from L2 and writes directly to memory (with write combining.)
+// This minimizes bus turnaround and maintains good DRAM page locality.
+// Note that for this scheme to work, kVeryLong must be a large fraction of L2 cache
+// size.  Otherwise, it is counter-productive to bypass L2 on the stores.
+//
+// We are called from the commpage bcopy loops when they encounter very long
+// operands, with the standard ABI.
+//
+//	void longcopy(const void *dest, void *sou, size_t len)
+ 
+        .text
+        .align	5, 0x90
+Llongcopy_sse4:				// void longcopy(const void *dest, void *sou, size_t len)
+	pushl	%ebp			// set up a frame for backtraces
+	movl	%esp,%ebp
+        pushl   %esi
+        pushl   %edi
+	pushl	%ebx			// we'll need to use this too
+        movl    8(%ebp),%edi		// get dest ptr
+        movl    12(%ebp),%esi           // get source ptr
+        movl    16(%ebp),%ecx           // get length
+	movl	%edi,%ebx		// copy dest ptr
+	negl	%ebx
+	andl	$63,%ebx		// get #bytes to cache line align destination
+	jz	LBigChunkLoop		// already aligned
+	
+// Cache line align destination, so temporal stores in copy loops work right.
+
+	pushl	%ebx			// arg3 - #bytes to align destination (1..63)
+	pushl	%esi			// arg2 - source
+	pushl	%edi			// arg1 - dest
+	movl	$(_COMM_PAGE_MEMCPY),%eax
+	call	*%eax			// align the destination
+	addl	$12,%esp
+        movl    8(%ebp),%edi		// recover dest ptr
+        movl    12(%ebp),%esi           // recover source ptr
+        movl    16(%ebp),%ecx           // recover length
+	addl	%ebx,%esi		// adjust ptrs and lengths past copy
+	addl	%ebx,%edi
+	subl	%ebx,%ecx
+	
+// Loop over big chunks.
+//      ecx = length remaining (>= 4096)
+//      edi = dest (64-byte aligned)
+//      esi = source (may be unaligned)
+	
+LBigChunkLoop:
+        movl    $(kBigChunk),%edx       // assume we can do a full chunk
+	cmpl	%edx,%ecx		// do we have a full chunk left to do?
+	cmovbl	%ecx,%edx		// if not, only move what we have left
+	andl	$-4096,%edx		// we work in page multiples
+	xor	%eax,%eax		// initialize chunk offset
+	jmp	LTouchLoop
+
+// Touch in the next chunk.  We try to keep the prefetch unit in "kick-start" mode,
+// by touching two adjacent cache lines every 8 lines of each page, in four slices.
+// Because the source may be unaligned, we use byte loads to touch.
+//      ecx = length remaining (including this chunk)
+//      edi = ptr to start of dest chunk
+//      esi = ptr to start of source chunk
+//	edx = chunk length (multiples of pages)
+//	ebx = scratch reg used to read a byte of each cache line
+//	eax = chunk offset
+
+	.align	4,0x90			// 16-byte align inner loops
+LTouchLoop:
+	movzb	(%esi,%eax),%ebx	// touch line 0, 2, 4, or 6 of page
+	movzb	1*64(%esi,%eax),%ebx	// touch line 1, 3, 5, or 7
+	movzb	8*64(%esi,%eax),%ebx	// touch line 8, 10, 12, or 14
+	movzb	9*64(%esi,%eax),%ebx	// etc
+	
+	movzb	16*64(%esi,%eax),%ebx
+	movzb	17*64(%esi,%eax),%ebx
+	movzb	24*64(%esi,%eax),%ebx
+	movzb	25*64(%esi,%eax),%ebx
+
+	movzb	32*64(%esi,%eax),%ebx
+	movzb	33*64(%esi,%eax),%ebx
+	movzb	40*64(%esi,%eax),%ebx
+	movzb	41*64(%esi,%eax),%ebx
+
+	movzb	48*64(%esi,%eax),%ebx
+	movzb	49*64(%esi,%eax),%ebx
+	movzb	56*64(%esi,%eax),%ebx
+	movzb	57*64(%esi,%eax),%ebx
+	
+	subl	$-128,%eax		// next slice of page (adding 128 w 8-bit immediate)
+	testl	$512,%eax		// done with this page?
+	jz	LTouchLoop		// no, next of four slices
+	addl	$(4096-512),%eax	// move on to next page
+	cmpl	%eax,%edx		// done with this chunk?
+	jnz	LTouchLoop		// no, do next page
+	
+// The chunk has been pre-fetched, now copy it using non-temporal stores.
+// There are two copy loops, depending on whether the source is 16-byte aligned
+// or not.
+
+	addl	%edx,%esi		// increment ptrs by chunk length
+	addl	%edx,%edi
+	subl	%edx,%ecx		// adjust remaining length
+	negl	%edx			// prepare loop index (counts up to 0)
+	testl	$15,%esi		// is source 16-byte aligned?
+	jnz	LVeryLongUnaligned	// source is not aligned
+	jmp	LVeryLongAligned
+
+	.align	4,0x90			// 16-byte align inner loops
+LVeryLongAligned:			// aligned loop over 128-bytes
+        movdqa  (%esi,%edx),%xmm0
+        movdqa  16(%esi,%edx),%xmm1
+        movdqa  32(%esi,%edx),%xmm2
+        movdqa  48(%esi,%edx),%xmm3
+        movdqa  64(%esi,%edx),%xmm4
+        movdqa  80(%esi,%edx),%xmm5
+        movdqa  96(%esi,%edx),%xmm6
+        movdqa  112(%esi,%edx),%xmm7
+
+        movntdq %xmm0,(%edi,%edx)
+        movntdq %xmm1,16(%edi,%edx)
+        movntdq %xmm2,32(%edi,%edx)
+        movntdq %xmm3,48(%edi,%edx)
+        movntdq %xmm4,64(%edi,%edx)
+        movntdq %xmm5,80(%edi,%edx)
+        movntdq %xmm6,96(%edi,%edx)
+        movntdq %xmm7,112(%edi,%edx)
+        
+        subl    $-128,%edx		// add 128 with an 8-bit immediate
+        jnz	LVeryLongAligned
+	jmp	LVeryLongChunkEnd
+
+	.align	4,0x90			// 16-byte align inner loops
+LVeryLongUnaligned:			// unaligned loop over 128-bytes
+        movdqu  (%esi,%edx),%xmm0
+        movdqu  16(%esi,%edx),%xmm1
+        movdqu  32(%esi,%edx),%xmm2
+        movdqu  48(%esi,%edx),%xmm3
+        movdqu  64(%esi,%edx),%xmm4
+        movdqu  80(%esi,%edx),%xmm5
+        movdqu  96(%esi,%edx),%xmm6
+        movdqu  112(%esi,%edx),%xmm7
+
+        movntdq %xmm0,(%edi,%edx)
+        movntdq %xmm1,16(%edi,%edx)
+        movntdq %xmm2,32(%edi,%edx)
+        movntdq %xmm3,48(%edi,%edx)
+        movntdq %xmm4,64(%edi,%edx)
+        movntdq %xmm5,80(%edi,%edx)
+        movntdq %xmm6,96(%edi,%edx)
+        movntdq %xmm7,112(%edi,%edx)
+        
+        subl    $-128,%edx		// add 128 with an 8-bit immediate
+        jnz     LVeryLongUnaligned
+
+LVeryLongChunkEnd:
+	cmpl	$4096,%ecx		// at least another page to go?
+	jae	LBigChunkLoop		// yes
+	
+// Done.  Call memcpy() again to handle the 0-4095 bytes at the end.
+	
+	sfence				// required by non-temporal stores
+	testl	%ecx,%ecx		// anything left to copy?
+	jz	1f
+	pushl	%ecx			// arg3 - #bytes to align destination (1..63)
+	pushl	%esi			// arg2 - source
+	pushl	%edi			// arg1 - dest
+	movl	$(_COMM_PAGE_MEMCPY),%eax
+	call	*%eax			// align the destination
+	addl	$12,%esp		// pop off arguments
+1:
+	popl	%ebx
+	popl	%edi
+	popl	%esi
+	popl	%ebp
+	ret
+
+	/* always match for now, as commpage_stuff_routine() will panic if no match */
+	COMMPAGE_DESCRIPTOR(longcopy_sse4, _COMM_PAGE_LONGCOPY, 0 ,0)
diff --git a/osfmk/i386/commpage/longcopy_sse4_64.s b/osfmk/i386/commpage/longcopy_sse4_64.s
new file mode 100644
index 000000000..bdd550304
--- /dev/null
+++ b/osfmk/i386/commpage/longcopy_sse4_64.s
@@ -0,0 +1,211 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+        
+#include <machine/cpu_capabilities.h>
+#include <machine/commpage.h>
+      
+
+/*
+ * The bcopy/memcpy loops for very long operands, tuned for 64-bit
+ * Pentium-M class processors with SSE4 and 64-byte cache lines.
+ * This is the 64-bit version.
+ *
+ * The following #defines are tightly coupled to the u-architecture:
+ */
+
+#define kBigChunk   (256*1024)          // outer loop chunk size for kVeryLong sized operands
+
+
+// Very long forward moves.  These are at least several pages, so we loop over big
+// chunks of memory (kBigChunk in size.)  We first prefetch the chunk, and then copy
+// it using non-temporal stores.  Hopefully all the reads occur in the prefetch loop,
+// so the copy loop reads from L2 and writes directly to memory (with write combining.)
+// This minimizes bus turnaround and maintains good DRAM page locality.
+// Note that for this scheme to work, kVeryLong must be a large fraction of L2 cache
+// size.  Otherwise, it is counter-productive to bypass L2 on the stores.
+//
+// We are called from the commpage bcopy loops when they encounter very long
+// operands, with the standard ABI:
+//      rdi = dest ptr
+//      rsi = source ptr
+//      rdx = length (>= 8kb, probably much bigger)
+ 
+        .text
+	.code64
+        .align 5, 0x90
+Llongcopy_sse4_64:			// void longcopy(const void *dest, void *sou, size_t len)
+	pushq	%rbp			// set up a frame for backtraces
+	movq	%rsp,%rbp
+	movl	%edi,%eax		// copy dest ptr
+	negl	%eax
+	andl	$63,%eax		// get #bytes to cache line align destination
+	jz	LBigChunkLoop		// already aligned
+	
+// Cache line align destination, so temporal stores in copy loops work right.
+// The recursive call returns with the source and dest ptrs properly updated.
+
+	subq	%rax,%rdx		// get length remaining after dest is aligned
+	pushq	%rdx			// save length remaining
+	movl	%eax,%edx		// #bytes to copy to align destination
+	movq	$_COMM_PAGE_32_TO_64(_COMM_PAGE_MEMCPY),%rax
+	call	*%rax
+	popq	%rdx			// recover adjusted length
+	
+// Loop over big chunks.
+//      rdx = length remaining (>= 4096)
+//      rdi = dest (64-byte aligned)
+//      rsi = source (may be unaligned)
+	
+LBigChunkLoop:
+        movl    $(kBigChunk),%r8d       // assume we can do a full chunk
+	cmpq	%r8,%rdx		// do we have a full chunk left to do?
+	cmovbl	%edx,%r8d		// if not, only move what we have left
+	andl	$-4096,%r8d		// we work in page multiples
+	xorl	%eax,%eax		// initialize chunk offset
+	jmp	LTouchLoop
+
+// Touch in the next chunk.  We try to keep the prefetch unit in "kick-start" mode,
+// by touching two adjacent cache lines every 8 lines of each page, in four slices.
+// Because the source may be unaligned, we use byte loads to touch.
+//      rdx = length remaining (including this chunk)
+//      rdi = ptr to start of dest chunk
+//      rsi = ptr to start of source chunk
+//	r8d = chunk length (multiples of pages, less than  2**32)
+//	ecx = scratch reg used to read a byte of each cache line
+//	eax = chunk offset
+
+	.align	4,0x90			// 16-byte align inner loops
+LTouchLoop:
+	movzb	(%rsi,%rax),%ecx	// touch line 0, 2, 4, or 6 of page
+	movzb	1*64(%rsi,%rax),%ecx	// touch line 1, 3, 5, or 7
+	movzb	8*64(%rsi,%rax),%ecx	// touch line 8, 10, 12, or 14
+	movzb	9*64(%rsi,%rax),%ecx	// etc
+	
+	movzb	16*64(%rsi,%rax),%ecx
+	movzb	17*64(%rsi,%rax),%ecx
+	movzb	24*64(%rsi,%rax),%ecx
+	movzb	25*64(%rsi,%rax),%ecx
+
+	movzb	32*64(%rsi,%rax),%ecx
+	movzb	33*64(%rsi,%rax),%ecx
+	movzb	40*64(%rsi,%rax),%ecx
+	movzb	41*64(%rsi,%rax),%ecx
+
+	movzb	48*64(%rsi,%rax),%ecx
+	movzb	49*64(%rsi,%rax),%ecx
+	movzb	56*64(%rsi,%rax),%ecx
+	movzb	57*64(%rsi,%rax),%ecx
+	
+	subl	$-128,%eax		// next slice of page (adding 128 w 8-bit immediate)
+	testl	$512,%eax		// done with this page?
+	jz	LTouchLoop		// no, next of four slices
+	addl	$(4096-512),%eax	// move on to next page
+	cmpl	%eax,%r8d		// done with this chunk?
+	jnz	LTouchLoop		// no, do next page
+	
+// The chunk has been pre-fetched, now copy it using non-temporal stores.
+// There are two copy loops, depending on whether the source is 16-byte aligned
+// or not.
+
+	movl	%r8d,%ecx		// copy chunk size to a reg that doesn't use REX prefix
+	addq	%rcx,%rsi		// increment ptrs by chunk length
+	addq	%rcx,%rdi
+	subq	%rcx,%rdx		// adjust remaining length
+	negq	%rcx			// prepare loop index (counts up to 0)
+	testl	$15,%esi		// is source 16-byte aligned?
+	jnz	LVeryLongUnaligned	// no
+	jmp	LVeryLongAligned
+
+	.align	4,0x90			// 16-byte align inner loops
+LVeryLongAligned:			// aligned loop over 128-bytes
+        movdqa  (%rsi,%rcx),%xmm0
+        movdqa  16(%rsi,%rcx),%xmm1
+        movdqa  32(%rsi,%rcx),%xmm2
+        movdqa  48(%rsi,%rcx),%xmm3
+        movdqa  64(%rsi,%rcx),%xmm4
+        movdqa  80(%rsi,%rcx),%xmm5
+        movdqa  96(%rsi,%rcx),%xmm6
+        movdqa  112(%rsi,%rcx),%xmm7
+
+        movntdq %xmm0,(%rdi,%rcx)
+        movntdq %xmm1,16(%rdi,%rcx)
+        movntdq %xmm2,32(%rdi,%rcx)
+        movntdq %xmm3,48(%rdi,%rcx)
+        movntdq %xmm4,64(%rdi,%rcx)
+        movntdq %xmm5,80(%rdi,%rcx)
+        movntdq %xmm6,96(%rdi,%rcx)
+        movntdq %xmm7,112(%rdi,%rcx)
+        
+        subq    $-128,%rcx		// add 128 with an 8-bit immediate
+        jnz	LVeryLongAligned
+	jmp	LVeryLongChunkEnd
+
+	.align	4,0x90			// 16-byte align inner loops
+LVeryLongUnaligned:			// unaligned loop over 128-bytes
+        movdqu  (%rsi,%rcx),%xmm0
+        movdqu  16(%rsi,%rcx),%xmm1
+        movdqu  32(%rsi,%rcx),%xmm2
+        movdqu  48(%rsi,%rcx),%xmm3
+        movdqu  64(%rsi,%rcx),%xmm4
+        movdqu  80(%rsi,%rcx),%xmm5
+        movdqu  96(%rsi,%rcx),%xmm6
+        movdqu  112(%rsi,%rcx),%xmm7
+
+        movntdq %xmm0,(%rdi,%rcx)
+        movntdq %xmm1,16(%rdi,%rcx)
+        movntdq %xmm2,32(%rdi,%rcx)
+        movntdq %xmm3,48(%rdi,%rcx)
+        movntdq %xmm4,64(%rdi,%rcx)
+        movntdq %xmm5,80(%rdi,%rcx)
+        movntdq %xmm6,96(%rdi,%rcx)
+        movntdq %xmm7,112(%rdi,%rcx)
+        
+        subq    $-128,%rcx		// add 128 with an 8-bit immediate
+        jnz     LVeryLongUnaligned
+
+LVeryLongChunkEnd:
+	cmpq	$4096,%rdx		// at least another page to go?
+	jae	LBigChunkLoop		// yes
+	
+// Done.  Call memcpy() again to handle the 0-4095 bytes at the end.
+// We still have the args in the right registers:
+//	rdi = destination ptr
+//	rsi = source ptr
+//	rdx = length remaining (0..4095)
+
+	sfence				// required by non-temporal stores
+	testl	%edx,%edx		// anything left to copy?
+	jz	1f
+	movq	$_COMM_PAGE_32_TO_64(_COMM_PAGE_MEMCPY),%rax
+	call	*%rax
+1:
+	popq	%rbp			// restore frame ptr
+	ret
+
+	/* always match for now, as commpage_stuff_routine() will panic if no match */
+	COMMPAGE_DESCRIPTOR(longcopy_sse4_64, _COMM_PAGE_LONGCOPY, 0 ,0)
diff --git a/osfmk/i386/commpage/memset_pattern_sse3.s b/osfmk/i386/commpage/memset_pattern_sse3.s
new file mode 100644
index 000000000..4ab62d315
--- /dev/null
+++ b/osfmk/i386/commpage/memset_pattern_sse3.s
@@ -0,0 +1,185 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <machine/cpu_capabilities.h>
+#include <machine/commpage.h>
+
+/* The common path for nonzero memset and the memset_pattern routines,
+ * tuned for Pentium-M class processors with SSE3 and 64-byte cache lines.
+ * This is used by the following functions:
+ *
+ *	void *memset(void *b, int c, size_t len);                   // when c!=0
+ *	void memset_pattern4(void *b, const void *c4, size_t len);
+ *	void memset_pattern8(void *b, const void *c8, size_t len);
+ *	void memset_pattern16(void *b, const void *c16, size_t len);
+ *
+ * Note bzero() and memset() of 0 are handled separately.
+ */
+
+#define	kShort		63
+#define	kVeryLong	(1024*1024)
+
+// Initial entry from Libc with parameters passed in registers.  Although we
+// correctly handle misaligned ptrs and short operands, they are inefficient.
+// Therefore our caller should filter out short operands and exploit local
+// knowledge (ie, original pattern length) to align the ptr if possible.
+// When called, we expect:
+//	%edi = ptr to memory to set (not necessarily aligned)
+//	%edx = length (may be short or even 0)
+//	%xmm0 = the pattern to store
+// Return conditions:
+//	%eax, %edi, %esi, %ecx, and %edx all trashed
+
+        .text
+        .align  5, 0x90
+Lmemset_pattern_sse3:
+        cmpl    $(kShort),%edx		// long enough to bother aligning?
+        ja	LNotShort		// yes
+	jmp	LShort			// no
+        
+// Here for short operands or the end of long ones.
+//      %edx = length
+//      %edi = ptr (may not be not aligned)
+//      %xmm0 = pattern
+
+LUnalignedStore16:
+	movdqu	%xmm0,(%edi)		// stuff in another 16 bytes
+	subl	$16,%edx
+	addl	$16,%edi
+LShort:	
+	cmpl	$16,%edx		// room for another vector?
+	jge	LUnalignedStore16	// yes
+LLessThan16:				// here at end of copy with < 16 bytes remaining
+	test	$8,%dl			// 8-byte store required?
+	jz	2f			// no
+	movq	%xmm0,(%edi)		// pack in 8 low bytes
+	psrldq	$8,%xmm0		// then shift vector down 8 bytes
+	addl	$8,%edi
+2:
+	test	$4,%dl			// 4-byte store required?
+	jz	3f			// no
+	movd	%xmm0,(%edi)		// pack in 4 low bytes
+	psrldq	$4,%xmm0		// then shift vector down 4 bytes
+	addl	$4,%edi
+3:
+	andl	$3,%edx			// more to go?
+	jz	5f			// no
+	movd	%xmm0,%eax		// move remainders out into %eax
+4:					// loop on up to three bytes
+	movb	%al,(%edi)		// pack in next byte
+	shrl	$8,%eax			// shift next byte into position
+	inc	%edi
+	dec	%edx
+	jnz	4b
+5:	ret
+        
+// Long enough to justify aligning ptr.  Note that we have to rotate the
+// pattern to account for any alignment.  We do this by doing two unaligned
+// stores, and then an aligned load from the middle of the two stores.
+// This will stall on store forwarding alignment mismatch, and the unaligned
+// stores can be pretty slow too, but the alternatives aren't any better.
+// Fortunately, in most cases our caller has already aligned the ptr.
+//      %edx = length (> kShort)
+//      %edi = ptr (may not be aligned)
+//      %xmm0 = pattern
+
+LNotShort:
+        movl    %edi,%ecx		// copy dest ptr
+        negl    %ecx
+        andl    $15,%ecx                // mask down to #bytes to 16-byte align
+	jz	LAligned		// skip if already aligned
+	movdqu	%xmm0,(%edi)		// store 16 unaligned bytes
+	movdqu	%xmm0,16(%edi)		// and 16 more, to be sure we have an aligned chunk
+	addl	%ecx,%edi		// now point to the aligned chunk
+	subl	%ecx,%edx		// adjust remaining count
+	movdqa	(%edi),%xmm0		// get the rotated pattern (probably stalling)
+	addl	$16,%edi		// skip past the aligned chunk
+	subl	$16,%edx
+
+// Set up for 64-byte loops.
+//      %edx = length remaining
+//      %edi = ptr (aligned)
+//      %xmm0 = rotated pattern
+
+LAligned:
+	movl	%edx,%ecx		// copy length remaining
+        andl    $63,%edx                // mask down to residual length (0..63)
+        andl    $-64,%ecx               // %ecx <- #bytes we will zero in by-64 loop
+	jz	LNoMoreChunks		// no 64-byte chunks
+        addl    %ecx,%edi               // increment ptr by length to move
+	cmpl	$(kVeryLong),%ecx	// long enough to justify non-temporal stores?
+	jge	LVeryLong		// yes
+        negl    %ecx			// negate length to move
+	jmp	1f
+	
+// Loop over 64-byte chunks, storing into cache.
+
+	.align	4,0x90			// keep inner loops 16-byte aligned
+1:
+        movdqa  %xmm0,(%edi,%ecx)
+        movdqa  %xmm0,16(%edi,%ecx)
+        movdqa  %xmm0,32(%edi,%ecx)
+        movdqa  %xmm0,48(%edi,%ecx)
+        addl    $64,%ecx
+        jne     1b
+	
+	jmp	LNoMoreChunks
+	
+// Very long operands: use non-temporal stores to bypass cache.
+
+LVeryLong:
+        negl    %ecx			// negate length to move
+	jmp	1f
+	
+	.align	4,0x90			// keep inner loops 16-byte aligned
+1:
+        movntdq %xmm0,(%edi,%ecx)
+        movntdq %xmm0,16(%edi,%ecx)
+        movntdq %xmm0,32(%edi,%ecx)
+        movntdq %xmm0,48(%edi,%ecx)
+        addl    $64,%ecx
+        jne     1b
+
+        sfence                          // required by non-temporal stores
+	jmp	LNoMoreChunks
+	
+// Handle leftovers: loop by 16.
+//      %edx = length remaining (<64)
+//      %edi = ptr (aligned)
+//      %xmm0 = rotated pattern
+
+LLoopBy16:
+	movdqa	%xmm0,(%edi)		// pack in 16 more bytes
+	subl	$16,%edx		// decrement count
+	addl	$16,%edi		// increment ptr
+LNoMoreChunks:
+	cmpl	$16,%edx		// more to go?
+	jge	LLoopBy16		// yes
+	jmp	LLessThan16		// handle up to 15 remaining bytes
+
+	COMMPAGE_DESCRIPTOR(memset_pattern_sse3,_COMM_PAGE_MEMSET_PATTERN,kHasSSE2,0)
diff --git a/osfmk/i386/commpage/memset_pattern_sse3_64.s b/osfmk/i386/commpage/memset_pattern_sse3_64.s
new file mode 100644
index 000000000..a7e5693e7
--- /dev/null
+++ b/osfmk/i386/commpage/memset_pattern_sse3_64.s
@@ -0,0 +1,187 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <machine/cpu_capabilities.h>
+#include <machine/commpage.h>
+
+/* The common path for nonzero memset and the memset_pattern routines,
+ * tuned for Pentium-M class processors with SSE3 and 64-byte cache lines.
+ * This is the 64-bit bersion.  It is used by the following functions:
+ *
+ *	void *memset(void *b, int c, size_t len);                   // when c!=0
+ *	void memset_pattern4(void *b, const void *c4, size_t len);
+ *	void memset_pattern8(void *b, const void *c8, size_t len);
+ *	void memset_pattern16(void *b, const void *c16, size_t len);
+ *
+ * Note bzero() and memset() of 0 are handled separately.
+ */
+
+#define	kShort		63
+#define	kVeryLong	(1024*1024)
+
+// Initial entry from Libc with parameters passed in registers.  Although we
+// correctly handle misaligned ptrs and short operands, they are inefficient.
+// Therefore our caller should filter out short operands and exploit local
+// knowledge (ie, original pattern length) to align the ptr if possible.
+// When called, we expect:
+//	%rdi = ptr to memory to set (not necessarily aligned)
+//	%rdx = length (may be short or even 0)
+//	%xmm0 = the pattern to store
+// Return conditions:
+//	%rax, %rdi, %rsi, %rcx, and %rdx all trashed
+//	we preserve %r8, %r9, %r10, and %r11
+
+        .text
+        .align  5, 0x90
+	.code64
+Lmemset_pattern_sse3_64:
+        cmpq    $(kShort),%rdx		// long enough to bother aligning?
+        ja	LNotShort		// yes
+	jmp	LShort			// no
+        
+// Here for short operands or the end of long ones.
+//      %rdx = length (<= kShort)
+//      %rdi = ptr (may not be not aligned)
+//      %xmm0 = pattern
+
+LUnalignedStore16:
+	movdqu	%xmm0,(%rdi)		// stuff in another 16 bytes
+	subl	$16,%edx
+	addq	$16,%rdi
+LShort:	
+	cmpl	$16,%edx		// room for another vector?
+	jge	LUnalignedStore16	// yes
+LLessThan16:				// here at end of copy with < 16 bytes remaining
+	test	$8,%dl			// 8-byte store required?
+	jz	2f			// no
+	movq	%xmm0,(%rdi)		// pack in 8 low bytes
+	psrldq	$8,%xmm0		// then shift vector down 8 bytes
+	addq	$8,%rdi
+2:
+	test	$4,%dl			// 4-byte store required?
+	jz	3f			// no
+	movd	%xmm0,(%rdi)		// pack in 4 low bytes
+	psrldq	$4,%xmm0		// then shift vector down 4 bytes
+	addq	$4,%rdi
+3:
+	andl	$3,%edx			// more to go?
+	jz	5f			// no
+	movd	%xmm0,%eax		// move remainders out into %eax
+4:					// loop on up to three bytes
+	movb	%al,(%rdi)		// pack in next byte
+	shrl	$8,%eax			// shift next byte into position
+	incq	%rdi
+	dec	%edx
+	jnz	4b
+5:	ret
+        
+// Long enough to justify aligning ptr.  Note that we have to rotate the
+// pattern to account for any alignment.  We do this by doing two unaligned
+// stores, and then an aligned load from the middle of the two stores.
+// This will stall on store forwarding alignment mismatch, and the unaligned
+// stores can be pretty slow too, but the alternatives aren't any better.
+// Fortunately, in most cases our caller has already aligned the ptr.
+//      %rdx = length (> kShort)
+//      %rdi = ptr (may not be aligned)
+//      %xmm0 = pattern
+
+LNotShort:
+        movl    %edi,%ecx		// copy low bits of dest ptr
+        negl    %ecx
+        andl    $15,%ecx                // mask down to #bytes to 16-byte align
+	jz	LAligned		// skip if already aligned
+	movdqu	%xmm0,(%rdi)		// store 16 unaligned bytes
+	movdqu	%xmm0,16(%rdi)		// and 16 more, to be sure we have an aligned chunk
+	addq	%rcx,%rdi		// now point to the aligned chunk
+	subq	%rcx,%rdx		// adjust remaining count
+	movdqa	(%rdi),%xmm0		// get the rotated pattern (probably stalling)
+	addq	$16,%rdi		// skip past the aligned chunk
+	subq	$16,%rdx
+
+// Set up for 64-byte loops.
+//      %rdx = length remaining
+//      %rdi = ptr (aligned)
+//      %xmm0 = rotated pattern
+
+LAligned:
+	movq	%rdx,%rcx		// copy length remaining
+        andl    $63,%edx                // mask down to residual length (0..63)
+        andq    $-64,%rcx               // %ecx <- #bytes we will zero in by-64 loop
+	jz	LNoMoreChunks		// no 64-byte chunks
+        addq    %rcx,%rdi               // increment ptr by length to move
+	cmpq	$(kVeryLong),%rcx	// long enough to justify non-temporal stores?
+	jge	LVeryLong		// yes
+        negq    %rcx			// negate length to move
+	jmp	1f
+	
+// Loop over 64-byte chunks, storing into cache.
+
+	.align	4,0x90			// keep inner loops 16-byte aligned
+1:
+        movdqa  %xmm0,(%rdi,%rcx)
+        movdqa  %xmm0,16(%rdi,%rcx)
+        movdqa  %xmm0,32(%rdi,%rcx)
+        movdqa  %xmm0,48(%rdi,%rcx)
+        addq    $64,%rcx
+        jne     1b
+	
+	jmp	LNoMoreChunks
+	
+// Very long operands: use non-temporal stores to bypass cache.
+
+LVeryLong:
+        negq    %rcx			// negate length to move
+	jmp	1f
+	
+	.align	4,0x90			// keep inner loops 16-byte aligned
+1:
+        movntdq %xmm0,(%rdi,%rcx)
+        movntdq %xmm0,16(%rdi,%rcx)
+        movntdq %xmm0,32(%rdi,%rcx)
+        movntdq %xmm0,48(%rdi,%rcx)
+        addq    $64,%rcx
+        jne     1b
+
+        sfence                          // required by non-temporal stores
+	jmp	LNoMoreChunks
+	
+// Handle leftovers: loop by 16.
+//      %edx = length remaining (<64)
+//      %edi = ptr (aligned)
+//      %xmm0 = rotated pattern
+
+LLoopBy16:
+	movdqa	%xmm0,(%rdi)		// pack in 16 more bytes
+	subl	$16,%edx		// decrement count
+	addq	$16,%rdi		// increment ptr
+LNoMoreChunks:
+	cmpl	$16,%edx		// more to go?
+	jge	LLoopBy16		// yes
+	jmp	LLessThan16		// handle up to 15 remaining bytes
+
+	COMMPAGE_DESCRIPTOR(memset_pattern_sse3_64,_COMM_PAGE_MEMSET_PATTERN,kHasSSE3,0)
diff --git a/osfmk/i386/commpage/pthreads.s b/osfmk/i386/commpage/pthreads.s
index 30600f75d..484857e13 100644
--- a/osfmk/i386/commpage/pthreads.s
+++ b/osfmk/i386/commpage/pthreads.s
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -30,21 +30,41 @@
 #include <machine/cpu_capabilities.h>
 #include <machine/commpage.h>
 
-#define _PTHREAD_TSD_OFFSET 0x48
+#define _PTHREAD_TSD_OFFSET32 0x48
+#define _PTHREAD_TSD_OFFSET64 0x60
 
+
+/* These routines do not need to be on the copmmpage on Intel.  They are for now
+ * to avoid revlock, but the code should move to Libc, and we should eventually remove
+ * these.
+ */
         .text
         .align  2, 0x90
 
 Lpthread_getspecific:
 	movl	4(%esp), %eax
-	movl	%gs:_PTHREAD_TSD_OFFSET(,%eax,4), %eax
+	movl	%gs:_PTHREAD_TSD_OFFSET32(,%eax,4), %eax
 	ret
 
 	COMMPAGE_DESCRIPTOR(pthread_getspecific,_COMM_PAGE_PTHREAD_GETSPECIFIC,0,0)
 
 Lpthread_self:
-	movl	4(%esp), %eax
-	movl	%gs:_PTHREAD_TSD_OFFSET, %eax
+	movl	%gs:_PTHREAD_TSD_OFFSET32, %eax
 	ret
 
 	COMMPAGE_DESCRIPTOR(pthread_self,_COMM_PAGE_PTHREAD_SELF,0,0)
+
+/* the 64-bit versions: */
+	
+	.code64
+Lpthread_getspecific_64:
+	movq	%gs:_PTHREAD_TSD_OFFSET64(,%rdi,8), %rax
+	ret
+
+	COMMPAGE_DESCRIPTOR(pthread_getspecific_64,_COMM_PAGE_PTHREAD_GETSPECIFIC,0,0)
+
+Lpthread_self_64:
+	movq	%gs:_PTHREAD_TSD_OFFSET64, %rax
+	ret
+
+	COMMPAGE_DESCRIPTOR(pthread_self_64,_COMM_PAGE_PTHREAD_SELF,0,0)
diff --git a/osfmk/i386/commpage/spinlocks.s b/osfmk/i386/commpage/spinlocks.s
index e69f875bd..e582635a0 100644
--- a/osfmk/i386/commpage/spinlocks.s
+++ b/osfmk/i386/commpage/spinlocks.s
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -29,23 +29,9 @@
 #include <sys/appleapiopts.h>
 #include <machine/cpu_capabilities.h>
 #include <machine/commpage.h>
+#include <mach/i386/syscall_sw.h>
+	
 
-/*
- * We need a relative branch within the comm page, and don't want the linker
- * to relocate it, so we have to hand-code the instructions. LEN is to account
- * for the length of a .long, since the jmp is relative to the next instruction.
- */
-
-#define JNZ .byte 0x0f, 0x85; .long
-#define JMP .byte 0xe9; .long
-#define LEN 4 
-
-/*
- * Branch prediction prefixes
- */
-
-#define LIKELY		.byte 0x3e
-#define UNLIKELY	.byte 0x2e
 
 #define MP_SPIN_TRIES	1024
 
@@ -63,6 +49,7 @@ Lspin_lock_try_up:
 
 	COMMPAGE_DESCRIPTOR(spin_lock_try_up,_COMM_PAGE_SPINLOCK_TRY,kUP,0)
  
+ 
 	.align 4, 0x90
 Lspin_lock_try_mp:
 	movl		4(%esp), %ecx 
@@ -76,21 +63,29 @@ Lspin_lock_try_mp:
 
 	COMMPAGE_DESCRIPTOR(spin_lock_try_mp,_COMM_PAGE_SPINLOCK_TRY,0,kUP)
 
-.set Lrelinquish_off,	_COMM_PAGE_RELINQUISH - _COMM_PAGE_SPINLOCK_LOCK
 
 	.align 4, 0x90
 Lspin_lock_up:
 	movl		4(%esp), %ecx
 	xorl		%eax, %eax
-.set Lretry,		. - Lspin_lock_up
 	orl		$-1, %edx
 	cmpxchgl	%edx, (%ecx)
-	UNLIKELY
-	JNZ		Lrelinquish_off - . + Lspin_lock_up - LEN
+	jnz,pn		1f		/* predict not taken */
 	ret
+1:
+	/* failed to get lock so relinquish the processor immediately on UP */
+	pushl		$1		/* 1 ms				*/
+	pushl		$1		/* SWITCH_OPTION_DEPRESS	*/
+	pushl		$0		/* THREAD_NULL			*/
+	pushl		$0		/* push dummy stack ret addr    */
+	movl		$-61,%eax	/* SYSCALL_THREAD_SWITCH */
+	int		$(MACH_INT)
+	addl		$16, %esp	/* adjust stack*/
+	jmp		Lspin_lock_up
 
 	COMMPAGE_DESCRIPTOR(spin_lock_up,_COMM_PAGE_SPINLOCK_LOCK,kUP,0)
 
+
 	.align 4, 0x90
 Lspin_lock_mp:
 	movl		4(%esp), %ecx
@@ -99,8 +94,7 @@ Lspin_lock_mp:
 	orl		$-1, %edx
 	lock
 	cmpxchgl	%edx, (%ecx)
-	UNLIKELY
-	jnz		1f
+	jnz,pn		1f		/* predict not taken */
 	ret
 1:
 	xorl		%eax, %eax
@@ -108,15 +102,22 @@ Lspin_lock_mp:
 2:
 	pause	
 	cmpl		%eax, (%ecx)
-	LIKELY
-	jz		0b
+	jz,pt		0b		/* favor success and slow down spin loop */
 	decl		%edx
-	LIKELY
-	jnz		2b
-	JMP		Lrelinquish_off - . + Lspin_lock_mp - LEN
+	jnz,pn		2b		/* slow down spin loop with a mispredict */
+	/* failed to get lock after spinning so relinquish  */
+	pushl		$1		/* 1 ms				*/
+	pushl		$1		/* SWITCH_OPTION_DEPRESS	*/
+	pushl		$0		/* THREAD_NULL			*/
+	pushl		$0		/* push dummy stack ret addr    */
+	movl		$-61,%eax	/* SYSCALL_THREAD_SWITCH */
+	int		$(MACH_INT)
+	addl		$16, %esp	/* adjust stack*/
+	jmp		Lspin_lock_mp
  
 	COMMPAGE_DESCRIPTOR(spin_lock_mp,_COMM_PAGE_SPINLOCK_LOCK,0,kUP)
 
+
 	.align 4, 0x90
 Lspin_unlock:
 	movl		4(%esp), %ecx
@@ -125,16 +126,93 @@ Lspin_unlock:
 
 	COMMPAGE_DESCRIPTOR(spin_unlock,_COMM_PAGE_SPINLOCK_UNLOCK,0,0)
 
+
+/* ============================ 64-bit versions follow ===================== */
+
+
+	.text
+	.code64
 	.align 4, 0x90
-Lrelinquish:				/* relinquish the processor	*/
-	pushl		$1		/* 1 ms				*/
-	pushl		$1		/* SWITCH_OPTION_DEPRESS	*/
-	pushl		$0		/* THREAD_NULL			*/
-	pushl		$0		/* push dummy stack ret addr    */
-	movl		$-61, %eax	/* syscall_thread_switch	*/
-	lcall		$7, $0
-	addl		$16, %esp	/* adjust stack*/
-	xorl		%eax, %eax	/* set %eax to 0 again		*/
-	JMP		Lretry - Lrelinquish_off - . + Lrelinquish - LEN
 
-	COMMPAGE_DESCRIPTOR(relinquish,_COMM_PAGE_RELINQUISH,0,0)
+Lspin_lock_try_up_64:
+	xorl		%eax, %eax
+	orl		$-1, %edx
+	cmpxchgl	%edx, (%rdi)
+	setz		%dl
+	movzbl		%dl, %eax
+	ret
+
+	COMMPAGE_DESCRIPTOR(spin_lock_try_up_64,_COMM_PAGE_SPINLOCK_TRY,kUP,0)
+
+
+	.align 4, 0x90
+Lspin_lock_try_mp_64:
+	xorl		%eax, %eax
+	orl		$-1, %edx
+	lock
+	cmpxchgl	%edx, (%rdi)
+	setz		%dl
+	movzbl		%dl, %eax
+	ret
+
+	COMMPAGE_DESCRIPTOR(spin_lock_try_mp_64,_COMM_PAGE_SPINLOCK_TRY,0,kUP)
+
+
+	.align 4, 0x90
+Lspin_lock_up_64:
+	movq		%rdi,%r8
+0:
+	xorl		%eax, %eax
+	orl		$-1, %edx
+	cmpxchgl	%edx, (%r8)
+	jnz,pn		1f		/* predict not taken */
+	ret
+1:
+	/* failed to get lock so relinquish the processor immediately on UP */
+	xorl		%edi,%edi	/* THREAD_NULL			*/
+	movl		$1,%esi		/* SWITCH_OPTION_DEPRESS	*/
+	movl		$1,%edx		/* 1 ms				*/
+	movl		$(SYSCALL_CONSTRUCT_MACH(61)),%eax	/* 61 = thread_switch */
+	syscall
+	jmp		0b
+
+	COMMPAGE_DESCRIPTOR(spin_lock_up_64,_COMM_PAGE_SPINLOCK_LOCK,kUP,0)
+	
+	
+
+	.align 4, 0x90
+Lspin_lock_mp_64:
+	movq		%rdi,%r8
+0:
+	xorl		%eax, %eax
+	orl		$-1, %edx
+	lock
+	cmpxchgl	%edx, (%r8)
+	jnz,pn		1f		/* predict not taken */
+	ret
+1:
+	xorl		%eax, %eax
+	movl		$(MP_SPIN_TRIES), %edx
+2:					/* spin for awhile before relinquish */
+	pause	
+	cmpl		%eax, (%r8)
+	jz		0b
+	decl		%edx
+	jnz		2b
+	/* failed to get lock after spinning so relinquish  */
+	xorl		%edi,%edi	/* THREAD_NULL			*/
+	movl		$1,%esi		/* SWITCH_OPTION_DEPRESS	*/
+	movl		$1,%edx		/* 1 ms				*/
+	movl		$(SYSCALL_CONSTRUCT_MACH(61)),%eax	/* 61 = thread_switch */
+	syscall
+	jmp		0b
+ 
+	COMMPAGE_DESCRIPTOR(spin_lock_mp_64,_COMM_PAGE_SPINLOCK_LOCK,0,kUP)
+
+
+	.align 4, 0x90
+Lspin_unlock_64:
+	movl		$0, (%rdi)
+	ret
+
+	COMMPAGE_DESCRIPTOR(spin_unlock_64,_COMM_PAGE_SPINLOCK_UNLOCK,0,0)
diff --git a/osfmk/i386/cpu.c b/osfmk/i386/cpu.c
index 52460c149..5ad19a057 100644
--- a/osfmk/i386/cpu.c
+++ b/osfmk/i386/cpu.c
@@ -41,8 +41,9 @@
 #include <i386/pmap.h>
 #include <i386/misc_protos.h>
 #include <i386/cpu_threads.h>
+#include <i386/rtclock.h>
 #include <vm/vm_kern.h>
-
+#include "cpuid.h"
 
 struct processor	processor_master;
 
@@ -86,6 +87,8 @@ cpu_sleep(void)
 {
 	cpu_data_t	*proc_info = current_cpu_datap();
 
+	proc_info->cpu_running = FALSE;
+
 	PE_cpu_machine_quiesce(proc_info->cpu_id);
 
 	cpu_thread_halt();
@@ -96,14 +99,9 @@ cpu_init(void)
 {
 	cpu_data_t	*cdp = current_cpu_datap();
 
-#ifdef	MACH_BSD
-	/* FIXME */
-	cdp->cpu_type = CPU_TYPE_I386;
-	cdp->cpu_subtype = CPU_SUBTYPE_PENTPRO;
-#else
-	cdp->cpu_type = cpuid_cputype(0);
-	cdp->cpu_subtype = CPU_SUBTYPE_AT386;
-#endif
+	cdp->cpu_type = cpuid_cputype();
+	cdp->cpu_subtype = cpuid_cpusubtype();
+
 	cdp->cpu_running = TRUE;
 }
 
@@ -136,10 +134,11 @@ void
 cpu_machine_init(
 	void)
 {
-	int	cpu;
+	cpu_data_t	*cdp = current_cpu_datap();
 
-	cpu = get_cpu_number();
-	PE_cpu_machine_init(cpu_datap(cpu)->cpu_id, TRUE);
+	PE_cpu_machine_init(cdp->cpu_id, !cdp->cpu_boot_complete);
+	cdp->cpu_boot_complete = TRUE;
+	cdp->cpu_running = TRUE;
 #if 0
 	if (cpu_datap(cpu)->hibernate)
 	{
diff --git a/osfmk/i386/cpu_capabilities.h b/osfmk/i386/cpu_capabilities.h
index 40dafc33c..c7e22b1e3 100644
--- a/osfmk/i386/cpu_capabilities.h
+++ b/osfmk/i386/cpu_capabilities.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -35,10 +35,7 @@
 #endif
  
 /*
- * This is the authoritative way to determine from user mode what
- * implementation-specific processor features are available.
  * This API only supported for Apple internal use.
- * 
  */
 
 /* Bit definitions for _cpu_capabilities: */
@@ -47,15 +44,18 @@
 #define	kHasSSE				0x00000002
 #define	kHasSSE2			0x00000004
 #define	kHasSSE3			0x00000008
-#define	kCache32			0x00000010	// cache line size is 32 bytes
+#define	kCache32			0x00000010	/* cache line size is 32 bytes */
 #define	kCache64			0x00000020
 #define	kCache128			0x00000040
-#define	kFastThreadLocalStorage		0x00000080	// TLS ptr is kept in a user-mode-readable register
+#define	kFastThreadLocalStorage		0x00000080	/* TLS ptr is kept in a user-mode-readable register */
+#define kHasSupplementalSSE3	0x00000100
+#define kHasMNI				kHasSupplementalSSE3
+#define	k64Bit				0x00000200	/* processor supports EM64T (not what mode you're running in) */
 
-#define	kUP				0x00008000	// set if (kNumCPUs == 1)
-#define	kNumCPUs			0x00FF0000	// number of CPUs (see _NumCPUs() below)
+#define	kUP				0x00008000	/* set if (kNumCPUs == 1) */
+#define	kNumCPUs			0x00FF0000	/* number of CPUs (see _NumCPUs() below) */
 
-#define	kNumCPUsShift		16			// see _NumCPUs() below
+#define	kNumCPUsShift			16		/* see _NumCPUs() below */
 
 #ifndef	__ASSEMBLER__
 #include <sys/cdefs.h>
@@ -76,92 +76,137 @@ int _NumCPUs( void )
 /*
  * The shared kernel/user "comm page(s)":
  *
- * The last eight pages of every address space are reserved for the kernel/user
- * "comm area". During system initialization, the kernel populates the comm page with
+ * The last several pages of every address space are reserved for the kernel/user
+ * "comm area". During system initialization, the kernel populates the comm pages with
  * code customized for the particular processor and platform.
  *
- * Because Mach VM cannot map the last page of an address space, the max length of
- * the comm area is seven pages.
+ * Because Mach VM cannot map the last page of an address space, we don't use it.
  */
  
-#define	_COMM_PAGE_AREA_LENGTH	(19*4096)
-					// reserved length of entire comm area
-#define _COMM_PAGE_BASE_ADDRESS (-20*4096)
-					// VM_MAX_ADDRESS-_COMM_PAGE_AREA_LENGTH
-#define _COMM_PAGE_START_ADDRESS (-16*4096)
-					// VM_MAX_ADDRESS-_COMM_PAGE_AREA_LENGTH
-#define _COMM_PAGE_SIGS_OFFSET	0x8000
-					// offset to routine signatures
+#define	_COMM_PAGE32_AREA_LENGTH	( 19 * 4096 )				/* reserved length of entire comm area */
+#define _COMM_PAGE32_BASE_ADDRESS	( -20 * 4096 )				/* base address of allocated memory */
+#define _COMM_PAGE32_START_ADDRESS	( -16 * 4096 )				/* address traditional commpage code starts on */
+#define _COMM_PAGE32_AREA_USED		( 19 * 4096 )				/* this is the amt actually allocated */
+
+#define	_COMM_PAGE64_AREA_LENGTH	( 2 * 1024 * 1024 )			/* reserved length of entire comm area (2MB) */
+#define _COMM_PAGE64_BASE_ADDRESS	( 0x00007fffffe00000ULL )		/* base address of allocated memory */
+#define _COMM_PAGE64_START_ADDRESS	( _COMM_PAGE64_BASE_ADDRESS )		/* address traditional commpage code starts on */
+#define _COMM_PAGE64_AREA_USED		( 2 * 4096 )				/* this is the amt actually populated */
+
+
+#if defined(__i386__)
+
+#define	_COMM_PAGE_AREA_LENGTH		_COMM_PAGE32_AREA_LENGTH
+#define _COMM_PAGE_BASE_ADDRESS		_COMM_PAGE32_BASE_ADDRESS
+#define _COMM_PAGE_START_ADDRESS	_COMM_PAGE32_START_ADDRESS
+#define _COMM_PAGE_AREA_USED		_COMM_PAGE32_AREA_USED
+#define _COMM_PAGE_SIGS_OFFSET		0x8000					/* offset to routine signatures */
+
+#elif defined(__x86_64__)
+
+#define	_COMM_PAGE_AREA_LENGTH		_COMM_PAGE64_AREA_LENGTH
+#define _COMM_PAGE_BASE_ADDRESS		_COMM_PAGE64_BASE_ADDRESS
+#define _COMM_PAGE_START_ADDRESS	_COMM_PAGE64_START_ADDRESS
+#define _COMM_PAGE_AREA_USED		_COMM_PAGE64_AREA_USED
+
+#else
+#error architecture not supported
+#endif
  
 /* data in the comm page */
  
-#define _COMM_PAGE_SIGNATURE			(_COMM_PAGE_START_ADDRESS+0x000)	// first few bytes are a signature
-#define _COMM_PAGE_VERSION				(_COMM_PAGE_START_ADDRESS+0x01E)	// 16-bit version#
-#define	_COMM_PAGE_THIS_VERSION			3								// version of the commarea format
+#define _COMM_PAGE_SIGNATURE		(_COMM_PAGE_START_ADDRESS+0x000)	/* first few bytes are a signature */
+#define _COMM_PAGE_VERSION		(_COMM_PAGE_START_ADDRESS+0x01E)	/* 16-bit version# */
+#define	_COMM_PAGE_THIS_VERSION		6					/* version of the commarea format */
   
-#define _COMM_PAGE_CPU_CAPABILITIES		(_COMM_PAGE_START_ADDRESS+0x020)	// uint32_t _cpu_capabilities
-#define _COMM_PAGE_NCPUS				(_COMM_PAGE_START_ADDRESS+0x021)	// uint8_t number of configured CPUs
-#define _COMM_PAGE_VECTOR_FLAVOR		(_COMM_PAGE_START_ADDRESS+0x024)	// uint8_t SSE/SSE2/SSE3 
-#define _COMM_PAGE_CACHE_LINESIZE		(_COMM_PAGE_START_ADDRESS+0x026)	// uint16_t cache line size
- 
-#define _COMM_PAGE_UNUSED1				(_COMM_PAGE_START_ADDRESS+0x030)	// 16 unused bytes
- 
-#define _COMM_PAGE_2_TO_52				(_COMM_PAGE_START_ADDRESS+0x040)	// double float constant 2**52
-#define _COMM_PAGE_10_TO_6				(_COMM_PAGE_START_ADDRESS+0x048)	// double float constant 10**6
+#define _COMM_PAGE_CPU_CAPABILITIES	(_COMM_PAGE_START_ADDRESS+0x020)	/* uint32_t _cpu_capabilities */
+#define _COMM_PAGE_NCPUS		(_COMM_PAGE_START_ADDRESS+0x022)	/* uint8_t number of configured CPUs */
+#define _COMM_PAGE_CACHE_LINESIZE	(_COMM_PAGE_START_ADDRESS+0x026)	/* uint16_t cache line size */
+
+#define _COMM_PAGE_UNUSED1		(_COMM_PAGE_START_ADDRESS+0x028)	/* 24 unused bytes */
  
-#define _COMM_PAGE_UNUSED2				(_COMM_PAGE_START_ADDRESS+0x050)	// 16 unused bytes
+#if defined(__i386__)								/* following are not defined in 64-bit */
+#define _COMM_PAGE_2_TO_52		(_COMM_PAGE_START_ADDRESS+0x040)	/* double float constant 2**52 */
+#define _COMM_PAGE_10_TO_6		(_COMM_PAGE_START_ADDRESS+0x048)	/* double float constant 10**6 */
+#else
+#define _COMM_PAGE_UNUSED2		(_COMM_PAGE_START_ADDRESS+0x040)	/* 16 unused bytes */
+#endif
+
+#define _COMM_PAGE_NT_TSC_BASE		(_COMM_PAGE_START_ADDRESS+0x050)	/* used by nanotime() */
+#define _COMM_PAGE_NT_SCALE		(_COMM_PAGE_START_ADDRESS+0x058)	/* used by nanotime() */
+#define _COMM_PAGE_NT_SHIFT		(_COMM_PAGE_START_ADDRESS+0x05c)	/* used by nanotime() */
+#define _COMM_PAGE_NT_NS_BASE		(_COMM_PAGE_START_ADDRESS+0x060)	/* used by nanotime() */
  
-#define _COMM_PAGE_TIMEBASE				(_COMM_PAGE_START_ADDRESS+0x060)	// used by gettimeofday()
-#define _COMM_PAGE_TIMESTAMP			(_COMM_PAGE_START_ADDRESS+0x068)	// used by gettimeofday()
-#define _COMM_PAGE_SEC_PER_TICK			(_COMM_PAGE_START_ADDRESS+0x070)	// used by gettimeofday()
+#define _COMM_PAGE_TIMEBASE		(_COMM_PAGE_START_ADDRESS+0x068)	/* used by gettimeofday() */
+#define _COMM_PAGE_TIMESTAMP		(_COMM_PAGE_START_ADDRESS+0x070)	/* used by gettimeofday() */
+#define _COMM_PAGE_TIMEENABLE		(_COMM_PAGE_START_ADDRESS+0x078)	/* used by gettimeofday() */
+
+#define	_NT_TSC_BASE			0					/* offsets into nanotime data */
+#define	_NT_SCALE			8
+#define	_NT_SHIFT			12
+#define	_NT_NS_BASE			16
+#define	_TIMEBASE			0					/* offsets into gettimeofday data */
+#define	_TIMESTAMP			8
+#define	_TIMEENABLE			16
  
- /* jump table (bla to this address, which may be a branch to the actual code somewhere else) */
+ /* jump table (jmp to this address, which may be a branch to the actual code somewhere else) */
  /* When new jump table entries are added, corresponding symbols should be added below         */
 
-#define _COMM_PAGE_COMPARE_AND_SWAP32   (_COMM_PAGE_START_ADDRESS+0x080) // compare-and-swap word
-#define _COMM_PAGE_COMPARE_AND_SWAP64   (_COMM_PAGE_START_ADDRESS+0x0c0) // compare-and-swap doubleword
-#define _COMM_PAGE_ENQUEUE              (_COMM_PAGE_START_ADDRESS+0x100) // enqueue
-#define _COMM_PAGE_DEQUEUE              (_COMM_PAGE_START_ADDRESS+0x140) // dequeue
-#define _COMM_PAGE_MEMORY_BARRIER       (_COMM_PAGE_START_ADDRESS+0x180) // add atomic doubleword
-#define _COMM_PAGE_ATOMIC_ADD32         (_COMM_PAGE_START_ADDRESS+0x1a0) // add atomic word
-#define _COMM_PAGE_ATOMIC_ADD64         (_COMM_PAGE_START_ADDRESS+0x1c0) // add atomic doubleword
-
-#define _COMM_PAGE_NANOTIME_INFO        (_COMM_PAGE_START_ADDRESS+0x1e0) // 32 bytes used by nanotime()
+#define _COMM_PAGE_COMPARE_AND_SWAP32   (_COMM_PAGE_START_ADDRESS+0x080)	/* compare-and-swap word */
+#define _COMM_PAGE_COMPARE_AND_SWAP64   (_COMM_PAGE_START_ADDRESS+0x0c0)	/* compare-and-swap doubleword */
+#define _COMM_PAGE_ENQUEUE              (_COMM_PAGE_START_ADDRESS+0x100)	/* enqueue */
+#define _COMM_PAGE_DEQUEUE              (_COMM_PAGE_START_ADDRESS+0x140)	/* dequeue */
+#define _COMM_PAGE_MEMORY_BARRIER       (_COMM_PAGE_START_ADDRESS+0x180)	/* memory barrier */
+#define _COMM_PAGE_ATOMIC_ADD32         (_COMM_PAGE_START_ADDRESS+0x1a0)	/* add atomic word */
+#define _COMM_PAGE_ATOMIC_ADD64         (_COMM_PAGE_START_ADDRESS+0x1c0)	/* add atomic doubleword */
+
+#define	_COMM_PAGE_UNUSED3		(_COMM_PAGE_START_ADDRESS+0x1e0)	/* 32 unused bytes */
+
+#define _COMM_PAGE_ABSOLUTE_TIME	(_COMM_PAGE_START_ADDRESS+0x200)	/* mach_absolute_time() */
+#define _COMM_PAGE_SPINLOCK_TRY		(_COMM_PAGE_START_ADDRESS+0x220)	/* spinlock_try() */
+#define _COMM_PAGE_SPINLOCK_LOCK	(_COMM_PAGE_START_ADDRESS+0x260)	/* spinlock_lock() */
+#define _COMM_PAGE_SPINLOCK_UNLOCK	(_COMM_PAGE_START_ADDRESS+0x2a0)	/* spinlock_unlock() */
+#define _COMM_PAGE_PTHREAD_GETSPECIFIC  (_COMM_PAGE_START_ADDRESS+0x2c0)	/* pthread_getspecific() */
+#define _COMM_PAGE_GETTIMEOFDAY		(_COMM_PAGE_START_ADDRESS+0x2e0)	/* used by gettimeofday() */
+#define _COMM_PAGE_FLUSH_DCACHE		(_COMM_PAGE_START_ADDRESS+0x4e0)	/* sys_dcache_flush() */
+#define _COMM_PAGE_FLUSH_ICACHE		(_COMM_PAGE_START_ADDRESS+0x520)	/* sys_icache_invalidate() */
+#define _COMM_PAGE_PTHREAD_SELF		(_COMM_PAGE_START_ADDRESS+0x580)	/* pthread_self() */
+
+#define _COMM_PAGE_UNUSED4		(_COMM_PAGE_START_ADDRESS+0x5a0)	/* 32 unused bytes */
+
+#define _COMM_PAGE_RELINQUISH		(_COMM_PAGE_START_ADDRESS+0x5c0)	/* used by spinlocks */ 
+#define _COMM_PAGE_BTS		        (_COMM_PAGE_START_ADDRESS+0x5e0)	/* bit test-and-set */
+#define _COMM_PAGE_BTC			(_COMM_PAGE_START_ADDRESS+0x5f0)	/* bit test-and-clear */
  
-#define _COMM_PAGE_ABSOLUTE_TIME		(_COMM_PAGE_START_ADDRESS+0x200)	// mach_absolute_time()
-#define _COMM_PAGE_SPINLOCK_TRY			(_COMM_PAGE_START_ADDRESS+0x220)	// spinlock_try()
-#define _COMM_PAGE_SPINLOCK_LOCK		(_COMM_PAGE_START_ADDRESS+0x260)	// spinlock_lock()
-#define _COMM_PAGE_SPINLOCK_UNLOCK		(_COMM_PAGE_START_ADDRESS+0x2a0)	// spinlock_unlock()
-#define _COMM_PAGE_PTHREAD_GETSPECIFIC  (_COMM_PAGE_START_ADDRESS+0x2c0)	// pthread_getspecific()
-#define _COMM_PAGE_GETTIMEOFDAY			(_COMM_PAGE_START_ADDRESS+0x2e0)	// used by gettimeofday()
-#define _COMM_PAGE_FLUSH_DCACHE			(_COMM_PAGE_START_ADDRESS+0x4e0)	// sys_dcache_flush()
-#define _COMM_PAGE_FLUSH_ICACHE			(_COMM_PAGE_START_ADDRESS+0x520)	// sys_icache_invalidate()
-#define _COMM_PAGE_PTHREAD_SELF			(_COMM_PAGE_START_ADDRESS+0x580)	// pthread_self()
-#define _COMM_PAGE_UNUSED4				(_COMM_PAGE_START_ADDRESS+0x5a0)	// 32 unused bytes
-#define _COMM_PAGE_RELINQUISH			(_COMM_PAGE_START_ADDRESS+0x5c0) // used by spinlocks
- 
-#define _COMM_PAGE_BTS		        	(_COMM_PAGE_START_ADDRESS+0x5e0) // bit test-and-set 
-#define _COMM_PAGE_BTC					(_COMM_PAGE_START_ADDRESS+0x5f0) // bit test-and-clear 
- 
-#define _COMM_PAGE_BZERO				(_COMM_PAGE_START_ADDRESS+0x600)	// bzero()
-#define _COMM_PAGE_BCOPY				(_COMM_PAGE_START_ADDRESS+0x780)	// bcopy()
-#define	_COMM_PAGE_MEMCPY				(_COMM_PAGE_START_ADDRESS+0x7a0)	// memcpy()
-#define	_COMM_PAGE_MEMMOVE				(_COMM_PAGE_START_ADDRESS+0x7a0)	// memmove()
+#define _COMM_PAGE_BZERO		(_COMM_PAGE_START_ADDRESS+0x600)	/* bzero() */
+#define _COMM_PAGE_BCOPY		(_COMM_PAGE_START_ADDRESS+0x780)	/* bcopy() */
+#define	_COMM_PAGE_MEMCPY		(_COMM_PAGE_START_ADDRESS+0x7a0)	/* memcpy() */
+#define	_COMM_PAGE_MEMMOVE		(_COMM_PAGE_START_ADDRESS+0x7a0)	/* memmove() */
+
+#define	_COMM_PAGE_OLD_NANOTIME		(_COMM_PAGE_START_ADDRESS+0xf80)	/* old nanotime location (deprecated) */
+#define	_COMM_PAGE_MEMSET_PATTERN       (_COMM_PAGE_START_ADDRESS+0x1000)	/* used by nonzero memset() */
+#define	_COMM_PAGE_LONGCOPY		(_COMM_PAGE_START_ADDRESS+0x1200)	/* used by bcopy() for very long operands */
+
+#define _COMM_PAGE_SYSTEM_INTEGRITY	(_COMM_PAGE_START_ADDRESS+0x1600)	/* system integrity data, 256 bytes */
 
-#define	_COMM_PAGE_NANOTIME				(_COMM_PAGE_START_ADDRESS+0xF80) // nanotime()
+#define	_COMM_PAGE_NANOTIME		(_COMM_PAGE_START_ADDRESS+0x1700)	/* nanotime() */
 
-#define	_COMM_PAGE_BIGCOPY				(_COMM_PAGE_START_ADDRESS+0x1000)// very-long-operand copies
+#define _COMM_PAGE_END			(_COMM_PAGE_START_ADDRESS+0x1780)	/* end of common page - insert new stuff here */
 
-#define _COMM_PAGE_END					(_COMM_PAGE_START_ADDRESS+0x1600)// end of common page
+/* _COMM_PAGE_COMPARE_AND_SWAP{32,64}B are not used on x86 and are
+ * maintained here for source compatability.  These will be removed at
+ * some point, so don't go relying on them. */
+#define _COMM_PAGE_COMPARE_AND_SWAP32B  (_COMM_PAGE_START_ADDRESS+0xf80)	/* compare-and-swap word w barrier */
+#define _COMM_PAGE_COMPARE_AND_SWAP64B  (_COMM_PAGE_START_ADDRESS+0xfc0)	/* compare-and-swap doubleword w barrier */
 
 #ifdef __ASSEMBLER__
 #ifdef __COMM_PAGE_SYMBOLS
 
 #define CREATE_COMM_PAGE_SYMBOL(symbol_name, symbol_address)		\
-				.org	(symbol_address - (_COMM_PAGE_BASE_ADDRESS & 0xFFFFE000)) ;\
+				.org	(symbol_address - (_COMM_PAGE_START_ADDRESS & 0xFFFFE000)) ;\
 symbol_name: nop
 
-	.text		// Required to make a well behaved symbol file
+	.text		/* Required to make a well behaved symbol file */
 
 	CREATE_COMM_PAGE_SYMBOL(___compare_and_swap32, _COMM_PAGE_COMPARE_AND_SWAP32)
 	CREATE_COMM_PAGE_SYMBOL(___compare_and_swap64, _COMM_PAGE_COMPARE_AND_SWAP64)
@@ -185,13 +230,15 @@ symbol_name: nop
 	CREATE_COMM_PAGE_SYMBOL(___bzero, _COMM_PAGE_BZERO)
 	CREATE_COMM_PAGE_SYMBOL(___bcopy, _COMM_PAGE_BCOPY)
 	CREATE_COMM_PAGE_SYMBOL(___memcpy, _COMM_PAGE_MEMCPY)
-//	CREATE_COMM_PAGE_SYMBOL(___memmove, _COMM_PAGE_MEMMOVE)
-	CREATE_COMM_PAGE_SYMBOL(___bigcopy, _COMM_PAGE_BIGCOPY)
+/*	CREATE_COMM_PAGE_SYMBOL(___memmove, _COMM_PAGE_MEMMOVE) */
+	CREATE_COMM_PAGE_SYMBOL(___old_nanotime, _COMM_PAGE_OLD_NANOTIME)
+	CREATE_COMM_PAGE_SYMBOL(___memset_pattern, _COMM_PAGE_MEMSET_PATTERN)
+	CREATE_COMM_PAGE_SYMBOL(___longcopy, _COMM_PAGE_LONGCOPY)
 	CREATE_COMM_PAGE_SYMBOL(___nanotime, _COMM_PAGE_NANOTIME)
 	CREATE_COMM_PAGE_SYMBOL(___end_comm_page, _COMM_PAGE_END)
 
-	.data		// Required to make a well behaved symbol file
-	.long	0	// Required to make a well behaved symbol file
+	.data		/* Required to make a well behaved symbol file */
+	.long	0	/* Required to make a well behaved symbol file */
 
 #endif /* __COMM_PAGE_SYMBOLS */
 #endif /* __ASSEMBLER__ */
diff --git a/osfmk/i386/cpu_data.h b/osfmk/i386/cpu_data.h
index 098cab211..3e4362e76 100644
--- a/osfmk/i386/cpu_data.h
+++ b/osfmk/i386/cpu_data.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -40,15 +40,18 @@
 #include <kern/assert.h>
 #include <kern/kern_types.h>
 #include <kern/processor.h>
+#include <kern/pms.h>
 #include <pexpert/pexpert.h>
-
+#include <mach/i386/thread_status.h>
+#include <i386/rtclock.h>
+#include <i386/pmCPU.h>
 
 /*
  * Data structures referenced (anonymously) from per-cpu data:
  */
 struct cpu_core;
 struct cpu_cons_buffer;
-struct mp_desc_table;
+struct cpu_desc_table;
 
 
 /*
@@ -61,12 +64,10 @@ typedef struct rtclock_timer {
 } rtclock_timer_t;
 
 typedef struct {
-	uint64_t	rnt_tsc;		/* timestamp */
-	uint64_t	rnt_nanos;		/* nanoseconds */
-	uint32_t	rnt_scale;		/* tsc -> nanosec multiplier */
-	uint32_t	rnt_shift;		/* tsc -> nanosec shift/div */
-	uint64_t	rnt_step_tsc;		/* tsc when scale applied */
-	uint64_t	rnt_step_nanos;		/* ns  when scale applied */
+	uint64_t	tsc_base;		/* timestamp */
+	uint64_t	ns_base;		/* nanoseconds */
+	uint32_t	scale;			/* tsc -> nanosec multiplier */
+	uint32_t	shift;			/* tsc -> nanosec shift/div */
 } rtc_nanotime_t;
 
 typedef struct {
@@ -77,8 +78,26 @@ typedef struct {
 	struct fake_descriptor  *cdi_gdt;
 	struct fake_descriptor  *cdi_idt;
 	struct fake_descriptor  *cdi_ldt;
+	vm_offset_t		cdi_sstk;
 } cpu_desc_index_t;
 
+typedef enum {
+	TASK_MAP_32BIT,			/* 32-bit, compatibility mode */ 
+	TASK_MAP_64BIT,			/* 64-bit, separate address space */ 
+	TASK_MAP_64BIT_SHARED		/* 64-bit, kernel-shared addr space */
+} task_map_t;
+
+/*
+ * This structure is used on entry into the (uber-)kernel on syscall from
+ * a 64-bit user. It contains the address of the machine state save area
+ * for the current thread and a temporary place to save the user's rsp
+ * before loading this address into rsp.
+ */
+typedef struct {
+	addr64_t	cu_isf;		/* thread->pcb->iss.isf */
+	uint64_t	cu_tmp;		/* temporary scratch */	
+        addr64_t	cu_user_gs_base;
+} cpu_uber_t;
 
 /*
  * Per-cpu data.
@@ -97,9 +116,9 @@ typedef struct cpu_data
 {
 	struct cpu_data		*cpu_this;		/* pointer to myself */
 	thread_t		cpu_active_thread;
-	thread_t		cpu_active_kloaded;
-	vm_offset_t		cpu_active_stack;
-	vm_offset_t		cpu_kernel_stack;
+	void			*cpu_int_state;		/* interrupt state */
+	vm_offset_t		cpu_active_stack;	/* kernel stack base */
+	vm_offset_t		cpu_kernel_stack;	/* kernel stack top */
 	vm_offset_t		cpu_int_stack_top;
 	int			cpu_preemption_level;
 	int			cpu_simple_lock_count;
@@ -114,27 +133,53 @@ typedef struct cpu_data
 	int			cpu_subtype;
 	int			cpu_threadtype;
 	int			cpu_running;
-	struct cpu_core		*cpu_core;		/* cpu's parent core */
-	uint64_t		cpu_rtc_tick_deadline;
-	uint64_t		cpu_rtc_intr_deadline;
-	rtclock_timer_t		cpu_rtc_timer;
-	rtc_nanotime_t		cpu_rtc_nanotime;
+	uint64_t		rtclock_intr_deadline;
+	rtclock_timer_t		rtclock_timer;
+        boolean_t		cpu_is64bit;
+        task_map_t		cpu_task_map;
+        addr64_t		cpu_task_cr3;
+        addr64_t		cpu_active_cr3;
+        addr64_t		cpu_kernel_cr3;
+	cpu_uber_t		cpu_uber;
+	void			*cpu_chud;
 	void			*cpu_console_buf;
+	struct cpu_core		*cpu_core;		/* cpu's parent core */
 	struct processor	*cpu_processor;
 	struct cpu_pmap		*cpu_pmap;
-	struct mp_desc_table	*cpu_desc_tablep;
+	struct cpu_desc_table	*cpu_desc_tablep;
+	struct fake_descriptor	*cpu_ldtp;
 	cpu_desc_index_t	cpu_desc_index;
-	boolean_t		cpu_iflag;
+	int			cpu_ldt;
 #ifdef MACH_KDB
 	/* XXX Untested: */
 	int			cpu_db_pass_thru;
-	vm_offset_t		cpu_db_stacks;
-	struct i386_saved_state *cpu_kdb_saved_state;
-	spl_t			cpu_kdb_saved_ipl;
+	vm_offset_t	cpu_db_stacks;
+	void		*cpu_kdb_saved_state;
+	spl_t		cpu_kdb_saved_ipl;
 	int			cpu_kdb_is_slave;
 	int			cpu_kdb_active;
 #endif /* MACH_KDB */
-        int                     cpu_hibernate;
+	boolean_t		cpu_iflag;
+	boolean_t		cpu_boot_complete;
+	int			cpu_hibernate;
+	pmsd			pms;					/* Power Management Stepper control */
+	uint64_t		rtcPop;			/* when the etimer wants a timer pop */
+	
+	vm_offset_t	cpu_copywindow_base;
+	uint64_t	*cpu_copywindow_pdp;
+	
+	vm_offset_t	cpu_physwindow_base;
+	uint64_t	*cpu_physwindow_ptep;
+	void 		*cpu_hi_iss;
+	boolean_t	cpu_tlb_invalid;
+	
+	uint64_t	*cpu_pmHpet;		/* Address of the HPET for this processor */
+	uint32_t	cpu_pmHpetVec;		/* Interrupt vector for HPET for this processor */
+/*	Statistics */
+	pmStats_t	cpu_pmStats;		/* Power management data */
+	uint32_t	cpu_hwIntCnt[256];		/* Interrupt counts */
+	
+	uint64_t		cpu_dr7; /* debug control register */
 } cpu_data_t;
 
 extern cpu_data_t	*cpu_data_ptr[];  
@@ -162,6 +207,13 @@ get_active_thread(void)
 #define current_thread_fast()		get_active_thread()
 #define current_thread()		current_thread_fast()
 
+static inline boolean_t
+get_is64bit(void)
+{
+	CPU_DATA_GET(cpu_is64bit, boolean_t)
+}
+#define cpu_mode_is64bit()		get_is64bit()
+
 static inline int
 get_preemption_level(void)
 {
diff --git a/osfmk/i386/cpu_threads.c b/osfmk/i386/cpu_threads.c
index 3731b8e0a..30a99af91 100644
--- a/osfmk/i386/cpu_threads.c
+++ b/osfmk/i386/cpu_threads.c
@@ -31,6 +31,7 @@
 #include <i386/cpuid.h>
 #include <i386/machine_cpu.h>
 #include <i386/lock.h>
+#include <i386/perfmon.h>
 
 /*
  * Kernel parameter determining whether threads are halted unconditionally
@@ -39,63 +40,81 @@
  */
 int idlehalt = 1;
 
-void
-cpu_thread_init(void)
+
+static boolean_t
+cpu_is_hyperthreaded(void)
 {
-	int		my_cpu = get_cpu_number();
-	int		my_core_base_cpu;
-	int		ret;
-	cpu_core_t	*my_core;
+	if  (cpuid_features() & CPUID_FEATURE_HTT)
+		return (cpuid_info()->cpuid_logical_per_package /
+			cpuid_info()->cpuid_cores_per_package) > 1;
+	else
+		return FALSE;
+}
 
-	/* Have we initialized already for this cpu? */
-	if (cpu_core())
-		return;
+void *
+cpu_thread_alloc(int cpu)
+{
+	int		core_base_cpu;
+	int		ret;
+	cpu_core_t	*core;
 
-	if (cpuid_features() & CPUID_FEATURE_HTT) {
+	/*
+ 	 * Assume that all cpus have the same features.
+	 */
+	if (cpu_is_hyperthreaded()) {
 		/*
 		 * Get the cpu number of the base thread in the core.
 		 */
-		my_core_base_cpu = cpu_to_core_cpu(my_cpu);
-		current_cpu_datap()->cpu_threadtype = CPU_THREADTYPE_INTEL_HTT;
+		core_base_cpu = cpu_to_core_cpu(cpu);
+		cpu_datap(cpu)->cpu_threadtype = CPU_THREADTYPE_INTEL_HTT;
 	} else {
-		my_core_base_cpu = my_cpu;
-		current_cpu_datap()->cpu_threadtype = CPU_THREADTYPE_NONE;
+		core_base_cpu = cpu;
+		cpu_datap(cpu)->cpu_threadtype = CPU_THREADTYPE_NONE;
 	}
 
-	/*
-	 * Allocate the base cpu_core struct if none exists.
-	 * Since we could be racing with other threads in the same core,
-	 * this needs care without using locks. We allocate a new core
-	 * structure and assign it atomically, freeing it if we lost the race.
-	 */
-	my_core = (cpu_core_t *) cpu_to_core(my_core_base_cpu);
-	if (my_core == NULL) {
-		cpu_core_t	*new_core;
-
+	core = (cpu_core_t *) cpu_to_core(core_base_cpu);
+	if (core == NULL) {
 		ret = kmem_alloc(kernel_map,
-				 (void *) &new_core, sizeof(cpu_core_t));
+				 (void *) &core, sizeof(cpu_core_t));
 		if (ret != KERN_SUCCESS)
-			panic("cpu_thread_init() kmem_alloc ret=%d\n", ret);
-		bzero((void *) new_core, sizeof(cpu_core_t));
-		new_core->base_cpu = my_core_base_cpu;
-		if (atomic_cmpxchg((uint32_t *) &cpu_to_core(my_core_base_cpu),
-				    0, (uint32_t) new_core)) {
-			atomic_incl((long *) &machine_info.physical_cpu, 1);
-			atomic_incl((long *) &machine_info.physical_cpu_max, 1);
-		} else {
-			kmem_free(kernel_map,
-				  (vm_offset_t)new_core, sizeof(cpu_core_t));
-		}
-		my_core = (cpu_core_t *) cpu_to_core(my_core_base_cpu);
+			panic("cpu_thread_alloc() kmem_alloc ret=%d\n", ret);
+		bzero((void *) core, sizeof(cpu_core_t));
+
+		core->base_cpu = core_base_cpu;
+
+		atomic_incl((long *) &machine_info.physical_cpu_max, 1);
+
+		/* Allocate performance counter data area (if available) */
+		core->pmc = pmc_alloc();
 	}
+	atomic_incl((long *) &machine_info.logical_cpu_max, 1);
 
-	cpu_to_core(my_cpu) = (struct cpu_core *) my_core;
+	return (void *) core;
+}
+
+void
+cpu_thread_init(void)
+{
+	int		my_cpu = get_cpu_number();
+	cpu_core_t	*my_core;
+
+	/*
+	 * If we're the boot processor we allocate the core structure here.
+	 * Otherwise the core has already been allocated (by the boot cpu).
+	 */
+	if (my_cpu == master_cpu)
+		cpu_to_core(master_cpu) = cpu_thread_alloc(master_cpu);
+
+	my_core = cpu_core();
+	if (my_core == NULL)
+		panic("cpu_thread_init() no core allocated for cpu %d", my_cpu);
 
 	atomic_incl((long *) &my_core->active_threads, 1);
-	atomic_incl((long *) &my_core->num_threads, 1);
 	atomic_incl((long *) &machine_info.logical_cpu, 1);
-	atomic_incl((long *) &machine_info.logical_cpu_max, 1);
-
+	/* Note: cpus are started serially so this isn't as racey as it looks */
+	if (my_core->num_threads == 0)
+		atomic_incl((long *) &machine_info.physical_cpu, 1);
+	atomic_incl((long *) &my_core->num_threads, 1);
 }
 
 /*
@@ -107,10 +126,10 @@ cpu_thread_halt(void)
 {
 	cpu_core_t	*my_core = cpu_core();
 
-	/* Note: don't ever decrement the number of physical processors */
-	atomic_decl((long *) &my_core->active_threads, 1);
-	atomic_decl((long *) &my_core->num_threads, 1);
 	atomic_decl((long *) &machine_info.logical_cpu, 1);
+	atomic_decl((long *) &my_core->active_threads, 1);
+	if (atomic_decl_and_test((long *) &my_core->num_threads, 1))
+		atomic_decl((long *) &machine_info.physical_cpu, 1);
 
 	cpu_halt();
 }
diff --git a/osfmk/i386/cpu_threads.h b/osfmk/i386/cpu_threads.h
index 770e11693..d8da7cd30 100644
--- a/osfmk/i386/cpu_threads.h
+++ b/osfmk/i386/cpu_threads.h
@@ -53,10 +53,10 @@ typedef struct {
 
 #define cpu_is_same_core(cpu1,cpu2)	(cpu_to_core(cpu1) == cpu_to_core(cpu2))
 
+extern void *cpu_thread_alloc(int);
 extern void cpu_thread_init(void);
 extern void cpu_thread_halt(void);
 
 extern int idlehalt;
 
-extern int ncore;
 #endif /* _I386_CPU_THREADS_H_ */
diff --git a/osfmk/i386/cpuid.c b/osfmk/i386/cpuid.c
index d4879d3a3..3fe712adb 100644
--- a/osfmk/i386/cpuid.c
+++ b/osfmk/i386/cpuid.c
@@ -28,12 +28,28 @@
 /*
  * @OSF_COPYRIGHT@
  */
-
+#include <platforms.h>
+#include <mach_kdb.h>
 #include <pexpert/pexpert.h>
 
 #include "cpuid.h"
+#if MACH_KDB
+#include <i386/db_machdep.h>
+#include <ddb/db_aout.h>
+#include <ddb/db_access.h>
+#include <ddb/db_sym.h>
+#include <ddb/db_variables.h>
+#include <ddb/db_command.h>
+#include <ddb/db_output.h>
+#include <ddb/db_expr.h>
+#endif
 
 #define min(a,b) ((a) < (b) ? (a) : (b))
+#define quad(hi,lo)	(((uint64_t)(hi)) << 32 | (lo))
+
+#define bit(n)		(1UL << (n))
+#define bitmask(h,l)	((bit(h)|(bit(h)-1)) & ~(bit(l)-1))
+#define bitfield(x,h,l)	(((x) & bitmask(h,l)) >> l)
 
 /*
  * CPU identification routines.
@@ -44,6 +60,7 @@
 
 static unsigned int	cpuid_maxcpuid;
 
+static i386_cpu_info_t	*cpuid_cpu_infop = NULL;
 static i386_cpu_info_t	cpuid_cpu_info;
 
 uint32_t		cpuid_feature;		/* XXX obsolescent for compat */
@@ -78,10 +95,10 @@ cpuid_get_info(i386_cpu_info_t *info_p)
 
 	/* do cpuid 0 to get vendor */
 	do_cpuid(0, cpuid_result);
-	cpuid_maxcpuid = cpuid_result[0];
-	bcopy((char *)&cpuid_result[1], &info_p->cpuid_vendor[0], 4); /* ugh */
-	bcopy((char *)&cpuid_result[2], &info_p->cpuid_vendor[8], 4);
-	bcopy((char *)&cpuid_result[3], &info_p->cpuid_vendor[4], 4);
+	cpuid_maxcpuid = cpuid_result[eax];
+	bcopy((char *)&cpuid_result[ebx], &info_p->cpuid_vendor[0], 4); /* ug */
+	bcopy((char *)&cpuid_result[ecx], &info_p->cpuid_vendor[8], 4);
+	bcopy((char *)&cpuid_result[edx], &info_p->cpuid_vendor[4], 4);
 	info_p->cpuid_vendor[12] = 0;
 
 	/* look up vendor */
@@ -161,6 +178,8 @@ CACHE_DESC(CPUID_CACHE_UCACHE_1M,	L2U,	1*1024*1024,	32, \
 	"Unified L2 cache, 1M, 4-way set associative, 32byte line size"),
 CACHE_DESC(CPUID_CACHE_UCACHE_2M,	L2U,	2*1024*1024,	32, \
 	"Unified L2 cache, 2M, 4-way set associative, 32byte line size"),
+CACHE_DESC(CPUID_CACHE_UCACHE_4M,	L2U,	4*1024*1024,	64, \
+	"Unified L2 cache, 4M, 16-way set associative, 64byte line size"),
 CACHE_DESC(CPUID_CACHE_UCACHE_128K_64,	L2U,	128*1024,	64, \
 	"Unified L2 cache, 128K, 8-way set associative, 64byte line size"),
 CACHE_DESC(CPUID_CACHE_UCACHE_256K_64,	L2U,	256*1024,	64, \
@@ -211,9 +230,12 @@ CACHE_DESC(CPUID_CACHE_NULL, Lnone, 0, 0, \
 	(char *)0),
 };
 
-static const char * get_intel_model_string( i386_cpu_info_t * info_p )
+static const char * get_intel_model_string( i386_cpu_info_t * info_p, cpu_type_t* type, cpu_subtype_t* subtype)
 {
-    /* check for brand id */
+	*type = CPU_TYPE_X86;
+	*subtype = CPU_SUBTYPE_X86_ARCH1;
+
+    /* check for brand id string */
     switch(info_p->cpuid_brand) {
         case CPUID_BRAND_UNSUPPORTED:
             /* brand ID not supported; use alternate method. */
@@ -242,14 +264,13 @@ static const char * get_intel_model_string( i386_cpu_info_t * info_p )
                         default:
                             return "Unknown Intel P6 Family";
                     }
-                case CPUID_FAMILY_ITANIUM:
-                    return "Intel Itanium";
                 case CPUID_FAMILY_EXTENDED:
                     switch (info_p->cpuid_extfamily) {
                         case CPUID_EXTFAMILY_PENTIUM4:
+			    *subtype = CPU_SUBTYPE_PENTIUM_4;
                             return "Intel Pentium 4";
-                        case CPUID_EXTFAMILY_ITANIUM2:
-                            return "Intel Itanium 2";
+						default:
+		                    return "Unknown Intel Extended Family";
                     }
                 default:
                     return "Unknown Intel Family";
@@ -263,10 +284,11 @@ static const char * get_intel_model_string( i386_cpu_info_t * info_p )
         case CPUID_BRAND_PENTIUM_III_4:
             return "Pentium III";
         case CPUID_BRAND_PIII_XEON:
-            if (info_p->cpuid_signature == 0x6B1)
-                return "Intel Celeron";
-            else
-                return "Intel Pentium III Xeon";
+			if (info_p->cpuid_signature == 0x6B1) {
+				return "Intel Celeron";
+			} else {
+				return "Intel Pentium III Xeon";
+			}
         case CPUID_BRAND_PENTIUM_III_M:
             return "Mobile Intel Pentium III-M";
         case CPUID_BRAND_M_CELERON_7:
@@ -276,16 +298,19 @@ static const char * get_intel_model_string( i386_cpu_info_t * info_p )
             return "Mobile Intel Celeron";
         case CPUID_BRAND_PENTIUM4_8:
         case CPUID_BRAND_PENTIUM4_9:
+	    *subtype = CPU_SUBTYPE_PENTIUM_4;
             return "Intel Pentium 4";
         case CPUID_BRAND_XEON:
             return "Intel Xeon";
         case CPUID_BRAND_XEON_MP:
             return "Intel Xeon MP";
         case CPUID_BRAND_PENTIUM4_M:
-            if (info_p->cpuid_signature == 0xF13)
-                return "Intel Xeon";
-            else
-                return "Mobile Intel Pentium 4";
+			if (info_p->cpuid_signature == 0xF13) {
+				return "Intel Xeon";
+			} else {
+				*subtype = CPU_SUBTYPE_PENTIUM_4;
+				return "Mobile Intel Pentium 4";
+			}
         case CPUID_BRAND_CELERON_M:
             return "Intel Celeron M";
         case CPUID_BRAND_PENTIUM_M:
@@ -294,7 +319,6 @@ static const char * get_intel_model_string( i386_cpu_info_t * info_p )
         case CPUID_BRAND_MOBILE_17:
             return "Mobile Intel";
     }        
-
     return "Unknown Intel";
 }
 
@@ -353,17 +377,69 @@ static void set_intel_cache_info( i386_cpu_info_t * info_p )
         /* If we have no L2 cache, use the L1 data cache line size */
         if (info_p->cache_size[L2U] == 0)
             info_p->cache_linesize = l1d_cache_linesize;
+
+	/*
+	 * Get cache sharing info if available.
+	 */
+	do_cpuid(0, cpuid_result);
+	if (cpuid_result[eax] >= 4) {
+		uint32_t	reg[4];
+		uint32_t	index;
+		for (index = 0;; index++) {
+			/*
+			 * Scan making calls for cpuid with %eax = 4
+			 * to get info about successive cache levels
+			 * until a null type is returned.
+			 */
+			cache_type_t	type = Lnone;
+			uint32_t	cache_type;
+			uint32_t	cache_level;
+			uint32_t	cache_sharing;
+
+			reg[eax] = 4;		/* cpuid request 4 */
+			reg[ecx] = index;	/* index starting at 0 */
+			cpuid(reg);
+//kprintf("cpuid(4) index=%d eax=%p\n", index, reg[eax]);
+			cache_type = bitfield(reg[eax], 4, 0);
+			if (cache_type == 0)
+				break;		/* done with cache info */
+			cache_level   = bitfield(reg[eax],  7,  5);
+			cache_sharing = bitfield(reg[eax], 25, 14);
+			info_p->cpuid_cores_per_package = 
+					bitfield(reg[eax], 31, 26) + 1;
+			switch (cache_level) {
+			case 1:
+				type = cache_type == 1 ? L1D :
+				       cache_type == 2 ? L1I :
+							 Lnone;
+				break;
+			case 2:
+				type = cache_type == 3 ? L2U :
+							 Lnone;
+				break;
+			case 3:
+				type = cache_type == 3 ? L3U :
+							 Lnone;
+			}
+			if (type != Lnone)
+				info_p->cache_sharing[type] = cache_sharing + 1;
+		} 
+	}
 }
 
 static void set_cpu_intel( i386_cpu_info_t * info_p )
 {
     set_cpu_generic(info_p);
     set_intel_cache_info(info_p);
-    info_p->cpuid_model_string = get_intel_model_string(info_p);
+    info_p->cpuid_model_string = get_intel_model_string(info_p, &info_p->cpuid_cpu_type, &info_p->cpuid_cpu_subtype);
 }
 
-static const char * get_amd_model_string( i386_cpu_info_t * info_p )
+static const char * get_amd_model_string( i386_cpu_info_t * info_p, cpu_type_t* type, cpu_subtype_t* subtype )
 {
+	*type = CPU_TYPE_X86;
+	*subtype = CPU_SUBTYPE_X86_ARCH1;
+
+    /* check for brand id string */
     switch (info_p->cpuid_family)
     {
         case CPUID_FAMILY_486:
@@ -444,11 +520,11 @@ static void set_amd_cache_info( i386_cpu_info_t * info_p )
     /* (ignore) */
     
     /* ECX: L1 Data Cache Information */
-    info_p->cache_size[L1D] = ((cpuid_result[2] >> 24) & 0xFF) * 1024;
-    info_p->cache_linesize = (cpuid_result[2] & 0xFF);
+    info_p->cache_size[L1D] = ((cpuid_result[ecx] >> 24) & 0xFF) * 1024;
+    info_p->cache_linesize = (cpuid_result[ecx] & 0xFF);
     
     /* EDX: L1 Instruction Cache Information */
-    info_p->cache_size[L1I] = ((cpuid_result[3] >> 24) & 0xFF) * 1024;
+    info_p->cache_size[L1I] = ((cpuid_result[edx] >> 24) & 0xFF) * 1024;
 
     /* L2 Cache Information */
     do_cpuid(0x80000006, cpuid_result);
@@ -460,16 +536,16 @@ static void set_amd_cache_info( i386_cpu_info_t * info_p )
     /* (ignore) */
     
     /* ECX: L2 Cache Information */
-    info_p->cache_size[L2U] = ((cpuid_result[2] >> 16) & 0xFFFF) * 1024;
+    info_p->cache_size[L2U] = ((cpuid_result[ecx] >> 16) & 0xFFFF) * 1024;
     if (info_p->cache_size[L2U] > 0)
-        info_p->cache_linesize = cpuid_result[2] & 0xFF;
+        info_p->cache_linesize = cpuid_result[ecx] & 0xFF;
 }
 
 static void set_cpu_amd( i386_cpu_info_t * info_p )
 {
     set_cpu_generic(info_p);
     set_amd_cache_info(info_p);
-    info_p->cpuid_model_string = get_amd_model_string(info_p);
+    info_p->cpuid_model_string = get_amd_model_string(info_p, &info_p->cpuid_cpu_type, &info_p->cpuid_cpu_subtype);
 }
 
 static void set_cpu_nsc( i386_cpu_info_t * info_p )
@@ -477,12 +553,16 @@ static void set_cpu_nsc( i386_cpu_info_t * info_p )
     set_cpu_generic(info_p);
     set_amd_cache_info(info_p);
 
-    if (info_p->cpuid_family == CPUID_FAMILY_586 && info_p->cpuid_model == CPUID_MODEL_GX1)
+    /* check for brand id string */
+    if (info_p->cpuid_family == CPUID_FAMILY_586 && info_p->cpuid_model == CPUID_MODEL_GX1) {
         info_p->cpuid_model_string = "AMD Geode GX1";
-    else if (info_p->cpuid_family == CPUID_FAMILY_586 && info_p->cpuid_model == CPUID_MODEL_GX2)
+    } else if (info_p->cpuid_family == CPUID_FAMILY_586 && info_p->cpuid_model == CPUID_MODEL_GX2) {
         info_p->cpuid_model_string = "AMD Geode GX";
-    else
+    } else {
         info_p->cpuid_model_string = "Unknown National Semiconductor";
+    }
+    info_p->cpuid_cpu_type = CPU_TYPE_X86;
+    info_p->cpuid_cpu_subtype = CPU_SUBTYPE_X86_ARCH1;
 }
 
 static void
@@ -494,7 +574,7 @@ set_cpu_generic(i386_cpu_info_t *info_p)
 
 	/* get extended cpuid results */
 	do_cpuid(0x80000000, cpuid_result);
-	max_extid = cpuid_result[0];
+	max_extid = cpuid_result[eax];
 
 	/* check to see if we can get brand string */
 	if (max_extid >= 0x80000004) {
@@ -526,15 +606,23 @@ set_cpu_generic(i386_cpu_info_t *info_p)
     
 	/* get processor signature and decode */
 	do_cpuid(1, cpuid_result);
-	info_p->cpuid_signature =  cpuid_result[0];
-	info_p->cpuid_stepping  =  cpuid_result[0]        & 0x0f;
-	info_p->cpuid_model     = (cpuid_result[0] >> 4)  & 0x0f;
-	info_p->cpuid_family    = (cpuid_result[0] >> 8)  & 0x0f;
-	info_p->cpuid_type      = (cpuid_result[0] >> 12) & 0x03;
-	info_p->cpuid_extmodel  = (cpuid_result[0] >> 16) & 0x0f;
-	info_p->cpuid_extfamily = (cpuid_result[0] >> 20) & 0xff;
-	info_p->cpuid_brand     =  cpuid_result[1]        & 0xff;
-	info_p->cpuid_features  =  cpuid_result[3];
+	info_p->cpuid_signature = cpuid_result[eax];
+	info_p->cpuid_stepping  = bitfield(cpuid_result[eax],  3,  0);
+	info_p->cpuid_model     = bitfield(cpuid_result[eax],  7,  4);
+	info_p->cpuid_family    = bitfield(cpuid_result[eax], 11,  8);
+	info_p->cpuid_type      = bitfield(cpuid_result[eax], 13, 12);
+	info_p->cpuid_extmodel  = bitfield(cpuid_result[eax], 19, 16);
+	info_p->cpuid_extfamily = bitfield(cpuid_result[eax], 27, 20);
+	info_p->cpuid_brand     = bitfield(cpuid_result[ebx],  7,  0);
+	info_p->cpuid_logical_per_package =
+				  bitfield(cpuid_result[ebx], 23, 16);
+	info_p->cpuid_features  = quad(cpuid_result[ecx], cpuid_result[edx]);
+
+	if (max_extid >= 0x80000001) {
+		do_cpuid(0x80000001, cpuid_result);
+		info_p->cpuid_extfeatures =
+				quad(cpuid_result[ecx], cpuid_result[edx]);
+	}
 
 	return;
 }
@@ -547,9 +635,9 @@ set_cpu_unknown(__unused i386_cpu_info_t *info_p)
 
 
 static struct {
-	uint32_t	mask;
+	uint64_t	mask;
 	const char	*name;
-} feature_names[] = {
+} feature_map[] = {
 	{CPUID_FEATURE_FPU,   "FPU",},
 	{CPUID_FEATURE_VME,   "VME",},
 	{CPUID_FEATURE_DE,    "DE",},
@@ -578,25 +666,76 @@ static struct {
 	{CPUID_FEATURE_SS,    "SS",},
 	{CPUID_FEATURE_HTT,   "HTT",},
 	{CPUID_FEATURE_TM,    "TM",},
+	{CPUID_FEATURE_SSE3,    "SSE3"},
+	{CPUID_FEATURE_MONITOR, "MON"},
+	{CPUID_FEATURE_DSCPL,   "DSCPL"},
+	{CPUID_FEATURE_VMX,     "VMX"},
+	{CPUID_FEATURE_SMX,     "SMX"},
+	{CPUID_FEATURE_EST,     "EST"},
+	{CPUID_FEATURE_TM2,     "TM2"},
+	{CPUID_FEATURE_MNI,     "MNI"},
+	{CPUID_FEATURE_CID,     "CID"},
+	{CPUID_FEATURE_CX16,    "CX16"},
+	{CPUID_FEATURE_xTPR,    "TPR"},
+	{0, 0}
+},
+extfeature_map[] = {
+	{CPUID_EXTFEATURE_SYSCALL, "SYSCALL"},
+	{CPUID_EXTFEATURE_XD,      "XD"},
+	{CPUID_EXTFEATURE_EM64T,   "EM64T"},
+	{CPUID_EXTFEATURE_LAHF,    "LAHF"},
 	{0, 0}
 };
 
+i386_cpu_info_t	*
+cpuid_info(void)
+{
+	/* Set-up the cpuid_indo stucture lazily */
+	if (cpuid_cpu_infop == NULL) {
+		cpuid_get_info(&cpuid_cpu_info);
+		cpuid_cpu_infop = &cpuid_cpu_info;
+	}
+	return cpuid_cpu_infop;
+}
+
 char *
-cpuid_get_feature_names(uint32_t feature, char *buf, unsigned buf_len)
+cpuid_get_feature_names(uint64_t features, char *buf, unsigned buf_len)
 {
+	int	len = -1;
+	char	*p = buf;
 	int	i;
-	int	len;
+
+	for (i = 0; feature_map[i].mask != 0; i++) {
+		if ((features & feature_map[i].mask) == 0)
+			continue;
+		if (len > 0)
+			*p++ = ' ';
+		len = min(strlen(feature_map[i].name), (buf_len-1) - (p-buf));
+		if (len == 0)
+			break;
+		bcopy(feature_map[i].name, p, len);
+		p += len;
+	}
+	*p = '\0';
+	return buf;
+}
+
+char *
+cpuid_get_extfeature_names(uint64_t extfeatures, char *buf, unsigned buf_len)
+{
+	int	len = -1;
 	char	*p = buf;
+	int	i;
 
-	for (i = 0; feature_names[i].mask != 0; i++) {
-		if ((feature & feature_names[i].mask) == 0)
+	for (i = 0; extfeature_map[i].mask != 0; i++) {
+		if ((extfeatures & extfeature_map[i].mask) == 0)
 			continue;
-		if (i > 0)
+		if (len > 0)
 			*p++ = ' ';
-		len = min(strlen(feature_names[i].name), (buf_len-1) - (p-buf));
+		len = min(strlen(extfeature_map[i].name), (buf_len-1)-(p-buf));
 		if (len == 0)
 			break;
-		bcopy(feature_names[i].name, p, len);
+		bcopy(extfeature_map[i].name, p, len);
 		p += len;
 	}
 	*p = '\0';
@@ -605,37 +744,69 @@ cpuid_get_feature_names(uint32_t feature, char *buf, unsigned buf_len)
 
 void
 cpuid_feature_display(
-	const char	*header,
-	__unused int	my_cpu)
+	const char	*header)
+{
+	char	buf[256];
+
+	kprintf("%s: %s\n", header,
+		  cpuid_get_feature_names(cpuid_features(),
+						buf, sizeof(buf)));
+	if (cpuid_features() & CPUID_FEATURE_HTT) {
+#define s_if_plural(n)	((n > 1) ? "s" : "")
+		kprintf("  HTT: %d core%s per package;"
+			     " %d logical cpu%s per package\n",
+			cpuid_cpu_info.cpuid_cores_per_package,
+			s_if_plural(cpuid_cpu_info.cpuid_cores_per_package),
+			cpuid_cpu_info.cpuid_logical_per_package,
+			s_if_plural(cpuid_cpu_info.cpuid_logical_per_package));
+	}
+}
+
+void
+cpuid_extfeature_display(
+	const char	*header)
 {
 	char	buf[256];
 
-	printf("%s: %s\n", header,
-		  cpuid_get_feature_names(cpuid_features(), buf, sizeof(buf)));
+	kprintf("%s: %s\n", header,
+		  cpuid_get_extfeature_names(cpuid_extfeatures(),
+						buf, sizeof(buf)));
 }
 
 void
 cpuid_cpu_display(
-	const char	*header,
-	__unused int	my_cpu)
+	const char	*header)
 {
-    if (cpuid_cpu_info.cpuid_brand_string[0] != '\0') {
-	printf("%s: %s\n", header,
-               cpuid_cpu_info.cpuid_brand_string);
+    if (cpuid_info()->cpuid_brand_string[0] != '\0') {
+	kprintf("%s: %s\n", header, cpuid_cpu_info.cpuid_brand_string);
     }
 }
 
 unsigned int
 cpuid_family(void)
 {
-	return cpuid_cpu_info.cpuid_family;
+	return cpuid_info()->cpuid_family;
 }
 
-unsigned int
+cpu_type_t
+cpuid_cputype(void)
+{
+	return cpuid_info()->cpuid_cpu_type;
+}
+
+cpu_subtype_t
+cpuid_cpusubtype(void)
+{
+	return cpuid_info()->cpuid_cpu_subtype;
+}
+
+uint64_t
 cpuid_features(void)
 {
 	static int checked = 0;
 	char	fpu_arg[16] = { 0 };
+
+	(void) cpuid_info();
 	if (!checked) {
 		    /* check for boot-time fpu limitations */
 			if (PE_parse_boot_arg("_fpu", &fpu_arg[0])) {
@@ -653,17 +824,59 @@ cpuid_features(void)
 	return cpuid_cpu_info.cpuid_features;
 }
 
-i386_cpu_info_t	*
-cpuid_info(void)
+uint64_t
+cpuid_extfeatures(void)
 {
-	return &cpuid_cpu_info;
+	return cpuid_info()->cpuid_extfeatures;
 }
-
-/* XXX for temporary compatibility */
+ 
 void
-set_cpu_model(void)
+cpuid_set_info(void)
 {
 	cpuid_get_info(&cpuid_cpu_info);
-	cpuid_feature = cpuid_cpu_info.cpuid_features;	/* XXX compat */
 }
 
+#if MACH_KDB
+
+/*
+ *	Display the cpuid
+ * *		
+ *	cp
+ */
+void 
+db_cpuid(__unused db_expr_t addr,
+	 __unused int have_addr,
+	 __unused db_expr_t count,
+	 __unused char *modif)
+{
+
+	uint32_t        i, mid;
+	uint32_t        cpid[4];
+
+	do_cpuid(0, cpid);	/* Get the first cpuid which is the number of
+				 * basic ids */
+	db_printf("%08X - %08X %08X %08X %08X\n",
+		0, cpid[eax], cpid[ebx], cpid[ecx], cpid[edx]);
+
+	mid = cpid[eax];	/* Set the number */
+	for (i = 1; i <= mid; i++) {	/* Dump 'em out */
+		do_cpuid(i, cpid);	/* Get the next */
+		db_printf("%08X - %08X %08X %08X %08X\n",
+			i, cpid[eax], cpid[ebx], cpid[ecx], cpid[edx]);
+	}
+	db_printf("\n");
+
+	do_cpuid(0x80000000, cpid);	/* Get the first extended cpuid which
+					 * is the number of extended ids */
+	db_printf("%08X - %08X %08X %08X %08X\n",
+		0x80000000, cpid[eax], cpid[ebx], cpid[ecx], cpid[edx]);
+
+	mid = cpid[eax];	/* Set the number */
+	for (i = 0x80000001; i <= mid; i++) {	/* Dump 'em out */
+		do_cpuid(i, cpid);	/* Get the next */
+		db_printf("%08X - %08X %08X %08X %08X\n",
+			i, cpid[eax], cpid[ebx], cpid[ecx], cpid[edx]);
+	}
+}
+
+#endif
diff --git a/osfmk/i386/cpuid.h b/osfmk/i386/cpuid.h
index 736babb21..702d66afb 100644
--- a/osfmk/i386/cpuid.h
+++ b/osfmk/i386/cpuid.h
@@ -55,34 +55,66 @@
 
 #define CPUID_STRING_UNKNOWN    "Unknown CPU Typ"
 
-#define	CPUID_FEATURE_FPU    0x00000001	/* Floating point unit on-chip */
-#define	CPUID_FEATURE_VME    0x00000002	/* Virtual Mode Extension */
-#define	CPUID_FEATURE_DE     0x00000004	/* Debugging Extension */
-#define	CPUID_FEATURE_PSE    0x00000008	/* Page Size Extension */
-#define	CPUID_FEATURE_TSC    0x00000010	/* Time Stamp Counter */
-#define	CPUID_FEATURE_MSR    0x00000020	/* Model Specific Registers */
-#define CPUID_FEATURE_PAE    0x00000040 /* Physical Address Extension */
-#define	CPUID_FEATURE_MCE    0x00000080	/* Machine Check Exception */
-#define	CPUID_FEATURE_CX8    0x00000100	/* CMPXCHG8B */
-#define	CPUID_FEATURE_APIC   0x00000200	/* On-chip APIC */
-#define CPUID_FEATURE_SEP    0x00000800 /* Fast System Call */
-#define	CPUID_FEATURE_MTRR   0x00001000	/* Memory Type Range Register */
-#define	CPUID_FEATURE_PGE    0x00002000	/* Page Global Enable */
-#define	CPUID_FEATURE_MCA    0x00004000	/* Machine Check Architecture */
-#define	CPUID_FEATURE_CMOV   0x00008000	/* Conditional Move Instruction */
-#define CPUID_FEATURE_PAT    0x00010000 /* Page Attribute Table */
-#define CPUID_FEATURE_PSE36  0x00020000 /* 36-bit Page Size Extension */
-#define CPUID_FEATURE_PSN    0x00040000 /* Processor Serial Number */
-#define CPUID_FEATURE_CLFSH  0x00080000 /* CLFLUSH Instruction supported */
-#define CPUID_FEATURE_DS     0x00200000 /* Debug Store */
-#define CPUID_FEATURE_ACPI   0x00400000 /* Thermal Monitor, SW-controlled clock */
-#define CPUID_FEATURE_MMX    0x00800000 /* MMX supported */
-#define CPUID_FEATURE_FXSR   0x01000000 /* Fast floating point save/restore */
-#define CPUID_FEATURE_SSE    0x02000000 /* Streaming SIMD extensions */
-#define CPUID_FEATURE_SSE2   0x04000000 /* Streaming SIMD extensions 2 */
-#define CPUID_FEATURE_SS     0x08000000 /* Self-Snoop */
-#define CPUID_FEATURE_HTT    0x10000000 /* Hyper-Threading Technology */
-#define CPUID_FEATURE_TM     0x20000000 /* Thermal Monitor */
+#define _Bit(n)			(1ULL << n)
+#define _HBit(n)		(1ULL << ((n)+32))
+
+/*
+ * The CPUID_FEATURE_XXX values define 64-bit values
+ * returned in %ecx:%edx to a CPUID request with %eax of 1: 
+ */
+#define	CPUID_FEATURE_FPU     _Bit(0)	/* Floating point unit on-chip */
+#define	CPUID_FEATURE_VME     _Bit(1)	/* Virtual Mode Extension */
+#define	CPUID_FEATURE_DE      _Bit(2)	/* Debugging Extension */
+#define	CPUID_FEATURE_PSE     _Bit(3)	/* Page Size Extension */
+#define	CPUID_FEATURE_TSC     _Bit(4)	/* Time Stamp Counter */
+#define	CPUID_FEATURE_MSR     _Bit(5)	/* Model Specific Registers */
+#define CPUID_FEATURE_PAE     _Bit(6)	/* Physical Address Extension */
+#define	CPUID_FEATURE_MCE     _Bit(7)	/* Machine Check Exception */
+#define	CPUID_FEATURE_CX8     _Bit(8)	/* CMPXCHG8B */
+#define	CPUID_FEATURE_APIC    _Bit(9)	/* On-chip APIC */
+#define CPUID_FEATURE_SEP     _Bit(11)	/* Fast System Call */
+#define	CPUID_FEATURE_MTRR    _Bit(12)	/* Memory Type Range Register */
+#define	CPUID_FEATURE_PGE     _Bit(13)	/* Page Global Enable */
+#define	CPUID_FEATURE_MCA     _Bit(14)	/* Machine Check Architecture */
+#define	CPUID_FEATURE_CMOV    _Bit(15)	/* Conditional Move Instruction */
+#define CPUID_FEATURE_PAT     _Bit(16)	/* Page Attribute Table */
+#define CPUID_FEATURE_PSE36   _Bit(17)	/* 36-bit Page Size Extension */
+#define CPUID_FEATURE_PSN     _Bit(18)	/* Processor Serial Number */
+#define CPUID_FEATURE_CLFSH   _Bit(19)	/* CLFLUSH Instruction supported */
+#define CPUID_FEATURE_DS      _Bit(21)	/* Debug Store */
+#define CPUID_FEATURE_ACPI    _Bit(22)	/* Thermal monitor and Clock Ctrl */
+#define CPUID_FEATURE_MMX     _Bit(23)	/* MMX supported */
+#define CPUID_FEATURE_FXSR    _Bit(24)	/* Fast floating pt save/restore */
+#define CPUID_FEATURE_SSE     _Bit(25)	/* Streaming SIMD extensions */
+#define CPUID_FEATURE_SSE2    _Bit(26)	/* Streaming SIMD extensions 2 */
+#define CPUID_FEATURE_SS      _Bit(27)	/* Self-Snoop */
+#define CPUID_FEATURE_HTT     _Bit(28)	/* Hyper-Threading Technology */
+#define CPUID_FEATURE_TM      _Bit(29)	/* Thermal Monitor (TM1) */
+#define CPUID_FEATURE_PBE     _Bit(31)	/* Pend Break Enable */
+
+#define CPUID_FEATURE_SSE3    _HBit(0)	/* Prescott New Inst. */
+#define CPUID_FEATURE_MONITOR _HBit(3)	/* Monitor/mwait */
+#define CPUID_FEATURE_DSCPL   _HBit(4)	/* Debug Store CPL */
+#define CPUID_FEATURE_VMX     _HBit(5)	/* VMX */
+#define CPUID_FEATURE_SMX     _HBit(6)	/* SMX */
+#define CPUID_FEATURE_EST     _HBit(7)	/* Enhanced SpeedsTep (GV3) */
+#define CPUID_FEATURE_TM2     _HBit(8)	/* Thermal Monitor 2 */
+#define CPUID_FEATURE_SSSE3   _HBit(9)	/* Supplemental SSE3 instructions */
+#define CPUID_FEATURE_MNI     CPUID_FEATURE_SSSE3
+#define CPUID_FEATURE_CID     _HBit(10)	/* L1 Context ID */
+#define CPUID_FEATURE_CX16    _HBit(13)	/* CmpXchg16b instruction */
+#define CPUID_FEATURE_xTPR    _HBit(14)	/* Send Task PRiority msgs */
+
+/*
+ * The CPUID_EXTFEATURE_XXX values define 64-bit values
+ * returned in %ecx:%edx to a CPUID request with %eax of 0x80000001: 
+ */
+#define CPUID_EXTFEATURE_SYSCALL   _Bit(11)	/* SYSCALL/sysret */
+#define CPUID_EXTFEATURE_XD	   _Bit(20)	/* eXecute Disable */
+#define CPUID_EXTFEATURE_EM64T	   _Bit(29)	/* Extended Mem 64 Technology */
+
+#define CPUID_EXTFEATURE_LAHF	   _HBit(20)	/* LAFH/SAHF instructions */
+
 
 #define	CPUID_TYPE_OEM		    0x0	/* Original processor */
 #define	CPUID_TYPE_OVERDRIVE	    0x1	/* Overdrive processor */
@@ -152,6 +184,8 @@
 #define CPUID_MODEL_P6A             0xA /* Intel PIII Xeon model A */
 #define CPUID_MODEL_P6B             0xB /* Intel PIII model B */
 #define CPUID_MODEL_PMD             0xD /* Intel Pentium M model D */
+#define CPUID_MODEL_CORE            0xE /* Intel Core Solo & Duo */
+#define CPUID_MODEL_CORE2	    0xF /* Intel Core2 Duo */
 
 #define CPUID_MODEL_ATHLON_M1       0x1 /* AMD Athlon Model 1 */
 #define CPUID_MODEL_ATHLON_M2       0x2 /* AMD Athlon Model 2 */
@@ -243,6 +277,7 @@
 #define	CPUID_CACHE_UCACHE_512K	   0x43	/* 2nd-level cache, 512K */
 #define	CPUID_CACHE_UCACHE_1M	   0x44	/* 2nd-level cache, 1M */
 #define	CPUID_CACHE_UCACHE_2M	   0x45	/* 2nd-level cache, 2M */
+#define CPUID_CACHE_UCACHE_4M      0x49 /* 2nd-level cache, 4M */
 #define CPUID_CACHE_ITLB_64        0x50 /* Instruction TLB, 64 entries */
 #define CPUID_CACHE_ITLB_128       0x51 /* Instruction TLB, 128 entries */
 #define CPUID_CACHE_ITLB_256       0x52 /* Instruction TLB, 256 entries */
@@ -281,6 +316,20 @@
 #include <mach/machine.h>
 
 
+typedef enum { eax, ebx, ecx, edx } cpuid_register_t;
+static inline void
+cpuid(uint32_t *data)
+{
+	asm("cpuid"
+		: "=a" (data[eax]),
+		  "=b" (data[ebx]),
+		  "=c" (data[ecx]),
+		  "=d" (data[edx])
+		: "a"  (data[eax]),
+		  "b"  (data[ebx]),
+		  "c"  (data[ecx]),
+		  "d"  (data[edx]));
+}
 static inline void
 do_cpuid(uint32_t selector, uint32_t *data)
 {
@@ -315,20 +364,20 @@ typedef struct {
 	{ value, type, size, linesize }
 #endif /* KERNEL */
 
-/* Physical CPU info */
+/* Physical CPU info - this is exported out of the kernel (kexts), so be wary of changes */
 typedef struct {
 	char		cpuid_vendor[16];
 	char		cpuid_brand_string[48];
 	const char	*cpuid_model_string;
 
-	uint32_t	cpuid_value;
-	cpu_type_t	cpuid_type;
+	cpu_type_t	cpuid_type;					/* this is *not* a cpu_type_t in our <mach/machine.h> */
 	uint8_t		cpuid_family;
 	uint8_t		cpuid_model;
 	uint8_t		cpuid_extmodel;
 	uint8_t		cpuid_extfamily;
 	uint8_t		cpuid_stepping;
-	uint32_t	cpuid_features;
+	uint64_t	cpuid_features;
+	uint64_t	cpuid_extfeatures;
 	uint32_t	cpuid_signature;
 	uint8_t   	cpuid_brand; 
 	
@@ -337,6 +386,12 @@ typedef struct {
 
 	uint8_t		cache_info[64];    /* list of cache descriptors */
 
+	uint32_t	cpuid_cores_per_package;
+	uint32_t	cpuid_logical_per_package;
+	uint32_t	cache_sharing[LCACHE_MAX];
+
+	cpu_type_t		cpuid_cpu_type;			/* <mach/machine.h> */
+	cpu_subtype_t	cpuid_cpu_subtype;		/* <mach/machine.h> */	
 } i386_cpu_info_t;
 
 #ifdef __cplusplus
@@ -346,20 +401,22 @@ extern "C" {
 /*
  * External declarations
  */
-extern cpu_type_t	cpuid_cputype(int);
-extern void		cpuid_cpu_display(const char *, __unused int);
-extern void		cpuid_feature_display(const char *, __unused int);
-extern char *		cpuid_get_feature_names(uint32_t, char *, unsigned);
-
-extern uint32_t		cpuid_features(void);
+extern cpu_type_t	cpuid_cputype(void);
+extern cpu_subtype_t	cpuid_cpusubtype(void);
+extern void		cpuid_cpu_display(const char *);
+extern void		cpuid_feature_display(const char *);
+extern void		cpuid_extfeature_display(const char *);
+extern char *		cpuid_get_feature_names(uint64_t, char *, unsigned);
+extern char *		cpuid_get_extfeature_names(uint64_t, char *, unsigned);
+
+extern uint64_t		cpuid_features(void);
+extern uint64_t		cpuid_extfeatures(void);
 extern uint32_t		cpuid_family(void);
 	
 extern void		cpuid_get_info(i386_cpu_info_t *info_p);
 extern i386_cpu_info_t	*cpuid_info(void);
 
-/* XXX obsolescent: */
-extern uint32_t		cpuid_feature;
-extern void		set_cpu_model(void);
+extern void		cpuid_set_info(void);
 
 #ifdef __cplusplus
 }
diff --git a/osfmk/i386/cswitch.s b/osfmk/i386/cswitch.s
index c61d9ad36..ca3cc863e 100644
--- a/osfmk/i386/cswitch.s
+++ b/osfmk/i386/cswitch.s
@@ -72,6 +72,7 @@
 
 #define	CX(addr, reg)	addr(,reg,4)
 
+	.text
 /*
  * Context switch routines for i386.
  */
@@ -92,23 +93,27 @@ Entry(Load_context)
 	call	EXT(thread_continue)
 
 /*
- *	This really only has to save registers
+ *	This has to save registers only
  *	when there is no explicit continuation.
  */
 
 Entry(Switch_context)
-	movl	%gs:CPU_ACTIVE_STACK,%ecx /* get old kernel stack */
+	popl	%eax				/* pop return PC */
 
+	/* Test for a continuation and skip all state saving if so... */
+	cmpl	$0,4(%esp)
+	jne 	5f
+	movl	%gs:CPU_ACTIVE_STACK,%ecx	/* get old kernel stack */
 	movl	%ebx,KSS_EBX(%ecx)		/* save registers */
 	movl	%ebp,KSS_EBP(%ecx)
 	movl	%edi,KSS_EDI(%ecx)
 	movl	%esi,KSS_ESI(%ecx)
-	popl	KSS_EIP(%ecx)			/* save return PC */
+	movl	%eax,KSS_EIP(%ecx)		/* save return PC */
 	movl	%esp,KSS_ESP(%ecx)		/* save SP */
-
+5:
 	movl	0(%esp),%eax			/* return old thread */
 	movl	8(%esp),%ebx			/* get new thread */
-	movl    %ebx,%gs:CPU_ACTIVE_THREAD                /* new thread is active */
+	movl    %ebx,%gs:CPU_ACTIVE_THREAD      /* new thread is active */
 	movl	TH_KERNEL_STACK(%ebx),%ecx	/* get its kernel stack */
 	lea	KERNEL_STACK_SIZE-IKS_SIZE-IEL_SIZE(%ecx),%ebx
 						/* point to stack top */
@@ -116,9 +121,7 @@ Entry(Switch_context)
 	movl	%ecx,%gs:CPU_ACTIVE_STACK	/* set current stack */
 	movl	%ebx,%gs:CPU_KERNEL_STACK	/* set stack top */
 
-
-	movl	$0,%gs:CPU_ACTIVE_KLOADED
-
+	
 	movl	KSS_ESP(%ecx),%esp		/* switch stacks */
 	movl	KSS_ESI(%ecx),%esi		/* restore registers */
 	movl	KSS_EDI(%ecx),%edi
@@ -132,19 +135,17 @@ Entry(Thread_continue)
 	call	*%ebx				/* call real continuation */
 
 /*
- * void machine_processor_shutdown(thread_t thread,
- *				   void (*routine)(processor_t),
- *				   processor_t processor)
+ * thread_t Shutdown_context(thread_t thread,
+ *			 void (*routine)(processor_t),
+ *			 processor_t processor)
  *
  * saves the kernel context of the thread,
  * switches to the interrupt stack,
  * continues the thread (with thread_continue),
  * then runs routine on the interrupt stack.
  *
- * Assumes that the thread is a kernel thread (thus
- * has no FPU state)
  */
-Entry(machine_processor_shutdown)
+Entry(Shutdown_context)
 	movl	%gs:CPU_ACTIVE_STACK,%ecx /* get old kernel stack */
 	movl	%ebx,KSS_EBX(%ecx)		/* save registers */
 	movl	%ebp,KSS_EBP(%ecx)
@@ -163,10 +164,3 @@ Entry(machine_processor_shutdown)
 	pushl	%esi				/* push argument */
 	call	*%ebx				/* call routine to run */
 	hlt					/* (should never return) */
-
-
-        .text
-
-	.globl	EXT(locore_end)
-LEXT(locore_end)
-
diff --git a/osfmk/i386/db_interface.c b/osfmk/i386/db_interface.c
index 5ef734499..5601baab2 100644
--- a/osfmk/i386/db_interface.c
+++ b/osfmk/i386/db_interface.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -76,6 +76,8 @@
 #include <i386/setjmp.h>
 #include <i386/pmap.h>
 #include <i386/misc_protos.h>
+#include <i386/mp.h>
+#include <i386/machine_cpu.h>
 
 #include <mach/vm_param.h>
 #include <vm/vm_map.h>
@@ -92,9 +94,11 @@
 #include <ddb/db_break.h>
 #include <ddb/db_watch.h>
 
+#include <i386/cpu_data.h>
+
 int	 db_active = 0;
-struct	 i386_saved_state *i386_last_saved_statep;
-struct	 i386_saved_state i386_nested_saved_state;
+x86_saved_state32_t	*i386_last_saved_statep;
+x86_saved_state32_t	i386_nested_saved_state;
 unsigned i386_last_kdb_sp;
 
 extern	thread_t db_default_act;
@@ -117,7 +121,7 @@ struct int_regs {
 	int	esi;
 	int	ebp;
 	int	ebx;
-	struct i386_interrupt_state *is;
+	x86_saved_state32_t *is;
 };
 
 extern char *	trap_type[];
@@ -166,10 +170,29 @@ extern jmp_buf_t *db_recover;
  * in a ktss, we hard-wire that in, rather than indexing the gdt
  * with tss_sel to derive a pointer to the desired tss.
  */
+
+/*
+ * Code used to synchronize kdb among all cpus, one active at a time, switch
+ * from one to another using cpu #cpu
+ */
+
+decl_simple_lock_data(, kdb_lock)	/* kdb lock			*/
+
+#define	db_simple_lock_init(l, e)	hw_lock_init(&((l)->interlock))
+#define	db_simple_lock_try(l)		hw_lock_try(&((l)->interlock))
+#define	db_simple_unlock(l)		hw_lock_unlock(&((l)->interlock))
+
+int			kdb_cpu = -1;	/* current cpu running kdb	*/
+int			kdb_debug = 1;
+volatile unsigned int	cpus_holding_bkpts;	/* counter for number of cpus
+						 * holding breakpoints
+						 */
+extern boolean_t	db_breakpoints_inserted;
+
 void
 db_tss_to_frame(
 	int tss_sel,
-	struct i386_saved_state *regs)
+	x86_saved_state32_t *regs)
 {
 	extern struct i386_tss ktss;
 	int mycpu = cpu_number();
@@ -180,7 +203,7 @@ db_tss_to_frame(
 	/*
 	 * ddb will overwrite whatever's in esp, so put esp0 elsewhere, too.
 	 */
-	regs->esp = tss->esp0;
+	regs->cr2 = tss->esp0;
 	regs->efl = tss->eflags;
 	regs->eip = tss->eip;
 	regs->trapno = tss->ss0;	/* XXX */
@@ -207,7 +230,7 @@ db_tss_to_frame(
  */
 boolean_t
 db_trap_from_asm(
-	struct i386_saved_state *regs)
+	x86_saved_state32_t *regs)
 {
 	int	code;
 	int	type;
@@ -221,12 +244,18 @@ int
 kdb_trap(
 	int			type,
 	int			code,
-	struct i386_saved_state *regs)
+	x86_saved_state32_t	*regs)
 {
 	extern char 		etext;
 	boolean_t		trap_from_user;
-	spl_t			s = splhigh();
+	spl_t			s;
+	int                     previous_console_device;
 
+	s = splhigh();
+
+	previous_console_device = switch_to_serial_console();
+
+	db_printf("kdb_trap(): type %d, code %d, regs->eip 0x%x\n", type, code, regs->eip);
 	switch (type) {
 	    case T_DEBUG:	/* single_step */
 	    {
@@ -280,8 +309,8 @@ kdb_trap(
 
 	if (!IS_USER_TRAP(regs, &etext)) {
 		bzero((char *)&ddb_regs, sizeof (ddb_regs));
-		*(struct i386_saved_state_from_kernel *)&ddb_regs =
-			*(struct i386_saved_state_from_kernel *)regs;
+		*(struct x86_saved_state32_from_kernel *)&ddb_regs =
+			*(struct x86_saved_state32_from_kernel *)regs;
 		trap_from_user = FALSE;
 	}
 	else {
@@ -306,6 +335,7 @@ kdb_trap(
 	regs->ecx    = ddb_regs.ecx;
 	regs->edx    = ddb_regs.edx;
 	regs->ebx    = ddb_regs.ebx;
+
 	if (trap_from_user) {
 	    /*
 	     * user mode - saved esp and ss valid
@@ -313,6 +343,7 @@ kdb_trap(
 	    regs->uesp = ddb_regs.uesp;		/* user stack pointer */
 	    regs->ss   = ddb_regs.ss & 0xffff;	/* user stack segment */
 	}
+
 	regs->ebp    = ddb_regs.ebp;
 	regs->esi    = ddb_regs.esi;
 	regs->edi    = ddb_regs.edi;
@@ -330,16 +361,13 @@ kdb_trap(
 					       trap_from_user))
 	                      == BKPT_INST))
 	    regs->eip += BKPT_SIZE;
-
+	
+	switch_to_old_console(previous_console_device);
 kdb_exit:
 	kdb_leave();
 
 	current_cpu_datap()->cpu_kdb_saved_state = 0;
 
-#if	MACH_MP_DEBUG
-	current_cpu_datap()->cpu_masked_state_cnt = 0;
-#endif	/* MACH_MP_DEBUG */
-
 	enable_preemption();
 
 	splx(s);
@@ -369,8 +397,8 @@ kdb_kentry(
 {
 	extern char etext;
 	boolean_t trap_from_user;
-	struct i386_interrupt_state *is = int_regs->is;
-	struct i386_saved_state regs;
+	x86_saved_state32_t *is = int_regs->is;
+	x86_saved_state32_t regs;
 	spl_t s;
 
 	s = splhigh();
@@ -471,18 +499,28 @@ db_user_to_kernel_address(
 	int		flag)
 {
 	register pt_entry_t *ptp;
+	vm_offset_t src;
+
+	/*
+	 * must not pre-empted while using the pte pointer passed
+	 * back since it's been mapped through a per-cpu window
+	 */
+        mp_disable_preemption();
 	
-	ptp = pmap_pte(task->map->pmap, addr);
+	ptp = pmap_pte(task->map->pmap, (vm_map_offset_t)addr);
 	if (ptp == PT_ENTRY_NULL || (*ptp & INTEL_PTE_VALID) == 0) {
 	    if (flag) {
 		db_printf("\nno memory is assigned to address %08x\n", addr);
 		db_error(0);
 		/* NOTREACHED */
 	    }
+	    mp_enable_preemption();
 	    return(-1);
 	}
-
 	src = (vm_offset_t)pte_to_pa(*ptp);
+
+	mp_enable_preemption();
+
 	*(int *) DMAP1 = INTEL_PTE_VALID | INTEL_PTE_RW | (src & PG_FRAME) | 
 	  INTEL_PTE_REF | INTEL_PTE_MOD;
 #if defined(I386_CPU)
@@ -574,7 +612,7 @@ db_write_bytes(
 	if (addr >= VM_MIN_KERNEL_ADDRESS &&
 	    addr <= (vm_offset_t)&etext)
 	{
-	    ptep0 = pmap_pte(kernel_pmap, addr);
+	    ptep0 = pmap_pte(kernel_pmap, (vm_map_offset_t)addr);
 	    oldmap0 = *ptep0;
 	    *ptep0 |= INTEL_PTE_WRITE;
 
@@ -582,7 +620,7 @@ db_write_bytes(
 	    if (i386_trunc_page(addr) != addr1) {
 		/* data crosses a page boundary */
 
-		ptep1 = pmap_pte(kernel_pmap, addr1);
+		ptep1 = pmap_pte(kernel_pmap, (vm_map_offset_t)addr1);
 		oldmap1 = *ptep1;
 		*ptep1 |= INTEL_PTE_WRITE;
 	    }
@@ -760,24 +798,6 @@ db_task_name(
 	    db_printf(" ");
 }
 
-/*
- * Code used to synchronize kdb among all cpus, one active at a time, switch
- * from on to another using kdb_on! #cpu or cpu #cpu
- */
-
-decl_simple_lock_data(, kdb_lock)	/* kdb lock			*/
-
-#define	db_simple_lock_init(l, e)	hw_lock_init(&((l)->interlock))
-#define	db_simple_lock_try(l)		hw_lock_try(&((l)->interlock))
-#define	db_simple_unlock(l)		hw_lock_unlock(&((l)->interlock))
-
-int			kdb_cpu = -1;	/* current cpu running kdb	*/
-int			kdb_debug = 0;
-volatile unsigned int	cpus_holding_bkpts;	/* counter for number of cpus holding
-						   breakpoints (ie: cpus that did not
-						   insert back breakpoints) */
-extern boolean_t	db_breakpoints_inserted;
-
 void
 db_machdep_init(void)
 {
@@ -785,16 +805,14 @@ db_machdep_init(void)
 
 	db_simple_lock_init(&kdb_lock, 0);
 	for (c = 0; c < real_ncpus; ++c) {
-		db_stacks[c] = (vm_offset_t) (db_stack_store +
-			(INTSTACK_SIZE * (c + 1)) - sizeof (natural_t));
 		if (c == master_cpu) {
-			dbtss.esp0 = (int)(db_task_stack_store +
+			master_dbtss.esp0 = (int)(db_task_stack_store +
 				(INTSTACK_SIZE * (c + 1)) - sizeof (natural_t));
-			dbtss.esp = dbtss.esp0;
-			dbtss.eip = (int)&db_task_start;
+			master_dbtss.esp = master_dbtss.esp0;
+			master_dbtss.eip = (int)&db_task_start;
 			/*
 			 * The TSS for the debugging task on each slave CPU
-			 * is set up in mp_desc_init().
+			 * is set up in cpu_desc_init().
 			 */
 		}
 	}
@@ -811,12 +829,12 @@ db_machdep_init(void)
 int
 kdb_enter(int pc)
 {
-	int mycpu;
+	int my_cpu;
 	int retval;
 
 	disable_preemption();
 
-	mycpu = cpu_number();
+	my_cpu = cpu_number();
 
 	if (current_cpu_datap()->cpu_db_pass_thru) {
 		retval = 0;
@@ -824,16 +842,17 @@ kdb_enter(int pc)
 	}
 
 	current_cpu_datap()->cpu_kdb_active++;
+
 	lock_kdb();
 
-	if (kdb_debug)
-		db_printf("kdb_enter: cpu %d, is_slave %d, kdb_cpu %d, run mode %d pc %x (%x) holds %d\n",
-			  my_cpu, current_cpu_datap()->cpu_kdb_is_slave, kdb_cpu,
-			  db_run_mode, pc, *(int *)pc, cpus_holding_bkpts);
+	db_printf("kdb_enter(): cpu_number %d, kdb_cpu %d\n", my_cpu, kdb_cpu);
+	
 	if (db_breakpoints_inserted)
 		cpus_holding_bkpts++;
+
 	if (kdb_cpu == -1 && !current_cpu_datap()->cpu_kdb_is_slave) {
 		kdb_cpu = my_cpu;
+		db_printf("Signaling other processors..\n");
 		remote_kdb();	/* stop other cpus */
 		retval = 1;
 	} else if (kdb_cpu == my_cpu) 
@@ -874,6 +893,8 @@ kdb_leave(void)
 	unlock_kdb();
 	current_cpu_datap()->cpu_kdb_active--;
 
+	mp_kdb_exit();
+
 	enable_preemption();
 
 	if (wait) {
@@ -886,14 +907,12 @@ lock_kdb(void)
 {
 	int		my_cpu;
 	register	i;
-	extern void	kdb_console(void);
 
 	disable_preemption();
 
 	my_cpu = cpu_number();
 
 	for(;;) {
-		kdb_console();
 		if (kdb_cpu != -1 && kdb_cpu != my_cpu) {
 			continue;
 		}
@@ -988,21 +1007,17 @@ kdb_on(
 	}
 }
 
+/*
+ * system reboot
+ */
+
+extern void kdp_reboot(void);
+
 void db_reboot(
 	db_expr_t	addr,
 	boolean_t	have_addr,
 	db_expr_t	count,
 	char		*modif)
 {
-	boolean_t	reboot = TRUE;
-	char		*cp, c;
-	
-	cp = modif;
-	while ((c = *cp++) != 0) {
-		if (c == 'r')	/* reboot */
-			reboot = TRUE;
-		if (c == 'h')	/* halt */
-			reboot = FALSE;
-	}
-	halt_all_cpus(reboot);
+	kdp_reboot();
 }
diff --git a/osfmk/i386/db_machdep.h b/osfmk/i386/db_machdep.h
index ae10d5261..ac435e87c 100644
--- a/osfmk/i386/db_machdep.h
+++ b/osfmk/i386/db_machdep.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -69,11 +69,13 @@
 #include <i386/thread.h>		/* for thread_status */
 #include <i386/eflags.h>
 #include <i386/trap.h>
+#include <i386/pmCPU.h>
+#include <i386/hpet.h>
 
-typedef	vm_offset_t	db_addr_t;	/* address - unsigned */
-typedef	int		db_expr_t;	/* expression - signed */
+typedef	addr64_t	db_addr_t;	/* address - unsigned */
+typedef	uint64_t	db_expr_t;	/* expression */
 
-typedef struct i386_saved_state db_regs_t;
+typedef struct x86_saved_state32 db_regs_t;
 db_regs_t	ddb_regs;	/* register state */
 #define	DDB_REGS	(&ddb_regs)
 extern int	db_active;	/* ddb is active */
@@ -113,9 +115,7 @@ int db_inst_store(unsigned long);
 	db_check_access(addr,size,task)
 #define DB_PHYS_EQ(task1,addr1,task2,addr2)			\
 	db_phys_eq(task1,addr1,task2,addr2)
-#define DB_VALID_KERN_ADDR(addr)				\
-	((addr) >= VM_MIN_KERNEL_ADDRESS && 			\
-	 (addr) < VM_MAX_KERNEL_ADDRESS)
+#define DB_VALID_KERN_ADDR(addr)		(1)
 #define DB_VALID_ADDRESS(addr,user)				\
 	((!(user) && DB_VALID_KERN_ADDR(addr)) ||		\
 	 ((user) && (addr) < VM_MAX_ADDRESS))
@@ -160,6 +160,19 @@ extern void		db_reboot(
 				db_expr_t	count,
 				char		*modif);
 
+extern void db_display_kmod(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
+extern void db_display_real(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
+extern void db_display_iokit(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
+extern void db_cpuid(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
+extern void db_msr(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
+extern void db_apic(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
+extern void db_test(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
+extern void db_intcnt(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
+extern void db_display_hpet(hpetReg_t *hpt);
+extern void db_hpet(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
+extern void db_cfg(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
+extern void db_dtimers(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
+
 /* macros for printing OS server dependent task name */
 
 #define DB_TASK_NAME(task)	db_task_name(task)
@@ -172,21 +185,26 @@ extern void		db_task_name(
 
 /* macro for checking if a thread has used floating-point */
 
-#define db_act_fp_used(act)	(act && act->machine.pcb->ims.ifps)
+#define db_act_fp_used(act)	(act && act->machine.pcb->ifps)
 
 extern void		db_tss_to_frame(
 				int			tss_sel,
-				struct i386_saved_state	*regs);
+				x86_saved_state32_t	*regs);
 extern int		kdb_trap(
 				int			type,
 				int			code,
-				struct i386_saved_state	*regs);
+				x86_saved_state32_t	*regs);
 extern boolean_t	db_trap_from_asm(
-				struct i386_saved_state *regs);
+				x86_saved_state32_t *regs);
 extern int		dr6(void);
 extern void		kdb_on(
 				int			cpu);
-extern void		cnpollc(
-				boolean_t		on);
+
+#if MACH_KDB
+extern void db_getpmgr(pmData_t *pmj);
+extern void db_chkpmgr(void);
+#endif /* MACH_KDB */
+extern void db_pmgr(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
+extern void db_nap(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
 
 #endif	/* _I386_DB_MACHDEP_H_ */
diff --git a/osfmk/i386/db_trace.c b/osfmk/i386/db_trace.c
index deb72c689..2a61d9bf4 100644
--- a/osfmk/i386/db_trace.c
+++ b/osfmk/i386/db_trace.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -67,6 +67,13 @@
 #include <machine/db_machdep.h>
 #include <machine/setjmp.h>
 #include <mach/machine.h>
+#include <mach/kmod.h>
+
+#include <i386/mp.h>
+#include <i386/pio.h>
+#include <i386/cpuid.h>
+#include <i386/proc_reg.h>
+#include <i386/machine_routines.h>
 
 #include <ddb/db_access.h>
 #include <ddb/db_sym.h>
@@ -76,29 +83,17 @@
 #include <ddb/db_output.h>
 
 extern jmp_buf_t *db_recover;
-extern struct i386_saved_state *saved_state[];
+struct x86_kernel_state32 ddb_null_kregs;
+extern kmod_info_t *kmod;
 
-struct i386_kernel_state ddb_null_kregs;
 
 /*
  * Stack trace.
  */
 
-extern vm_offset_t vm_min_inks_addr;	/* set by db_clone_symtabXXX */
-#define INKSERVER(va)	(((vm_offset_t)(va)) >= vm_min_inks_addr)
-
-extern vm_offset_t interrupt_stack[];
-#define	ININTSTACK(va)						\
-	(((vm_offset_t)(va)) >= interrupt_stack[cpu_number()] &&\
-	 (((vm_offset_t)(va)) < interrupt_stack[cpu_number()] +	\
-	                        INTSTACK_SIZE))
+#define	INKERNELSTACK(va, th) 1
 
-#define	INKERNELSTACK(va, th)					\
-	(th == THREAD_NULL ||				\
-	 (((vm_offset_t)(va)) >= th->thread->kernel_stack &&	\
-	  (((vm_offset_t)(va)) < th->thread->kernel_stack +	\
-	                         KERNEL_STACK_SIZE)) ||		\
-         ININTSTACK(va))
+#define DB_NUMARGS_MAX  5
 
 struct i386_frame {
 	struct i386_frame	*f_frame;
@@ -119,20 +114,20 @@ boolean_t	db_trace_symbols_found = FALSE;
 
 struct i386_kregs {
 	char	*name;
-	int	offset;
+	unsigned int offset;
 } i386_kregs[] = {
-	{ "ebx", (int)(&((struct i386_kernel_state *)0)->k_ebx) },
-	{ "esp", (int)(&((struct i386_kernel_state *)0)->k_esp) },
-	{ "ebp", (int)(&((struct i386_kernel_state *)0)->k_ebp) },
-	{ "edi", (int)(&((struct i386_kernel_state *)0)->k_edi) },
-	{ "esi", (int)(&((struct i386_kernel_state *)0)->k_esi) },
-	{ "eip", (int)(&((struct i386_kernel_state *)0)->k_eip) },
-	{ 0 },
+	{ "ebx", (unsigned int)(&((struct x86_kernel_state32 *)0)->k_ebx) },
+	{ "esp", (unsigned int)(&((struct x86_kernel_state32 *)0)->k_esp) },
+	{ "ebp", (unsigned int)(&((struct x86_kernel_state32 *)0)->k_ebp) },
+	{ "edi", (unsigned int)(&((struct x86_kernel_state32 *)0)->k_edi) },
+	{ "esi", (unsigned int)(&((struct x86_kernel_state32 *)0)->k_esi) },
+	{ "eip", (unsigned int)(&((struct x86_kernel_state32 *)0)->k_eip) },
+	{ 0 }
 };
 
 /* Forward */
 
-extern int *	db_lookup_i386_kreg(
+extern unsigned int *	db_lookup_i386_kreg(
 			char			*name,
 			int			*kregp);
 extern int	db_i386_reg_value(
@@ -157,26 +152,26 @@ extern int	_setjmp(
  * Machine register set.
  */
 struct db_variable db_regs[] = {
-	{ "cs",	(int *)&ddb_regs.cs,  db_i386_reg_value, 0, 0, 0, 0, TRUE },
-	{ "ds",	(int *)&ddb_regs.ds,  db_i386_reg_value, 0, 0, 0, 0, TRUE },
-	{ "es",	(int *)&ddb_regs.es,  db_i386_reg_value, 0, 0, 0, 0, TRUE },
-	{ "fs",	(int *)&ddb_regs.fs,  db_i386_reg_value, 0, 0, 0, 0, TRUE },
-	{ "gs",	(int *)&ddb_regs.gs,  db_i386_reg_value, 0, 0, 0, 0, TRUE },
-	{ "ss",	(int *)&ddb_regs.ss,  db_i386_reg_value, 0, 0, 0, 0, TRUE },
-	{ "eax",(int *)&ddb_regs.eax, db_i386_reg_value, 0, 0, 0, 0, TRUE },
-	{ "ecx",(int *)&ddb_regs.ecx, db_i386_reg_value, 0, 0, 0, 0, TRUE },
-	{ "edx",(int *)&ddb_regs.edx, db_i386_reg_value, 0, 0, 0, 0, TRUE },
-	{ "ebx",(int *)&ddb_regs.ebx, db_i386_reg_value, 0, 0, 0, 0, TRUE },
-	{ "esp",(int *)&ddb_regs.uesp,db_i386_reg_value, 0, 0, 0, 0, TRUE },
-	{ "ebp",(int *)&ddb_regs.ebp, db_i386_reg_value, 0, 0, 0, 0, TRUE },
-	{ "esi",(int *)&ddb_regs.esi, db_i386_reg_value, 0, 0, 0, 0, TRUE },
-	{ "edi",(int *)&ddb_regs.edi, db_i386_reg_value, 0, 0, 0, 0, TRUE },
-	{ "eip",(int *)&ddb_regs.eip, db_i386_reg_value, 0, 0, 0, 0, TRUE },
-	{ "efl",(int *)&ddb_regs.efl, db_i386_reg_value, 0, 0, 0, 0, TRUE },
+	{ "cs",	(unsigned int *)&ddb_regs.cs,  db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 },
+	{ "ds",	(unsigned int *)&ddb_regs.ds,  db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 },
+	{ "es",	(unsigned int *)&ddb_regs.es,  db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 },
+	{ "fs",	(unsigned int *)&ddb_regs.fs,  db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 },
+	{ "gs",	(unsigned int *)&ddb_regs.gs,  db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 },
+	{ "ss",	(unsigned int *)&ddb_regs.ss,  db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 },
+	{ "eax",(unsigned int *)&ddb_regs.eax, db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 },
+	{ "ecx",(unsigned int *)&ddb_regs.ecx, db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 },
+	{ "edx",(unsigned int *)&ddb_regs.edx, db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 },
+	{ "ebx",(unsigned int *)&ddb_regs.ebx, db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 },
+	{ "esp",(unsigned int *)&ddb_regs.uesp,db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 },
+	{ "ebp",(unsigned int *)&ddb_regs.ebp, db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 },
+	{ "esi",(unsigned int *)&ddb_regs.esi, db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 },
+	{ "edi",(unsigned int *)&ddb_regs.edi, db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 },
+	{ "eip",(unsigned int *)&ddb_regs.eip, db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 },
+	{ "efl",(unsigned int *)&ddb_regs.efl, db_i386_reg_value, 0, 0, 0, 0, TRUE, 0, 0, (int *)0, 0 }
 };
 struct db_variable *db_eregs = db_regs + sizeof(db_regs)/sizeof(db_regs[0]);
 
-int *
+unsigned int *
 db_lookup_i386_kreg(
 	char	*name,
 	int	*kregp)
@@ -185,7 +180,7 @@ db_lookup_i386_kreg(
 
 	for (kp = i386_kregs; kp->name; kp++) {
 	    if (strcmp(name, kp->name) == 0)
-		return((int *)((int)kregp + kp->offset));
+		return((unsigned int *)((int)kregp + kp->offset));
 	}
 	return(0);
 }
@@ -198,11 +193,9 @@ db_i386_reg_value(
 	db_var_aux_param_t	ap)
 {
 	extern char		etext;
-	int			*dp = 0;
+	unsigned int		*dp = 0;
 	db_expr_t		null_reg = 0;
 	register thread_t	thr_act = ap->thr_act;
-	extern unsigned		int_stack_high;
-	int			cpu;
 
 	if (db_option(ap->modif, 'u')) {
 	    if (thr_act == THREAD_NULL) {
@@ -212,38 +205,36 @@ db_i386_reg_value(
 	    if (thr_act == current_thread()) {
 		if (IS_USER_TRAP(&ddb_regs, &etext))
 		    dp = vp->valuep;
-		else if (ddb_regs.ebp < int_stack_high)
-		    db_error("cannot get/set user registers in nested interrupt\n");
 	    }
 	} else {
 	    if (thr_act == THREAD_NULL || thr_act == current_thread()) {
 		dp = vp->valuep;
 	    } else {
-	      if (thr_act->thread &&
-		  !(thr_act->thread->state & TH_STACK_HANDOFF) && 
-			thr_act->thread->kernel_stack) {
+	      if (thr_act &&
+	      (thr_act->continuation != THREAD_CONTINUE_NULL) &&
+	      thr_act->kernel_stack) {
 		int cpu;
 
 		for (cpu = 0; cpu < real_ncpus; cpu++) {
 		    if (cpu_datap(cpu)->cpu_running == TRUE &&
-			cpu_datap(cpu)->cpu_active_thread == thr_act->thread && saved_state[cpu]) {
-			dp = (int *) (((int)saved_state[cpu]) +
-				      (((int) vp->valuep) -
-				       (int) &ddb_regs));
+			cpu_datap(cpu)->cpu_active_thread == thr_act && cpu_datap(cpu)->cpu_kdb_saved_state) {
+			dp = (unsigned int *) (((unsigned int)cpu_datap(cpu)->cpu_kdb_saved_state) +
+				      (((unsigned int) vp->valuep) -
+				       (unsigned int) &ddb_regs));
 			break;
 		    }
 		}
-		if (dp == 0 && thr_act && thr_act->thread)
+		if (dp == 0 && thr_act)
 		    dp = db_lookup_i386_kreg(vp->name,
-			 (int *)(STACK_IKS(thr_act->thread->kernel_stack)));
+			 (unsigned int *)(STACK_IKS(thr_act->kernel_stack)));
 		if (dp == 0)
 		    dp = &null_reg;
-	      } else if (thr_act->thread &&
-			 (thr_act->thread->state&TH_STACK_HANDOFF)){
-		/* only EIP is valid */
-		if (vp->valuep == (int *) &ddb_regs.eip) {
-		    dp = (int *)(&thr_act->thread->continuation);
-		} else {
+	      } else if (thr_act &&
+	      (thr_act->continuation != THREAD_CONTINUE_NULL)) {
+ 		/* only EIP is valid  */
+ 		if (vp->valuep == (unsigned int *) &ddb_regs.eip) {
+ 		    dp = (unsigned int *)(&thr_act->continuation);
+ 		} else {
 		    dp = &null_reg;
 		}
 	      }
@@ -255,10 +246,10 @@ db_i386_reg_value(
 	    if (!db_option(ap->modif, 'u')) {
 		for (cpu = 0; cpu < real_ncpus; cpu++) {
 		    if (cpu_datap(cpu)->cpu_running == TRUE &&
-		    	cpu_datap(cpu)->cpu_active_thread == thr_act->thread && saved_state[cpu]) {
-		    	    dp = (int *) (((int)saved_state[cpu]) +
-					  (((int) vp->valuep) -
-					   (int) &ddb_regs));
+		    	cpu_datap(cpu)->cpu_active_thread == thr_act && cpu_datap(cpu)->cpu_kdb_saved_state) {
+		    	    dp = (unsigned int *) (((unsigned int)cpu_datap(cpu)->cpu_kdb_saved_state) +
+					  (((unsigned int) vp->valuep) -
+					   (unsigned int) &ddb_regs));
 			    break;
 		    }
 		}
@@ -266,8 +257,8 @@ db_i386_reg_value(
 	    if (dp == 0) {
 		if (!thr_act || thr_act->machine.pcb == 0)
 		    db_error("no pcb\n");
-		dp = (int *)((int)(&thr_act->machine.pcb->iss) + 
-			     ((int)vp->valuep - (int)&ddb_regs));
+		dp = (unsigned int *)((unsigned int)(thr_act->machine.pcb->iss) + 
+			     ((unsigned int)vp->valuep - (unsigned int)&ddb_regs));
 	    }
 	}
 	if (flag == DB_VAR_SET)
@@ -372,33 +363,33 @@ db_nextframe(
 	int			frame_type,	/* in */
 	thread_t		thr_act)	/* in */
 {
+        x86_saved_state32_t	*iss32;
 	extern char *	trap_type[];
 	extern int	TRAP_TYPES;
 
-	struct i386_saved_state *saved_regs;
 	struct interrupt_frame *ifp;
-	struct i386_interrupt_state *isp;
 	task_t task = (thr_act != THREAD_NULL)? thr_act->task: TASK_NULL;
 
 	switch(frame_type) {
 	case TRAP:
 	    /*
 	     * We know that trap() has 1 argument and we know that
-	     * it is an (strcut i386_saved_state *).
+	     * it is an (x86_saved_state32_t *).
 	     */
-	    saved_regs = (struct i386_saved_state *)
-			db_get_task_value((int)&((*fp)->f_arg0),4,FALSE,task);
-	    if (saved_regs->trapno >= 0 && saved_regs->trapno < TRAP_TYPES) {
-		db_printf(">>>>> %s trap at ",
-			trap_type[saved_regs->trapno]);
+	    iss32 = (x86_saved_state32_t *)
+	                 db_get_task_value((int)&((*fp)->f_arg0),4,FALSE,task);
+	    
+	    if (iss32->trapno >= 0 && iss32->trapno < TRAP_TYPES) {
+	            db_printf(">>>>> %s trap at ",
+			      trap_type[iss32->trapno]);
 	    } else {
-		db_printf(">>>>> trap (number %d) at ",
-			saved_regs->trapno & 0xffff);
+	            db_printf(">>>>> trap (number %d) at ",
+			      iss32->trapno & 0xffff);
 	    }
-	    db_task_printsym(saved_regs->eip, DB_STGY_PROC, task);
+	    db_task_printsym(iss32->eip, DB_STGY_PROC, task);
 	    db_printf(" <<<<<\n");
-	    *fp = (struct i386_frame *)saved_regs->ebp;
-	    *ip = (db_addr_t)saved_regs->eip;
+	    *fp = (struct i386_frame *)iss32->ebp;
+	    *ip = (db_addr_t)iss32->eip;
 	    break;
 	case INTERRUPT:
 	    if (*lfp == 0) {
@@ -408,19 +399,21 @@ db_nextframe(
 	    db_printf(">>>>> interrupt at "); 
 	    ifp = (struct interrupt_frame *)(*lfp);
 	    *fp = ifp->if_frame;
-	    if (ifp->if_iretaddr == db_return_to_iret_symbol_value)
-		*ip = ((struct i386_interrupt_state *) ifp->if_edx)->eip;
-	    else
-		*ip = (db_addr_t) ifp->if_eip;
+	    if (ifp->if_iretaddr == db_return_to_iret_symbol_value) {
+	            *ip = ((x86_saved_state32_t *) ifp->if_edx)->eip;
+	    } else
+	            *ip = (db_addr_t) ifp->if_eip;
 	    db_task_printsym(*ip, DB_STGY_PROC, task);
 	    db_printf(" <<<<<\n");
 	    break;
 	case SYSCALL:
 	    if (thr_act != THREAD_NULL && thr_act->machine.pcb) {
-		*ip = (db_addr_t) thr_act->machine.pcb->iss.eip;
-		*fp = (struct i386_frame *) thr_act->machine.pcb->iss.ebp;
-		break;
+		    iss32 = (x86_saved_state32_t *)thr_act->machine.pcb->iss;
+
+		    *ip = (db_addr_t)(iss32->eip);
+		    *fp = (struct i386_frame *)(iss32->ebp);
 	    }
+	    break;
 	    /* falling down for unknown case */
 	default:
 	miss_frame:
@@ -441,6 +434,7 @@ db_stack_trace_cmd(
 	char		*modif)
 {
 	struct i386_frame *frame, *lastframe;
+        x86_saved_state32_t	*iss32;
 	int		*argp;
 	db_addr_t	callpc, lastcallpc;
 	int		frame_type;
@@ -481,17 +475,17 @@ db_stack_trace_cmd(
 	    if (!have_addr && !trace_thread) {
 		have_addr = TRUE;
 		trace_thread = TRUE;
-		act_list = &(current_task()->thr_acts);
+		act_list = &(current_task()->threads);
 		addr = (db_expr_t) queue_first(act_list);
 	    } else if (trace_thread) {
 		if (have_addr) {
 		    if (!db_check_act_address_valid((thread_t)addr)) {
 			if (db_lookup_task((task_t)addr) == -1)
 			    return;
-			act_list = &(((task_t)addr)->thr_acts);
+			act_list = &(((task_t)addr)->threads);
 			addr = (db_expr_t) queue_first(act_list);
 		    } else {
-			act_list = &(((thread_t)addr)->task->thr_acts);
+			act_list = &(((thread_t)addr)->task->threads);
 			thcount = db_lookup_task_act(((thread_t)addr)->task,
 							(thread_t)addr);
 		    }
@@ -504,7 +498,7 @@ db_stack_trace_cmd(
 			return;
 		    }
 		    have_addr = TRUE;
-		    act_list = &th->task->thr_acts;
+		    act_list = &th->task->threads;
 		    addr = (db_expr_t) queue_first(act_list);
 		}
 	    }
@@ -524,11 +518,13 @@ db_stack_trace_cmd(
 	    callpc = (db_addr_t)ddb_regs.eip;
 	    th = current_thread();
 	    task = (th != THREAD_NULL)? th->task: TASK_NULL;
+	    db_printf("thread 0x%x, current_thread() is 0x%x, ebp is 0x%x, eip is 0x%x\n", th, current_thread(), ddb_regs.ebp, ddb_regs.eip);
 	} else if (trace_thread) {
 	    if (have_addr) {
 		th = (thread_t) addr;
-		if (!db_check_act_address_valid(th))
-		    return;
+		if (!db_check_act_address_valid(th)) {
+			return;
+		}
 	    } else {
 		th = db_default_act;
 		if (th == THREAD_NULL)
@@ -540,11 +536,11 @@ db_stack_trace_cmd(
 	    }
 	    if (trace_all_threads)
 		db_printf("---------- Thread 0x%x (#%d of %d) ----------\n",
-		      addr, thcount, th->task->thr_act_count);
+		addr, thcount, th->task->thread_count);
 
 	next_activation:
 	    user_frame = 0;
-
+//	    kprintf("th is %x, current_thread() is %x, ddb_regs.ebp is %x ddb_regs.eip is %x\n", th, current_thread(), ddb_regs.ebp, ddb_regs.eip);
 	    task = th->task;
 	    if (th == current_thread()) {
 	        frame = (struct i386_frame *)ddb_regs.ebp;
@@ -554,36 +550,31 @@ db_stack_trace_cmd(
 		    db_printf("thread has no pcb\n");
 		    return;
 		}
-		if (!th->thread) {
-		    register struct i386_saved_state *iss =
-						&th->machine.pcb->iss;
-
+		if (!th) {
 		    db_printf("thread has no shuttle\n");
-#if 0
-		    frame = (struct i386_frame *) (iss->ebp);
-		    callpc = (db_addr_t) (iss->eip);
-#else
+
 		    goto thread_done;
-#endif
 		}
-		else if ((th->thread->state & TH_STACK_HANDOFF) ||
-			  th->thread->kernel_stack == 0) {
-		    register struct i386_saved_state *iss =
-						&th->machine.pcb->iss;
+		else if ( (th->continuation != THREAD_CONTINUE_NULL) || 
+			  th->kernel_stack == 0) {
 
 		    db_printf("Continuation ");
-		    db_task_printsym((db_expr_t)th->thread->continuation,
+		    db_task_printsym((db_expr_t)th->continuation,
 							DB_STGY_PROC, task);
 		    db_printf("\n");
-		    frame = (struct i386_frame *) (iss->ebp);
-		    callpc = (db_addr_t) (iss->eip);
+
+		    iss32 = (x86_saved_state32_t *)th->machine.pcb->iss;
+
+		    frame = (struct i386_frame *) (iss32->ebp);
+		    callpc = (db_addr_t) (iss32->eip);
+
 		} else {
 		    int cpu;
 
 		    for (cpu = 0; cpu < real_ncpus; cpu++) {
 			if (cpu_datap(cpu)->cpu_running == TRUE &&
-			    cpu_datap(cpu)->cpu_active_thread == th->thread &&
-			    saved_state[cpu]) {
+			    cpu_datap(cpu)->cpu_active_thread == th &&
+			    cpu_datap(cpu)->cpu_kdb_saved_state) {
 			    break;
 			}
 		    }
@@ -593,16 +584,16 @@ db_stack_trace_cmd(
 			     * which is not the top_most one in the RPC chain:
 			     * use the activation's pcb.
 			     */
-			    register struct i386_saved_state *iss =
-				&th->machine.pcb->iss;
-			    frame = (struct i386_frame *) (iss->ebp);
-			    callpc = (db_addr_t) (iss->eip);
+		            iss32 = (x86_saved_state32_t *)th->machine.pcb->iss;
+
+			    frame = (struct i386_frame *) (iss32->ebp);
+			    callpc = (db_addr_t) (iss32->eip);
 		    } else {
-			if (cpu == NCPUS) {
-			    register struct i386_kernel_state *iks;
+			    if (cpu == real_ncpus) {
+			    register struct x86_kernel_state32 *iks;
 			    int r;
 
-			    iks = STACK_IKS(th->thread->kernel_stack);
+			    iks = STACK_IKS(th->kernel_stack);
 			    prev = db_recover;
 			    if ((r = _setjmp(db_recover = &db_jmp_buf)) == 0) {
 				frame = (struct i386_frame *) (iks->k_ebp);
@@ -624,9 +615,11 @@ db_stack_trace_cmd(
 			} else {
 			    db_printf(">>>>> active on cpu %d <<<<<\n",
 				      cpu);
-			    frame = (struct i386_frame *)
-				saved_state[cpu]->ebp;
-			    callpc = (db_addr_t) saved_state[cpu]->eip;
+
+			    iss32 = (x86_saved_state32_t *)cpu_datap(cpu)->cpu_kdb_saved_state;
+
+			    frame = (struct i386_frame *) (iss32->ebp);
+			    callpc = (db_addr_t) (iss32->eip);
 			}
 		    }
 	        }
@@ -646,19 +639,18 @@ db_stack_trace_cmd(
 	    if (kernel_only)
 		goto thread_done;
 	    user_frame++;
-	} else if (INKSERVER(callpc) && INKSERVER(frame)) {
-	    db_printf(">>>>> INKserver space <<<<<\n");
 	}
 
 	lastframe = 0;
 	lastcallpc = (db_addr_t) 0;
 	while (frame_count-- && frame != 0) {
-	    int narg;
+	    int narg = DB_NUMARGS_MAX;
 	    char *	name;
 	    db_expr_t	offset;
 	    db_addr_t call_func = 0;
 	    int r;
-
+	    db_addr_t	off;
+	    
 	    db_symbol_values(NULL,
 			     db_search_task_symbol_and_line(
 					callpc,
@@ -669,15 +661,22 @@ db_stack_trace_cmd(
 					(user_frame) ? task : 0,
 					&narg),
 			     &name, (db_expr_t *)&call_func);
+	    if ( name == NULL) {
+		    db_find_task_sym_and_offset(callpc, 
+		    &name, &off, (user_frame) ? task : 0);
+		    offset = (db_expr_t) off;
+		}
+
 	    if (user_frame == 0) {
-		if (call_func == db_user_trap_symbol_value ||
+		if (call_func && call_func == db_user_trap_symbol_value ||
 		    call_func == db_kernel_trap_symbol_value) {
 		    frame_type = TRAP;
 		    narg = 1;
-		} else if (call_func == db_interrupt_symbol_value) {
+		} else if (call_func &&
+		    call_func == db_interrupt_symbol_value) {
 		    frame_type = INTERRUPT;
 		    goto next_frame;
-		} else if (call_func == db_syscall_symbol_value) {
+		} else if (call_func && call_func == db_syscall_symbol_value) {
 		    frame_type = SYSCALL;
 		    goto next_frame;
 		} else {
@@ -765,10 +764,10 @@ db_stack_trace_cmd(
 			 (user_frame) ? th : THREAD_NULL);
 
 	    if (frame == 0) {
-		if (th->lower != THREAD_NULL) {
+		if (th->task_threads.prev != THREAD_NULL) {
 		    if (top_act == THREAD_NULL)
 			top_act = th;
-		    th = th->lower;
+		    th = th->task_threads.prev;
 		    db_printf(">>>>> next activation 0x%x ($task%d.%d) <<<<<\n",
 			      th,
 			      db_lookup_task(th->task),
@@ -785,14 +784,10 @@ db_stack_trace_cmd(
 		db_printf(">>>>> user space <<<<<\n");
 		if (kernel_only)
 		    break;
-	    } else if ((!INKSERVER(lastframe) || !INKSERVER(lastcallpc)) &&
-			(INKSERVER(callpc) && INKSERVER(frame))) {
-		db_printf(">>>>> inkserver space <<<<<\n");
 	    }
 	    if (frame <= lastframe) {
 		if ((INKERNELSTACK(lastframe, th) &&
-		     !INKERNELSTACK(frame, th)) ||
-		    (INKSERVER(lastframe) ^ INKSERVER(frame)))
+			!INKERNELSTACK(frame, th)))
 		    continue;
 		db_printf("Bad frame pointer: 0x%x\n", frame);
 		break;
@@ -803,7 +798,7 @@ db_stack_trace_cmd(
 	if (trace_all_threads) {
 	    if (top_act != THREAD_NULL)
 		th = top_act;
-	    th = (thread_t) queue_next(&th->thr_acts);
+	    th = (thread_t) queue_next(&th->task_threads);
 	    if (! queue_end(act_list, (queue_entry_t) th)) {
 		db_printf("\n");
 		addr = (db_expr_t) th;
@@ -813,3 +808,63 @@ db_stack_trace_cmd(
 	    }
 	}
 }
+
+extern int kdp_vm_read(caddr_t, caddr_t, unsigned int );
+extern boolean_t kdp_trans_off;
+/*
+ *		Print out 256 bytes of real storage
+ *		
+ *		dr [entaddr]
+ */
+void db_display_real(db_expr_t addr, __unused int have_addr,  __unused db_expr_t count,  __unused char * modif) {
+
+	int				i;
+	unsigned int xbuf[8];
+	unsigned read_result = 0;
+/* Print 256 bytes */
+	for(i=0; i<8; i++) {
+
+/* Do a physical read using kdp_vm_read(), rather than replicating the same
+ * facility
+ */
+		kdp_trans_off = 1;
+		read_result = kdp_vm_read(addr, &xbuf[0], 32);
+		kdp_trans_off = 0;
+
+		if (read_result != 32)
+			db_printf("Unable to read address\n");
+		else
+			db_printf("%016llX   %08X %08X %08X %08X  %08X %08X %08X %08X\n", addr,	/* Print a line */
+			    xbuf[0], xbuf[1], xbuf[2], xbuf[3], 
+			    xbuf[4], xbuf[5], xbuf[6], xbuf[7]);
+		addr = addr + 0x00000020;							/* Point to next address */
+	}
+	db_next = addr;
+}
+
+/*
+ *	Displays all of the kmods in the system.
+ *
+  *	dk
+ */
+void 
+db_display_kmod(__unused db_expr_t addr, __unused int have_addr, __unused db_expr_t count, __unused char *modif)
+{
+
+	kmod_info_t    *kmd;
+	unsigned int    strt, end;
+
+	kmd = kmod;		/* Start at the start */
+
+	db_printf("info      addr      start    - end       name ver\n");
+
+	while (kmd) {		/* Dump 'em all */
+		strt = (unsigned int) kmd->address + kmd->hdr_size;
+		end = (unsigned int) kmd->address + kmd->size;
+		db_printf("%08X  %08X  %08X - %08X: %s, %s\n",
+			kmd, kmd->address, strt, end, kmd->name, kmd->version);
+		kmd = kmd->next;
+	}
+
+	return;
+}
diff --git a/osfmk/i386/eflags.h b/osfmk/i386/eflags.h
index d2aaefbbf..9dc64783e 100644
--- a/osfmk/i386/eflags.h
+++ b/osfmk/i386/eflags.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -63,7 +63,7 @@
  *	i386 flags register
  */
 
-#ifndef EFL_CF		/* FIXME - this is pulled from mach/i386/eflags.h */
+#ifndef EFL_CF
 #define	EFL_CF		0x00000001		/* carry */
 #define	EFL_PF		0x00000004		/* parity of low 8 bits */
 #define	EFL_AF		0x00000010		/* carry out of bit 3 */
@@ -85,6 +85,9 @@
 #define EFL_ID		0x00200000		/* cpuID instruction */
 #endif
 
+#define EFL_CLR		0xfff88028
+#define EFL_SET		0x00000002
+
 #define	EFL_USER_SET	(EFL_IF)
 #define	EFL_USER_CLEAR	(EFL_IOPL|EFL_NT|EFL_RF)
 
diff --git a/osfmk/i386/fpu.c b/osfmk/i386/fpu.c
index 04b52128f..ed19ecb12 100644
--- a/osfmk/i386/fpu.c
+++ b/osfmk/i386/fpu.c
@@ -73,32 +73,14 @@
 #include <i386/thread.h>
 #include <i386/fpu.h>
 #include <i386/trap.h>
-#include <i386/pio.h>
+#include <architecture/i386/pio.h>
 #include <i386/cpuid.h>
 #include <i386/misc_protos.h>
+#include <i386/proc_reg.h>
 
-#if 0
-#include <i386/ipl.h>
-extern int curr_ipl;
-#define ASSERT_IPL(L) \
-{ \
-      if (curr_ipl != L) { \
-	      printf("IPL is %d, expected %d\n", curr_ipl, L); \
-	      panic("fpu: wrong ipl"); \
-      } \
-}
-#else
-#define ASSERT_IPL(L)
-#endif
-
-int		fp_kind = FP_387;	/* 80387 present */
+int		fp_kind = FP_NO;	/* not inited */
 zone_t		ifps_zone;		/* zone for FPU save area */
 
-#define	clear_fpu() \
-    { \
-	set_ts(); \
-    }
-
 #define ALIGNED(addr,size)	(((unsigned)(addr)&((size)-1))==0)
 
 /* Forward */
@@ -109,6 +91,63 @@ extern void		fp_save(
 extern void		fp_load(
 				thread_t	thr_act);
 
+static void configure_mxcsr_capability_mask(struct x86_fpsave_state *ifps);
+
+struct x86_fpsave_state starting_fp_state;
+
+
+/* Global MXCSR capability bitmask */
+static unsigned int mxcsr_capability_mask;
+
+/*
+ * Determine the MXCSR capability mask, which allows us to mask off any
+ * potentially unsafe "reserved" bits before restoring the FPU context.
+ * *Not* per-cpu, assumes symmetry.
+ */
+static void
+configure_mxcsr_capability_mask(struct x86_fpsave_state *ifps)
+{
+	/* FXSAVE requires a 16 byte aligned store */
+	assert(ALIGNED(ifps,16));
+	/* Clear, to prepare for the diagnostic FXSAVE */
+	bzero(ifps, sizeof(*ifps));
+	/* Disable FPU/SSE Device Not Available exceptions */
+	clear_ts();
+
+	__asm__ volatile("fxsave %0" : "=m" (ifps->fx_save_state));
+	mxcsr_capability_mask = ifps->fx_save_state.fx_MXCSR_MASK;
+
+	/* Set default mask value if necessary */
+	if (mxcsr_capability_mask == 0)
+		mxcsr_capability_mask = 0xffbf;
+	
+	/* Re-enable FPU/SSE DNA exceptions */
+	set_ts();
+}
+
+/*
+ * Allocate and initialize FP state for current thread.
+ * Don't load state.
+ */
+static struct x86_fpsave_state *
+fp_state_alloc(void)
+{
+	struct x86_fpsave_state *ifps;
+
+	ifps = (struct x86_fpsave_state *)zalloc(ifps_zone);
+	assert(ALIGNED(ifps,16));
+	bzero((char *)ifps, sizeof *ifps);
+
+	return ifps;
+}
+
+static inline void
+fp_state_free(struct x86_fpsave_state *ifps)
+{
+	zfree(ifps_zone, ifps);
+}
+
+
 /*
  * Look for FPU and initialize it.
  * Called on each CPU.
@@ -132,32 +171,41 @@ init_fpu(void)
 	if ((status & 0xff) == 0 &&
 	    (control & 0x103f) == 0x3f) 
         {
-            fp_kind = FP_387;	/* assume we have a 387 compatible instruction set */
 	    /* Use FPU save/restore instructions if available */
-            if (cpuid_features() & CPUID_FEATURE_FXSR) {
-	    	fp_kind = FP_FXSR;
-		set_cr4(get_cr4() | CR4_FXS);
-		printf("Enabling XMM register save/restore");
-		/* And allow SIMD instructions if present */
-		if (cpuid_features() & CPUID_FEATURE_SSE) {
-		    printf(" and SSE/SSE2");
-		    set_cr4(get_cr4() | CR4_XMM);
-		}
-		printf(" opcodes\n");
-	    }
+		if (cpuid_features() & CPUID_FEATURE_FXSR) {
+		        fp_kind = FP_FXSR;
+			set_cr4(get_cr4() | CR4_FXS);
+			printf("Enabling XMM register save/restore");
+			/* And allow SIMD instructions if present */
+			if (cpuid_features() & CPUID_FEATURE_SSE) {
+		    	printf(" and SSE/SSE2");
+		    	set_cr4(get_cr4() | CR4_XMM);
+			}
+			printf(" opcodes\n");
+	    } else
+			panic("fpu is not FP_FXSR");
 
 	    /*
-	     * Trap wait instructions.  Turn off FPU for now.
+	     * initialze FPU to normal starting 
+	     * position so that we can take a snapshot
+	     * of that state and store it for future use
+	     * when we're asked for the FPU state of a 
+	     * thread, and it hasn't initiated any yet
 	     */
-	    set_cr0(get_cr0() | CR0_TS | CR0_MP);
+	     fpinit();
+	     fxsave(&starting_fp_state.fx_save_state);
+
+	     /*
+	      * Trap wait instructions.  Turn off FPU for now.
+	      */
+	     set_cr0(get_cr0() | CR0_TS | CR0_MP);
 	}
 	else
 	{
 	    /*
 	     * NO FPU.
 	     */
-	    fp_kind = FP_NO;
-	    set_cr0(get_cr0() | CR0_EM);
+		panic("fpu is not FP_FXSR");
 	}
 }
 
@@ -167,10 +215,16 @@ init_fpu(void)
 void
 fpu_module_init(void)
 {
-	ifps_zone = zinit(sizeof(struct i386_fpsave_state),
-			  THREAD_MAX * sizeof(struct i386_fpsave_state),
-			  THREAD_CHUNK * sizeof(struct i386_fpsave_state),
-			  "i386 fpsave state");
+	struct x86_fpsave_state *new_ifps;
+	
+	ifps_zone = zinit(sizeof(struct x86_fpsave_state),
+			  THREAD_MAX * sizeof(struct x86_fpsave_state),
+			  THREAD_CHUNK * sizeof(struct x86_fpsave_state),
+			  "x86 fpsave state");
+	new_ifps = fp_state_alloc();
+	/* Determine MXCSR reserved bits */
+	configure_mxcsr_capability_mask(new_ifps);
+	fp_state_free(new_ifps);
 }
 
 /*
@@ -179,10 +233,9 @@ fpu_module_init(void)
  */
 void
 fpu_free(fps)
-	struct i386_fpsave_state *fps;
+	struct x86_fpsave_state *fps;
 {
-ASSERT_IPL(SPL0);
-	zfree(ifps_zone, fps);
+	fp_state_free(fps);
 }
 
 /*
@@ -196,71 +249,76 @@ ASSERT_IPL(SPL0);
  */
 kern_return_t
 fpu_set_fxstate(
-	thread_t		thr_act,
-	struct i386_float_state	*state)
+	thread_t	thr_act,
+	thread_state_t	tstate)
 {
-	register pcb_t	pcb;
-	register struct i386_fpsave_state *ifps;
-	register struct i386_fpsave_state *new_ifps;
+	struct x86_fpsave_state	*ifps;
+	struct x86_fpsave_state *new_ifps;
+	x86_float_state64_t	*state;
+	pcb_t	pcb;
 
-ASSERT_IPL(SPL0);
 	if (fp_kind == FP_NO)
-	    return KERN_FAILURE;
+	        return KERN_FAILURE;
+
+	state = (x86_float_state64_t *)tstate;
 
-        if (state->fpkind != FP_FXSR) {
-            /* strange if this happens, but in case someone builds one of these manually... */
-            return fpu_set_state(thr_act, state);
-        }
-        
 	assert(thr_act != THREAD_NULL);
 	pcb = thr_act->machine.pcb;
 
-	if (state->initialized == 0) {
-	    /*
-	     * new FPU state is 'invalid'.
-	     * Deallocate the fp state if it exists.
-	     */
-	    simple_lock(&pcb->lock);
-	    ifps = pcb->ims.ifps;
-	    pcb->ims.ifps = 0;
-	    simple_unlock(&pcb->lock);
-
-	    if (ifps != 0) {
-		zfree(ifps_zone, ifps);
-	    }
-	}
-	else {
-	    /*
-	     * Valid state.  Allocate the fp state if there is none.
-	     */
+	if (state == NULL) {
+	        /*
+		 * new FPU state is 'invalid'.
+		 * Deallocate the fp state if it exists.
+		 */
+	        simple_lock(&pcb->lock);
+
+		ifps = pcb->ifps;
+		pcb->ifps = 0;
 
-	    new_ifps = 0;
+		simple_unlock(&pcb->lock);
+
+		if (ifps != 0)
+		        fp_state_free(ifps);
+	} else {
+	        /*
+		 * Valid state.  Allocate the fp state if there is none.
+		 */
+	        new_ifps = 0;
 	Retry:
-	    simple_lock(&pcb->lock);
-	    ifps = pcb->ims.ifps;
-	    if (ifps == 0) {
-		if (new_ifps == 0) {
-		    simple_unlock(&pcb->lock);
-		    new_ifps = (struct i386_fpsave_state *) zalloc(ifps_zone);
-		    assert(ALIGNED(new_ifps,16));
-		    goto Retry;
+		simple_lock(&pcb->lock);
+
+		ifps = pcb->ifps;
+		if (ifps == 0) {
+		        if (new_ifps == 0) {
+			        simple_unlock(&pcb->lock);
+				new_ifps = fp_state_alloc();
+				goto Retry;
+			}
+			ifps = new_ifps;
+			new_ifps = 0;
+			pcb->ifps = ifps;
 		}
-		ifps = new_ifps;
-		new_ifps = 0;
-                bzero((char *)ifps, sizeof *ifps);
-		pcb->ims.ifps = ifps;
-	    }
+		/*
+		 * now copy over the new data.
+		 */
+		bcopy((char *)&state->fpu_fcw,
+		      (char *)&ifps->fx_save_state, sizeof(struct x86_fx_save));
 
-	    /*
-	     * now copy over the new data.
-	     */
-            bcopy((char *)&state->hw_state[0], (char *)&ifps->fx_save_state, sizeof(struct i386_fx_save));
-            ifps->fp_save_flavor = FP_FXSR;
-	    simple_unlock(&pcb->lock);
-	    if (new_ifps != 0)
-		zfree(ifps_zone, ifps);
-	}
+		/* XXX The layout of the state set from user-space may need to be
+		 * validated for consistency.
+		 */
+		ifps->fp_save_layout = thread_is_64bit(thr_act) ? FXSAVE64 : FXSAVE32;
+		/*
+		 * Clear any reserved bits in the MXCSR to prevent a GPF
+		 * when issuing an FXRSTOR.
+		 */
+		ifps->fx_save_state.fx_MXCSR &= mxcsr_capability_mask;
 
+		simple_unlock(&pcb->lock);
+
+		if (new_ifps != 0)
+		        fp_state_free(new_ifps);
+	}
 	return KERN_SUCCESS;
 }
 
@@ -272,234 +330,139 @@ ASSERT_IPL(SPL0);
  */
 kern_return_t
 fpu_get_fxstate(
-	thread_t				thr_act,
-	register struct i386_float_state	*state)
+	thread_t	thr_act,
+	thread_state_t	tstate)
 {
-	register pcb_t	pcb;
-	register struct i386_fpsave_state *ifps;
+	struct x86_fpsave_state	*ifps;
+	x86_float_state64_t	*state;
+	kern_return_t	ret = KERN_FAILURE;
+	pcb_t	pcb;
 
-ASSERT_IPL(SPL0);
-	if (fp_kind == FP_NO) {
+	if (fp_kind == FP_NO)
 	    return KERN_FAILURE;
-	} else if (fp_kind == FP_387) {
-	    return fpu_get_state(thr_act, state);
-	}
+
+	state = (x86_float_state64_t *)tstate;
 
 	assert(thr_act != THREAD_NULL);
 	pcb = thr_act->machine.pcb;
 
 	simple_lock(&pcb->lock);
-	ifps = pcb->ims.ifps;
+
+	ifps = pcb->ifps;
 	if (ifps == 0) {
-	    /*
-	     * No valid floating-point state.
-	     */
-	    simple_unlock(&pcb->lock);
-	    bzero((char *)state, sizeof(struct i386_float_state));
-	    return KERN_SUCCESS;
-	}
+	        /*
+		 * No valid floating-point state.
+		 */
+	        bcopy((char *)&starting_fp_state.fx_save_state,
+		      (char *)&state->fpu_fcw, sizeof(struct x86_fx_save));
+
+		simple_unlock(&pcb->lock);
 
-	/* Make sure we`ve got the latest fp state info */
-	/* If the live fpu state belongs to our target */
+		return KERN_SUCCESS;
+	}
+	/*
+	 * Make sure we`ve got the latest fp state info
+	 * If the live fpu state belongs to our target
+	 */
 	if (thr_act == current_thread())
 	{
-	    clear_ts();
-	    fp_save(thr_act);
-	    clear_fpu();
-	}
-
-	state->fpkind = fp_kind;
-	state->exc_status = 0;
-        state->initialized = ifps->fp_valid;
-        bcopy( (char *)&ifps->fx_save_state, (char *)&state->hw_state[0], sizeof(struct i386_fx_save));
-
-	simple_unlock(&pcb->lock);
-
-	return KERN_SUCCESS;
-}
-
-/*
- * Set the floating-point state for a thread.
- * If the thread is not the current thread, it is
- * not running (held).  Locking needed against
- * concurrent fpu_set_state or fpu_get_state.
- */
-kern_return_t
-fpu_set_state(
-	thread_t		thr_act,
-	struct i386_float_state	*state)
-{
-	register pcb_t	pcb;
-	register struct i386_fpsave_state *ifps;
-	register struct i386_fpsave_state *new_ifps;
+	        boolean_t	intr;
 
-ASSERT_IPL(SPL0);
-	if (fp_kind == FP_NO)
-	    return KERN_FAILURE;
+		intr = ml_set_interrupts_enabled(FALSE);
 
-	assert(thr_act != THREAD_NULL);
-	pcb = thr_act->machine.pcb;
+		clear_ts();
+		fp_save(thr_act);
+		clear_fpu();
 
-	if (state->initialized == 0) {
-	    /*
-	     * new FPU state is 'invalid'.
-	     * Deallocate the fp state if it exists.
-	     */
-	    simple_lock(&pcb->lock);
-	    ifps = pcb->ims.ifps;
-	    pcb->ims.ifps = 0;
-	    simple_unlock(&pcb->lock);
-
-	    if (ifps != 0) {
-		zfree(ifps_zone, ifps);
-	    }
+		(void)ml_set_interrupts_enabled(intr);
 	}
-	else {
-	    /*
-	     * Valid state.  Allocate the fp state if there is none.
-	     */
-	    register struct i386_fp_save *user_fp_state;
-	    register struct i386_fp_regs *user_fp_regs;
-
-	    user_fp_state = (struct i386_fp_save *) &state->hw_state[0];
-	    user_fp_regs  = (struct i386_fp_regs *)
-			&state->hw_state[sizeof(struct i386_fp_save)];
-
-	    new_ifps = 0;
-	Retry:
-	    simple_lock(&pcb->lock);
-	    ifps = pcb->ims.ifps;
-	    if (ifps == 0) {
-		if (new_ifps == 0) {
-		    simple_unlock(&pcb->lock);
-		    new_ifps = (struct i386_fpsave_state *) zalloc(ifps_zone);
-		    assert(ALIGNED(new_ifps,16));
-		    goto Retry;
-		}
-		ifps = new_ifps;
-		new_ifps = 0;
-                bzero((char *)ifps, sizeof *ifps); // zero ALL fields first
-		pcb->ims.ifps = ifps;
-	    }
-
-	    /*
-	     * Ensure that reserved parts of the environment are 0.
-	     */
-	    bzero((char *)&ifps->fp_save_state, sizeof(struct i386_fp_save));
-
-	    ifps->fp_save_state.fp_control = user_fp_state->fp_control;
-	    ifps->fp_save_state.fp_status  = user_fp_state->fp_status;
-	    ifps->fp_save_state.fp_tag     = user_fp_state->fp_tag;
-	    ifps->fp_save_state.fp_eip     = user_fp_state->fp_eip;
-	    ifps->fp_save_state.fp_cs      = user_fp_state->fp_cs;
-	    ifps->fp_save_state.fp_opcode  = user_fp_state->fp_opcode;
-	    ifps->fp_save_state.fp_dp      = user_fp_state->fp_dp;
-	    ifps->fp_save_state.fp_ds      = user_fp_state->fp_ds;
-	    ifps->fp_regs = *user_fp_regs;
-            ifps->fp_save_flavor = FP_387;
-	    simple_unlock(&pcb->lock);
-	    if (new_ifps != 0)
-		zfree(ifps_zone, ifps);
+	if (ifps->fp_valid) {
+        	bcopy((char *)&ifps->fx_save_state,
+	      	      (char *)&state->fpu_fcw, sizeof(struct x86_fx_save));
+		ret = KERN_SUCCESS;
 	}
+	simple_unlock(&pcb->lock);
 
-	return KERN_SUCCESS;
+	return ret;
 }
 
+
 /*
- * Get the floating-point state for a thread.
- * If the thread is not the current thread, it is
- * not running (held).  Locking needed against
- * concurrent fpu_set_state or fpu_get_state.
+ * the child thread is 'stopped' with the thread
+ * mutex held and is currently not known by anyone
+ * so no way for fpu state to get manipulated by an
+ * outside agency -> no need for pcb lock
  */
-kern_return_t
-fpu_get_state(
-	thread_t				thr_act,
-	register struct i386_float_state	*state)
+
+void
+fpu_dup_fxstate(
+	thread_t	parent,
+	thread_t	child)
 {
-	register pcb_t	pcb;
-	register struct i386_fpsave_state *ifps;
+	struct x86_fpsave_state *new_ifps = NULL;
+        boolean_t	intr;
+	pcb_t		ppcb;
 
-ASSERT_IPL(SPL0);
-	if (fp_kind == FP_NO)
-	    return KERN_FAILURE;
+	ppcb = parent->machine.pcb;
 
-	assert(thr_act != THREAD_NULL);
-	pcb = thr_act->machine.pcb;
+	if (ppcb->ifps == NULL)
+	        return;
 
-	simple_lock(&pcb->lock);
-	ifps = pcb->ims.ifps;
-	if (ifps == 0) {
-	    /*
-	     * No valid floating-point state.
-	     */
-	    simple_unlock(&pcb->lock);
-	    bzero((char *)state, sizeof(struct i386_float_state));
-	    return KERN_SUCCESS;
-	}
+        if (child->machine.pcb->ifps)
+	        panic("fpu_dup_fxstate: child's ifps non-null");
 
-	/* Make sure we`ve got the latest fp state info */
-	/* If the live fpu state belongs to our target */
-	if (thr_act == current_thread())
-	{
-	    clear_ts();
-	    fp_save(thr_act);
-	    clear_fpu();
-	}
+	new_ifps = fp_state_alloc();
 
-	state->fpkind = fp_kind;
-	state->exc_status = 0;
+	simple_lock(&ppcb->lock);
 
-	{
-	    register struct i386_fp_save *user_fp_state;
-	    register struct i386_fp_regs *user_fp_regs;
+	if (ppcb->ifps != NULL) {
+	        /*
+		 * Make sure we`ve got the latest fp state info
+		 */
+	        intr = ml_set_interrupts_enabled(FALSE);
 
-	    state->initialized = ifps->fp_valid;
+		clear_ts();
+		fp_save(parent);
+		clear_fpu();
 
-	    user_fp_state = (struct i386_fp_save *) &state->hw_state[0];
-	    user_fp_regs  = (struct i386_fp_regs *)
-			&state->hw_state[sizeof(struct i386_fp_save)];
+		(void)ml_set_interrupts_enabled(intr);
 
-	    /*
-	     * Ensure that reserved parts of the environment are 0.
-	     */
-	    bzero((char *)user_fp_state,  sizeof(struct i386_fp_save));
-
-	    user_fp_state->fp_control = ifps->fp_save_state.fp_control;
-	    user_fp_state->fp_status  = ifps->fp_save_state.fp_status;
-	    user_fp_state->fp_tag     = ifps->fp_save_state.fp_tag;
-	    user_fp_state->fp_eip     = ifps->fp_save_state.fp_eip;
-	    user_fp_state->fp_cs      = ifps->fp_save_state.fp_cs;
-	    user_fp_state->fp_opcode  = ifps->fp_save_state.fp_opcode;
-	    user_fp_state->fp_dp      = ifps->fp_save_state.fp_dp;
-	    user_fp_state->fp_ds      = ifps->fp_save_state.fp_ds;
-	    *user_fp_regs = ifps->fp_regs;
+		if (ppcb->ifps->fp_valid) {
+		        child->machine.pcb->ifps = new_ifps;
+
+			bcopy((char *)&(ppcb->ifps->fx_save_state),
+			      (char *)&(child->machine.pcb->ifps->fx_save_state), sizeof(struct x86_fx_save));
+
+			new_ifps->fp_save_layout = ppcb->ifps->fp_save_layout;
+			/*
+			 * Clear any reserved bits in the MXCSR to prevent a GPF
+			 * when issuing an FXRSTOR.
+			 */
+			new_ifps->fx_save_state.fx_MXCSR &= mxcsr_capability_mask;
+			new_ifps = NULL;
+		}
 	}
-	simple_unlock(&pcb->lock);
+	simple_unlock(&ppcb->lock);
 
-	return KERN_SUCCESS;
+	if (new_ifps != NULL)
+	        fp_state_free(new_ifps);
 }
 
+
 /*
  * Initialize FPU.
  *
- * Raise exceptions for:
- *	invalid operation
- *	divide by zero
- *	overflow
- *
- * Use 53-bit precision.
  */
 void
 fpinit(void)
 {
 	unsigned short	control;
 
-ASSERT_IPL(SPL0);
 	clear_ts();
 	fninit();
 	fnstcw(&control);
 	control &= ~(FPC_PC|FPC_RC); /* Clear precision & rounding control */
-	control |= (FPC_PC_53 |		/* Set precision */ 
+	control |= (FPC_PC_64 |		/* Set precision */ 
 			FPC_RC_RN | 	/* round-to-nearest */
 			FPC_ZE |	/* Suppress zero-divide */
 			FPC_OE |	/*  and overflow */
@@ -508,6 +471,9 @@ ASSERT_IPL(SPL0);
 			FPC_DE |	/* Allow denorms as operands  */
 			FPC_PE);	/* No trap for precision loss */
 	fldcw(control);
+
+	/* Initialize SSE/SSE2 */
+	__builtin_ia32_ldmxcsr(0x1f80);
 }
 
 /*
@@ -517,16 +483,27 @@ ASSERT_IPL(SPL0);
 void
 fpnoextflt(void)
 {
-	/*
-	 * Enable FPU use.
-	 */
-ASSERT_IPL(SPL0);
-	clear_ts();
+	boolean_t	intr;
 
-	/*
-	 * Load this thread`s state into the FPU.
-	 */
-	fp_load(current_thread());
+	intr = ml_set_interrupts_enabled(FALSE);
+
+	clear_ts();			/*  Enable FPU use */
+
+	if (get_interrupt_level()) {
+		/*
+		 * Save current coprocessor context if valid
+		 * Initialize coprocessor live context
+		 */
+		fp_save(current_thread());
+		fpinit();
+	} else {
+		/*
+		 * Load this thread`s state into coprocessor live context.
+		 */
+		fp_load(current_thread());
+	}
+
+	(void)ml_set_interrupts_enabled(intr);
 }
 
 /*
@@ -537,9 +514,17 @@ ASSERT_IPL(SPL0);
 void
 fpextovrflt(void)
 {
-	register thread_t	thr_act = current_thread();
-	register pcb_t		pcb;
-	register struct i386_fpsave_state *ifps;
+	thread_t	thr_act = current_thread();
+	pcb_t		pcb;
+	struct x86_fpsave_state *ifps;
+	boolean_t	intr;
+
+	intr = ml_set_interrupts_enabled(FALSE);
+
+	if (get_interrupt_level())
+		panic("FPU segment overrun exception at interrupt context\n");
+	if (current_task() == kernel_task)
+		panic("FPU segment overrun exception in kernel thread context\n");
 
 	/*
 	 * This is a non-recoverable error.
@@ -547,8 +532,8 @@ fpextovrflt(void)
 	 */
 	pcb = thr_act->machine.pcb;
 	simple_lock(&pcb->lock);
-	ifps = pcb->ims.ifps;
-	pcb->ims.ifps = 0;
+	ifps = pcb->ifps;
+	pcb->ifps = 0;
 	simple_unlock(&pcb->lock);
 
 	/*
@@ -562,6 +547,8 @@ fpextovrflt(void)
 	 */
 	clear_fpu();
 
+	(void)ml_set_interrupts_enabled(intr);
+
 	if (ifps)
 	    zfree(ifps_zone, ifps);
 
@@ -579,22 +566,33 @@ fpextovrflt(void)
 void
 fpexterrflt(void)
 {
-	register thread_t	thr_act = current_thread();
+	thread_t	thr_act = current_thread();
+	struct x86_fpsave_state *ifps = thr_act->machine.pcb->ifps;
+	boolean_t	intr;
+
+	intr = ml_set_interrupts_enabled(FALSE);
+
+	if (get_interrupt_level())
+		panic("FPU error exception at interrupt context\n");
+	if (current_task() == kernel_task)
+		panic("FPU error exception in kernel thread context\n");
 
-ASSERT_IPL(SPL0);
 	/*
 	 * Save the FPU state and turn off the FPU.
 	 */
 	fp_save(thr_act);
 
+	(void)ml_set_interrupts_enabled(intr);
+
 	/*
 	 * Raise FPU exception.
-	 * Locking not needed on pcb->ims.ifps,
+	 * Locking not needed on pcb->ifps,
 	 * since thread is running.
 	 */
 	i386_exception(EXC_ARITHMETIC,
 		       EXC_I386_EXTERR,
-		       thr_act->machine.pcb->ims.ifps->fp_save_state.fp_status);
+		       ifps->fx_save_state.fx_status);
+
 	/*NOTREACHED*/
 }
 
@@ -605,22 +603,30 @@ ASSERT_IPL(SPL0);
  * .	if called from fpu_get_state, pcb already locked.
  * .	if called from fpnoextflt or fp_intr, we are single-cpu
  * .	otherwise, thread is running.
+ * N.B.: Must be called with interrupts disabled
  */
+
 void
 fp_save(
 	thread_t	thr_act)
 {
-	register pcb_t pcb = thr_act->machine.pcb;
-	register struct i386_fpsave_state *ifps = pcb->ims.ifps;
+	pcb_t pcb = thr_act->machine.pcb;
+	struct x86_fpsave_state *ifps = pcb->ifps;
+
 	if (ifps != 0 && !ifps->fp_valid) {
-	    /* registers are in FPU */
-	    ifps->fp_valid = TRUE;
-            ifps->fp_save_flavor = FP_387;
-            if (FXSAFE()) {
-                fxsave(&ifps->fx_save_state);	// save the SSE2/Fp state in addition is enabled
-                ifps->fp_save_flavor = FP_FXSR;
-            }
-	    fnsave(&ifps->fp_save_state);  // also update the old save area for now...
+		assert((get_cr0() & CR0_TS) == 0);
+		/* registers are in FPU */
+		ifps->fp_valid = TRUE;
+
+		if (!thread_is_64bit(thr_act)) {
+			/* save the compatibility/legacy mode XMM+x87 state */
+			fxsave(&ifps->fx_save_state);
+			ifps->fp_save_layout = FXSAVE32;
+		}
+		else {
+			fxsave64(&ifps->fx_save_state);
+			ifps->fp_save_layout = FXSAVE64;
+		}
 	}
 }
 
@@ -634,76 +640,33 @@ void
 fp_load(
 	thread_t	thr_act)
 {
-	register pcb_t pcb = thr_act->machine.pcb;
-	register struct i386_fpsave_state *ifps;
-
-ASSERT_IPL(SPL0);
-	ifps = pcb->ims.ifps;
-	if (ifps == 0) {
-	    ifps = (struct i386_fpsave_state *) zalloc(ifps_zone);
-	    assert(ALIGNED(ifps,16));
-	    bzero((char *)ifps, sizeof *ifps);
-	    pcb->ims.ifps = ifps;
-	    fpinit();
-#if 1
-/* 
- * I'm not sure this is needed. Does the fpu regenerate the interrupt in
- * frstor or not? Without this code we may miss some exceptions, with it
- * we might send too many exceptions.
- */
-	} else if (ifps->fp_valid == 2) {
-		/* delayed exception pending */
-
-		ifps->fp_valid = TRUE;
-		clear_fpu();
-		/*
-		 * Raise FPU exception.
-		 * Locking not needed on pcb->ims.ifps,
-		 * since thread is running.
-		 */
-		i386_exception(EXC_ARITHMETIC,
-		       EXC_I386_EXTERR,
-		       thr_act->machine.pcb->ims.ifps->fp_save_state.fp_status);
-		/*NOTREACHED*/
-#endif
+	pcb_t pcb = thr_act->machine.pcb;
+	struct x86_fpsave_state *ifps;
+
+	ifps = pcb->ifps;
+	if (ifps == 0 || ifps->fp_valid == FALSE) {
+		if (ifps == 0) {
+			/* FIXME: This allocation mechanism should be revised
+			 * for scenarios where interrupts are disabled.
+			 */
+			ifps = fp_state_alloc();
+			pcb->ifps = ifps;
+		}
+		fpinit();
 	} else {
-            if (ifps->fp_save_flavor == FP_FXSR) fxrstor(&ifps->fx_save_state);
-	    else frstor(ifps->fp_save_state);
+		assert(ifps->fp_save_layout == FXSAVE32 || ifps->fp_save_layout == FXSAVE64);
+		if (ifps->fp_save_layout == FXSAVE32) {
+			/* Restore the compatibility/legacy mode XMM+x87 state */
+			fxrstor(&ifps->fx_save_state);
+		}
+		else if (ifps->fp_save_layout == FXSAVE64) {
+			fxrstor64(&ifps->fx_save_state);
+		}
 	}
 	ifps->fp_valid = FALSE;		/* in FPU */
 }
 
 
-/*
- * Allocate and initialize FP state for current thread.
- * Don't load state.
- *
- * Locking not needed; always called on the current thread.
- */
-void
-fp_state_alloc(void)
-{
-	pcb_t	pcb = current_thread()->machine.pcb;
-	struct i386_fpsave_state *ifps;
-
-	ifps = (struct i386_fpsave_state *)zalloc(ifps_zone);
-	assert(ALIGNED(ifps,16));
-	bzero((char *)ifps, sizeof *ifps);
-	pcb->ims.ifps = ifps;
-
-	ifps->fp_valid = TRUE;
-	ifps->fp_save_state.fp_control = (0x037f
-			& ~(FPC_IM|FPC_ZM|FPC_OM|FPC_PC))
-			| (FPC_PC_53|FPC_IC_AFF);
-	ifps->fp_save_state.fp_status = 0;
-	ifps->fp_save_state.fp_tag = 0xffff;	/* all empty */
-        ifps->fx_save_state.fx_control = ifps->fp_save_state.fp_control;
-        ifps->fx_save_state.fx_status = ifps->fp_save_state.fp_status;
-        ifps->fx_save_state.fx_tag = 0x00;
-        ifps->fx_save_state.fx_MXCSR = 0x1f80;
-        
-}
-
 
 /*
  * fpflush(thread_t)
@@ -718,44 +681,53 @@ fpflush(__unused thread_t thr_act)
 	/* not needed on MP x86s; fp not lazily evaluated */
 }
 
-
 /*
- *	Handle a coprocessor error interrupt on the AT386.
- *	This comes in on line 5 of the slave PIC at SPL1.
+ * SSE arithmetic exception handling code.
+ * Basically the same as the x87 exception handler with a different subtype
  */
 
 void
-fpintr(void)
+fpSSEexterrflt(void)
 {
-	spl_t	s;
-	thread_t thr_act = current_thread();
+	thread_t	thr_act = current_thread();
+	struct x86_fpsave_state *ifps = thr_act->machine.pcb->ifps;
+	boolean_t	intr;
 
-ASSERT_IPL(SPL1);
-	/*
-	 * Turn off the extended 'busy' line.
-	 */
-	outb(0xf0, 0);
+	intr = ml_set_interrupts_enabled(FALSE);
+
+	if (get_interrupt_level())
+		panic("SSE exception at interrupt context\n");
+	if (current_task() == kernel_task)
+		panic("SSE exception in kernel thread context\n");
 
 	/*
-	 * Save the FPU context to the thread using it.
+	 * Save the FPU state and turn off the FPU.
 	 */
-	clear_ts();
 	fp_save(thr_act);
-	fninit();
-	clear_fpu();
 
+	(void)ml_set_interrupts_enabled(intr);
 	/*
-	 * Since we are running on the interrupt stack, we must
-	 * signal the thread to take the exception when we return
-	 * to user mode.  Use an AST to do this.
-	 *
-	 * Don`t set the thread`s AST field.  If the thread is
-	 * descheduled before it takes the AST, it will notice
-	 * the FPU error when it reloads its FPU state.
+	 * Raise FPU exception.
+	 * Locking not needed on pcb->ifps,
+	 * since thread is running.
 	 */
-	s = splsched();
-	mp_disable_preemption();
-	ast_on(AST_I386_FP);
-	mp_enable_preemption();
-	splx(s);
+	assert(ifps->fp_save_layout == FXSAVE32 || ifps->fp_save_layout == FXSAVE64);
+	i386_exception(EXC_ARITHMETIC,
+		       EXC_I386_SSEEXTERR,
+		       ifps->fx_save_state.fx_status);
+	/*NOTREACHED*/
+}
+
+
+void
+fp_setvalid(boolean_t value) {
+        thread_t	thr_act = current_thread();
+	struct x86_fpsave_state *ifps = thr_act->machine.pcb->ifps;
+
+	if (ifps) {
+	        ifps->fp_valid = value;
+
+		if (value == TRUE)
+		        clear_fpu();
+	}
 }
diff --git a/osfmk/i386/fpu.h b/osfmk/i386/fpu.h
index a053bd56c..7979b66fa 100644
--- a/osfmk/i386/fpu.h
+++ b/osfmk/i386/fpu.h
@@ -64,12 +64,36 @@
  * Macro definitions for routines to manipulate the
  * floating-point processor.
  */
-
-#include <i386/proc_reg.h>
+#include <kern/thread.h>
 #include <i386/thread.h>
 #include <kern/kern_types.h>
 #include <mach/i386/kern_return.h>
 #include <mach/i386/thread_status.h>
+#include <i386/proc_reg.h>
+
+extern int		fp_kind;
+
+extern void		init_fpu(void);
+extern void		fpu_module_init(void);
+extern void		fpu_free(
+				struct x86_fpsave_state	* fps);
+extern kern_return_t	fpu_set_fxstate(
+				thread_t	thr_act,
+				thread_state_t	state);
+extern kern_return_t	fpu_get_fxstate(
+				thread_t	thr_act,
+				thread_state_t	state);
+extern void		fpu_dup_fxstate(
+				thread_t	parent,
+				thread_t	child);
+extern void		fpnoextflt(void);
+extern void		fpextovrflt(void);
+extern void		fpexterrflt(void);
+extern void		fpSSEexterrflt(void);
+extern void		fpflush(thread_t);
+extern void		fp_setvalid(boolean_t);
+extern void		fxsave64(struct x86_fx_save *);
+extern void		fxrstor64(struct x86_fx_save *);
 
 /*
  * FPU instructions.
@@ -109,58 +133,45 @@ extern __inline__ unsigned short fnstsw(void)
 
 #define FXSAFE() (fp_kind == FP_FXSR)
 
-#define	fpu_load_context(pcb)
+
+static inline void clear_fpu(void)
+{
+	set_ts();
+}
 
 /*
  * Save thread`s FPU context.
- * If only one CPU, we just set the task-switched bit,
- * to keep the new thread from using the coprocessor.
- * If multiple CPUs, we save the entire state.
- * NOTE: in order to provide backwards compatible support in the kernel. When saving SSE2 state, we also save the
- * FP state in it's old location. Otherwise fpu_get_state() and fpu_set_state() will stop working
  */
-#define	fpu_save_context(thread) \
-    { \
-	register struct i386_fpsave_state *ifps; \
-	ifps = (thread)->machine.pcb->ims.ifps; \
-	if (ifps != 0 && !ifps->fp_valid) { \
-	    /* registers are in FPU - save to memory */ \
-	    ifps->fp_valid = TRUE; \
-            ifps->fp_save_flavor = FP_387; \
-            if (FXSAFE()) { \
-		fxsave(&ifps->fx_save_state); \
-                ifps->fp_save_flavor = FP_FXSR; \
-	    } \
-            fnsave(&ifps->fp_save_state); \
-	} \
-	set_ts(); \
-    }
-	    
-
-
-extern int	fp_kind;
 
-extern void		init_fpu(void);
-extern void		fpu_module_init(void);
-extern void		fpu_free(
-				struct i386_fpsave_state	* fps);
-extern kern_return_t	fpu_set_state(
-				thread_t			thr_act,
-				struct i386_float_state		* st);
-extern kern_return_t	fpu_get_state(
-				thread_t			thr_act,
-				struct i386_float_state		* st);
-extern kern_return_t	fpu_set_fxstate(
-				thread_t			thr_act,
-				struct i386_float_state		* st);
-extern kern_return_t	fpu_get_fxstate(
-				thread_t			thr_act,
-				struct i386_float_state		* st);
-extern void		fpnoextflt(void);
-extern void		fpextovrflt(void);
-extern void		fpexterrflt(void);
-extern void		fp_state_alloc(void);
-extern void		fpintr(void);
-extern void		fpflush(thread_t);
+static inline void fpu_save_context(thread_t thread)
+{
+	struct x86_fpsave_state *ifps;
+
+	assert(ml_get_interrupts_enabled() == FALSE);
+	ifps = (thread)->machine.pcb->ifps;
+	if (ifps != 0 && !ifps->fp_valid) {
+		/* Clear CR0.TS in preparation for the FP context save. In
+		 * theory, this shouldn't be necessary since a live FPU should
+		 * indicate that TS is clear. However, various routines
+		 * (such as sendsig & sigreturn) manipulate TS directly.
+		 */
+		clear_ts();
+		/* registers are in FPU - save to memory */
+		ifps->fp_valid = TRUE;
+		if (!thread_is_64bit(thread) || is_saved_state32(thread->machine.pcb->iss)) {
+			/* save the compatibility/legacy mode XMM+x87 state */
+			fxsave(&ifps->fx_save_state);
+			ifps->fp_save_layout = FXSAVE32;
+		}
+		else {
+			/* Execute a brief jump to 64-bit mode to save the 64
+			 * bit state
+			 */
+			fxsave64(&ifps->fx_save_state);
+			ifps->fp_save_layout = FXSAVE64;
+		}
+	}
+	set_ts();
+}
 
 #endif	/* _I386_FPU_H_ */
diff --git a/osfmk/i386/gdt.c b/osfmk/i386/gdt.c
index 44f6a3c2f..084e11b5e 100644
--- a/osfmk/i386/gdt.c
+++ b/osfmk/i386/gdt.c
@@ -66,65 +66,61 @@
 #include <i386/cpu_data.h>
 #include <i386/mp_desc.h>
 
-#ifdef	MACH_BSD
-extern int	trap_unix_syscall(void), trap_mach25_syscall(void),
-		trap_machdep_syscall(void), syscall(void);
-#endif
-
-struct fake_descriptor gdt[GDTSZ] = {
-/* 0x000 */	{ 0, 0, 0, 0 },		/* always NULL */
-/* 0x008 */	{ 0,
-		  0xfffff,
-		  SZ_32|SZ_G,
-		  ACC_P|ACC_PL_K|ACC_CODE_R
-		},			/* kernel code */
-/* 0x010 */	{ 0,
-		  0xfffff,
-		  SZ_32|SZ_G,
-		  ACC_P|ACC_PL_K|ACC_DATA_W
-		},			/* kernel data */
-/* 0x018 */	{ (unsigned int)ldt,
-		  LDTSZ*sizeof(struct fake_descriptor)-1,
-		  0,
-		  ACC_P|ACC_PL_K|ACC_LDT
-		},			/* local descriptor table */
-/* 0x020 */	{ (unsigned int)&ktss,
-		  sizeof(struct i386_tss)-1,
-		  0,
-		  ACC_P|ACC_PL_K|ACC_TSS
-		},			/* TSS for this processor */
-#ifdef	MACH_BSD
-/* 0x28 */	{ (unsigned int) &trap_unix_syscall,
-		   KERNEL_CS,
-		   0,				/* no parameters */
-		   ACC_P|ACC_PL_U|ACC_CALL_GATE
-		},
-/* 0x30 */	{ (unsigned int) &trap_mach25_syscall,
-		   KERNEL_CS,
-		   0,				/* no parameters */
-		   ACC_P|ACC_PL_U|ACC_CALL_GATE
-		},
-/* 0x38 */	{ (unsigned int) &trap_machdep_syscall,
-		   KERNEL_CS,
-		   0,				/* no parameters */
-		   ACC_P|ACC_PL_U|ACC_CALL_GATE
-		},
-#else
-/* 0x028 */	{ 0, 0, 0, 0 },		/* per-thread LDT */
-/* 0x030 */	{ 0, 0, 0, 0 },		/* per-thread TSS for IO bitmap */
-/* 0x038 */	{ 0, 0, 0, 0 },
-#endif
-/* 0x040 */	{ 0, 0, 0, 0 },
-/* 0x048 */	{ (unsigned int)&cpu_data_master,
-		  sizeof(cpu_data_t)-1,
-		  SZ_32,
-		  ACC_P|ACC_PL_K|ACC_DATA_W
-		},			/* per-CPU current thread address */
+struct fake_descriptor master_gdt[GDTSZ] __attribute__ ((aligned (4096))) = {
+	[SEL_TO_INDEX(KERNEL_CS)] {	/* kernel code */
+		0,
+		0xfffff,
+		SZ_32|SZ_G,
+		ACC_P|ACC_PL_K|ACC_CODE_R,
+	},
+	[SEL_TO_INDEX(KERNEL_DS)] {	/* kernel data */
+		0,
+		0xfffff,
+		SZ_32|SZ_G,
+		ACC_P|ACC_PL_K|ACC_DATA_W
+	},
+	[SEL_TO_INDEX(KERNEL_LDT)] {	/* local descriptor table */
+		(uint32_t) &master_ldt,
+		LDTSZ_MIN*sizeof(struct fake_descriptor)-1,
+		0,
+		ACC_P|ACC_PL_K|ACC_LDT
+	},				/* The slot KERNEL_LDT_2 is reserved. */
+	[SEL_TO_INDEX(KERNEL_TSS)] {	/* TSS for this processor */
+		(uint32_t) &master_ktss,
+		sizeof(struct i386_tss)-1,
+		0,
+		ACC_P|ACC_PL_K|ACC_TSS
+	},				/* The slot KERNEL_TSS_2 is reserved. */
+	[SEL_TO_INDEX(CPU_DATA_GS)] {	/* per-CPU current thread address */
+		(uint32_t) &cpu_data_master,
+		sizeof(cpu_data_t)-1,
+		SZ_32,
+		ACC_P|ACC_PL_K|ACC_DATA_W
+	},
+	[SEL_TO_INDEX(USER_LDT)] {	/* user local descriptor table */
+		(uint32_t) &master_ldt,
+		LDTSZ_MIN*sizeof(struct fake_descriptor)-1,
+		0,
+		ACC_P|ACC_PL_K|ACC_LDT
+	},
+	[SEL_TO_INDEX(KERNEL64_CS)] {	/* kernel 64-bit code */ 
+		0,
+		0xfffff,
+		SZ_64|SZ_G,
+		ACC_P|ACC_PL_K|ACC_CODE_R
+	},
+	[SEL_TO_INDEX(KERNEL64_SS)] {	/* kernel 64-bit syscall stack */ 
+		0,
+		0xfffff,
+		SZ_32|SZ_G,
+		ACC_P|ACC_PL_K|ACC_DATA_W
+	},
 #if	MACH_KDB
-/* 0x050 */	{ (unsigned int)&dbtss,
-		  sizeof(struct i386_tss)-1,
-		  0,
-		  ACC_P|ACC_PL_K|ACC_TSS
-		}			/* TSS for this processor */
+	[SEL_TO_INDEX(DEBUG_TSS)] {	/* TSS for this processor */
+		(uint32_t)&master_dbtss,
+		sizeof(struct i386_tss)-1,
+		0,
+		ACC_P|ACC_PL_K|ACC_TSS
+	},
 #endif	/* MACH_KDB */
 };
diff --git a/osfmk/i386/genassym.c b/osfmk/i386/genassym.c
index 69c61ff46..77e29425f 100644
--- a/osfmk/i386/genassym.c
+++ b/osfmk/i386/genassym.c
@@ -67,21 +67,26 @@
 #include <kern/task.h>
 #include <kern/lock.h>
 #include <kern/locks.h>
+#include <kern/host.h>
+#include <kern/misc_protos.h>
 #include <ipc/ipc_space.h>
 #include <ipc/ipc_port.h>
 #include <ipc/ipc_pset.h>
-#include <kern/host.h>
-#include <kern/misc_protos.h>
+#include <vm/vm_map.h>
+#include <i386/cpu_data.h>
 #include <i386/thread.h>
-#include <mach/i386/vm_param.h>
 #include <i386/seg.h>
 #include <i386/pmap.h>
 #include <i386/tss.h>
 #include <i386/cpu_capabilities.h>
+#include <i386/cpuid.h>
+#include <i386/Diagnostics.h>
+#include <i386/pmCPU.h>
+#include <i386/hpet.h>
+#include <mach/i386/vm_param.h>
+#include <mach/i386/thread_status.h>
 #include <machine/commpage.h>
-#include <vm/vm_map.h>
 #include <i386/mp_desc.h>
-#include <i386/cpuid.h>
 #include <pexpert/i386/boot.h>
 
 /*
@@ -151,6 +156,8 @@ main(
 
 	DECLARE("TASK_MACH_EXC_PORT",
 		offsetof(task_t, exc_actions[EXC_MACH_SYSCALL].port));
+	DECLARE("TASK_SYSCALLS_MACH",	offsetof(struct task *, syscalls_mach));
+	DECLARE("TASK_SYSCALLS_UNIX",	offsetof(struct task *, syscalls_unix));
 
 	/* These fields are being added on demand */
 	DECLARE("ACT_MACH_EXC_PORT",
@@ -158,50 +165,131 @@ main(
 
 	DECLARE("ACT_TASK",	offsetof(thread_t, task));
 	DECLARE("ACT_PCB",	offsetof(thread_t, machine.pcb));
+	DECLARE("ACT_SPF",	offsetof(thread_t, machine.specFlags));
 	DECLARE("ACT_MAP",	offsetof(thread_t, map));
+	DECLARE("ACT_COPYIO_STATE", offsetof(thread_t, machine.copyio_state));
+	DECLARE("ACT_PCB_ISS", 	offsetof(thread_t, machine.xxx_pcb.iss));
+	DECLARE("ACT_PCB_IDS", 	offsetof(thread_t, machine.xxx_pcb.ids));
+
+	DECLARE("WINDOWS_CLEAN", WINDOWS_CLEAN);
 
 	DECLARE("MAP_PMAP",	offsetof(vm_map_t, pmap));
 
 #define IKS ((size_t) (STACK_IKS(0)))
 
-	DECLARE("KSS_EBX", IKS + offsetof(struct i386_kernel_state *, k_ebx));
-	DECLARE("KSS_ESP", IKS + offsetof(struct i386_kernel_state *, k_esp));
-	DECLARE("KSS_EBP", IKS + offsetof(struct i386_kernel_state *, k_ebp));
-	DECLARE("KSS_EDI", IKS + offsetof(struct i386_kernel_state *, k_edi));
-	DECLARE("KSS_ESI", IKS + offsetof(struct i386_kernel_state *, k_esi));
-	DECLARE("KSS_EIP", IKS + offsetof(struct i386_kernel_state *, k_eip));
+	DECLARE("KSS_EBX", IKS + offsetof(struct x86_kernel_state32 *, k_ebx));
+	DECLARE("KSS_ESP", IKS + offsetof(struct x86_kernel_state32 *, k_esp));
+	DECLARE("KSS_EBP", IKS + offsetof(struct x86_kernel_state32 *, k_ebp));
+	DECLARE("KSS_EDI", IKS + offsetof(struct x86_kernel_state32 *, k_edi));
+	DECLARE("KSS_ESI", IKS + offsetof(struct x86_kernel_state32 *, k_esi));
+	DECLARE("KSS_EIP", IKS + offsetof(struct x86_kernel_state32 *, k_eip));
 
-	DECLARE("IKS_SIZE",	sizeof(struct i386_kernel_state));
+	DECLARE("IKS_SIZE",	sizeof(struct x86_kernel_state32));
 	DECLARE("IEL_SIZE",	sizeof(struct i386_exception_link));
 
-	DECLARE("PCB_FPS",	offsetof(pcb_t, ims.ifps));
+	DECLARE("PCB_FPS",	offsetof(pcb_t, ifps));
 	DECLARE("PCB_ISS",	offsetof(pcb_t, iss));
 
-	DECLARE("FP_VALID",	offsetof(struct i386_fpsave_state *,fp_valid));
-	DECLARE("FP_SAVE_STATE",
-		offsetof(struct i386_fpsave_state *, fp_save_state));
-
-	DECLARE("R_CS",		offsetof(struct i386_saved_state *, cs));
-	DECLARE("R_SS",		offsetof(struct i386_saved_state *, ss));
-	DECLARE("R_UESP",	offsetof(struct i386_saved_state *, uesp));
-	DECLARE("R_EBP",	offsetof(struct i386_saved_state *, ebp));
-	DECLARE("R_EAX",	offsetof(struct i386_saved_state *, eax));
-	DECLARE("R_EBX",	offsetof(struct i386_saved_state *, ebx));
-	DECLARE("R_ECX",	offsetof(struct i386_saved_state *, ecx));
-	DECLARE("R_EDX",	offsetof(struct i386_saved_state *, edx));
-	DECLARE("R_ESI",	offsetof(struct i386_saved_state *, esi));
-	DECLARE("R_EDI",	offsetof(struct i386_saved_state *, edi));
-	DECLARE("R_TRAPNO",	offsetof(struct i386_saved_state *, trapno));
-	DECLARE("R_ERR",	offsetof(struct i386_saved_state *, err));
-	DECLARE("R_EFLAGS",	offsetof(struct i386_saved_state *, efl));
-	DECLARE("R_EIP",	offsetof(struct i386_saved_state *, eip));
-	DECLARE("R_CR2",	offsetof(struct i386_saved_state *, cr2));
-	DECLARE("ISS_SIZE",	sizeof (struct i386_saved_state));
-
-        DECLARE("I_ECX",	offsetof(struct i386_interrupt_state *, ecx));
-	DECLARE("I_EIP",	offsetof(struct i386_interrupt_state *, eip));
-	DECLARE("I_CS",		offsetof(struct i386_interrupt_state *, cs));
-	DECLARE("I_EFL",	offsetof(struct i386_interrupt_state *, efl));
+	DECLARE("DS_DR0",	offsetof(struct x86_debug_state32 *, dr0));
+	DECLARE("DS_DR1",	offsetof(struct x86_debug_state32 *, dr1));
+	DECLARE("DS_DR2",	offsetof(struct x86_debug_state32 *, dr2));
+	DECLARE("DS_DR3",	offsetof(struct x86_debug_state32 *, dr3));
+	DECLARE("DS_DR4",	offsetof(struct x86_debug_state32 *, dr4));
+	DECLARE("DS_DR5",	offsetof(struct x86_debug_state32 *, dr5));
+	DECLARE("DS_DR6",	offsetof(struct x86_debug_state32 *, dr6));
+	DECLARE("DS_DR7",	offsetof(struct x86_debug_state32 *, dr7));
+
+	DECLARE("DS64_DR0",	offsetof(struct x86_debug_state64 *, dr0));
+	DECLARE("DS64_DR1",	offsetof(struct x86_debug_state64 *, dr1));
+	DECLARE("DS64_DR2",	offsetof(struct x86_debug_state64 *, dr2));
+	DECLARE("DS64_DR3",	offsetof(struct x86_debug_state64 *, dr3));
+	DECLARE("DS64_DR4",	offsetof(struct x86_debug_state64 *, dr4));
+	DECLARE("DS64_DR5",	offsetof(struct x86_debug_state64 *, dr5));
+	DECLARE("DS64_DR6",	offsetof(struct x86_debug_state64 *, dr6));
+	DECLARE("DS64_DR7",	offsetof(struct x86_debug_state64 *, dr7));
+
+	DECLARE("FP_VALID",	offsetof(struct x86_fpsave_state *,fp_valid));
+
+	DECLARE("SS_FLAVOR",	offsetof(x86_saved_state_t *, flavor));
+	DECLARE("SS_32",	x86_SAVED_STATE32);
+	DECLARE("SS_64",	x86_SAVED_STATE64);
+
+#define R_(x)  offsetof(x86_saved_state_t *, ss_32.x)
+	DECLARE("R_CS",		R_(cs));
+	DECLARE("R_SS",		R_(ss));
+	DECLARE("R_DS",		R_(ds));
+	DECLARE("R_ES",		R_(es));
+	DECLARE("R_FS",		R_(fs));
+	DECLARE("R_GS",		R_(gs));
+	DECLARE("R_UESP",	R_(uesp));
+	DECLARE("R_EBP",	R_(ebp));
+	DECLARE("R_EAX",	R_(eax));
+	DECLARE("R_EBX",	R_(ebx));
+	DECLARE("R_ECX",	R_(ecx));
+	DECLARE("R_EDX",	R_(edx));
+	DECLARE("R_ESI",	R_(esi));
+	DECLARE("R_EDI",	R_(edi));
+	DECLARE("R_TRAPNO",	R_(trapno));
+	DECLARE("R_ERR",	R_(err));
+	DECLARE("R_EFLAGS",	R_(efl));
+	DECLARE("R_EIP",	R_(eip));
+	DECLARE("R_CR2",	R_(cr2));
+	DECLARE("ISS32_SIZE",	sizeof (x86_saved_state32_t));
+
+#define R64_(x)  offsetof(x86_saved_state_t *, ss_64.x)
+	DECLARE("R64_FS",	R64_(fs));
+	DECLARE("R64_GS",	R64_(gs));
+	DECLARE("R64_R8",	R64_(r8));
+	DECLARE("R64_R9",	R64_(r9));
+	DECLARE("R64_R10",	R64_(r10));
+	DECLARE("R64_R11",	R64_(r11));
+	DECLARE("R64_R12",	R64_(r12));
+	DECLARE("R64_R13",	R64_(r13));
+	DECLARE("R64_R14",	R64_(r14));
+	DECLARE("R64_R15",	R64_(r15));
+	DECLARE("R64_RBP",	R64_(rbp));
+	DECLARE("R64_RAX",	R64_(rax));
+	DECLARE("R64_RBX",	R64_(rbx));
+	DECLARE("R64_RCX",	R64_(rcx));
+	DECLARE("R64_RDX",	R64_(rdx));
+	DECLARE("R64_RSI",	R64_(rsi));
+	DECLARE("R64_RDI",	R64_(rdi));
+	DECLARE("R64_V_ARG6",	R64_(v_arg6));
+	DECLARE("R64_V_ARG7",	R64_(v_arg7));
+	DECLARE("R64_V_ARG8",	R64_(v_arg8));
+	DECLARE("R64_CS",	R64_(isf.cs));
+	DECLARE("R64_SS",	R64_(isf.ss));
+	DECLARE("R64_RSP",	R64_(isf.rsp));
+	DECLARE("R64_TRAPNO",	R64_(isf.trapno));
+	DECLARE("R64_TRAPFN",	R64_(isf.trapfn));
+	DECLARE("R64_ERR",	R64_(isf.err));
+	DECLARE("R64_RFLAGS",	R64_(isf.rflags));
+	DECLARE("R64_RIP",	R64_(isf.rip));
+	DECLARE("R64_CR2",	R64_(cr2));
+	DECLARE("ISS64_OFFSET",	R64_(isf));
+	DECLARE("ISS64_SIZE",	sizeof (x86_saved_state64_t));
+
+#define ISF64_(x)  offsetof(x86_64_intr_stack_frame_t *, x)
+	DECLARE("ISF64_TRAPNO",	ISF64_(trapno));
+	DECLARE("ISF64_TRAPFN",	ISF64_(trapfn));
+	DECLARE("ISF64_ERR",	ISF64_(err));
+	DECLARE("ISF64_RIP",	ISF64_(rip));
+	DECLARE("ISF64_CS",	ISF64_(cs));
+	DECLARE("ISF64_RFLAGS",	ISF64_(rflags));
+	DECLARE("ISF64_RSP",	ISF64_(rsp));
+	DECLARE("ISF64_SS",	ISF64_(ss));
+	DECLARE("ISF64_SIZE",	sizeof(x86_64_intr_stack_frame_t));
+
+	DECLARE("ISC32_OFFSET",	offsetof(x86_saved_state_compat32_t *, isf64));
+#define ISC32_(x)  offsetof(x86_saved_state_compat32_t *, isf64.x)
+	DECLARE("ISC32_TRAPNO", ISC32_(trapno));
+	DECLARE("ISC32_TRAPFN",	ISC32_(trapfn));
+	DECLARE("ISC32_ERR",	ISC32_(err));
+	DECLARE("ISC32_RIP",	ISC32_(rip));
+	DECLARE("ISC32_CS",	ISC32_(cs));
+	DECLARE("ISC32_RFLAGS",	ISC32_(rflags));
+	DECLARE("ISC32_RSP",	ISC32_(rsp));
+	DECLARE("ISC32_SS",	ISC32_(ss));
 
 	DECLARE("NBPG",			I386_PGBYTES);
 	DECLARE("PAGE_SIZE",            I386_PGBYTES);
@@ -214,6 +302,7 @@ main(
 	DECLARE("KERNELBASE",		VM_MIN_KERNEL_ADDRESS);
 	DECLARE("LINEAR_KERNELBASE",	LINEAR_KERNEL_ADDRESS);
 	DECLARE("KERNEL_STACK_SIZE",	KERNEL_STACK_SIZE);
+	DECLARE("KERNEL_UBER_BASE_HI32", KERNEL_UBER_BASE_HI32);
 
 	DECLARE("COMM_PAGE_BASE_ADDR",  _COMM_PAGE_BASE_ADDRESS);
 
@@ -226,7 +315,6 @@ main(
         DECLARE("PTE_PS",       INTEL_PTE_PS);
 	DECLARE("PTE_U",        INTEL_PTE_USER);
 	DECLARE("PTE_INVALID",	~INTEL_PTE_VALID);
-	DECLARE("CR4_PAE",      CR4_PAE);
 	DECLARE("NPGPTD", NPGPTD);
 
 	DECLARE("IDTSZ",	IDTSZ);
@@ -237,19 +325,27 @@ main(
 	DECLARE("KERNEL_DS",	KERNEL_DS);
 	DECLARE("USER_CS",	USER_CS);
 	DECLARE("USER_DS",	USER_DS);
+	DECLARE("KERNEL64_CS",  KERNEL64_CS);
+	DECLARE("USER64_CS",	USER64_CS);
 	DECLARE("KERNEL_TSS",	KERNEL_TSS);
 	DECLARE("KERNEL_LDT",	KERNEL_LDT);
+	DECLARE("DF_TSS",	DF_TSS);
+	DECLARE("MC_TSS",	MC_TSS);
 #if	MACH_KDB
 	DECLARE("DEBUG_TSS",	DEBUG_TSS);
 #endif	/* MACH_KDB */
         DECLARE("CPU_DATA_GS",	CPU_DATA_GS);
+	DECLARE("SYSENTER_CS",	SYSENTER_CS);
+	DECLARE("SYSENTER_TF_CS",SYSENTER_TF_CS);
+	DECLARE("SYSENTER_DS",	SYSENTER_DS);
+	DECLARE("SYSCALL_CS",	SYSCALL_CS);
+	DECLARE("USER_WINDOW_SEL",	USER_WINDOW_SEL);
+	DECLARE("PHYS_WINDOW_SEL",	PHYS_WINDOW_SEL);
 
         DECLARE("CPU_THIS",
 		offsetof(cpu_data_t *, cpu_this));
         DECLARE("CPU_ACTIVE_THREAD",
 		offsetof(cpu_data_t *, cpu_active_thread));
-        DECLARE("CPU_ACTIVE_KLOADED",
-		offsetof(cpu_data_t *, cpu_active_kloaded));
         DECLARE("CPU_ACTIVE_STACK",
 		offsetof(cpu_data_t *, cpu_active_stack));
         DECLARE("CPU_KERNEL_STACK",
@@ -274,10 +370,63 @@ main(
 		offsetof(cpu_data_t *,cpu_pending_ast));
 	DECLARE("CPU_DESC_TABLEP",
 		offsetof(cpu_data_t *,cpu_desc_tablep));
+	DECLARE("CPU_DESC_INDEX",
+		offsetof(cpu_data_t *,cpu_desc_index));
+	DECLARE("CDI_GDT",
+		offsetof(cpu_desc_index_t *,cdi_gdt));
+	DECLARE("CDI_IDT",
+		offsetof(cpu_desc_index_t *,cdi_idt));
 	DECLARE("CPU_PROCESSOR",
 		offsetof(cpu_data_t *,cpu_processor));
-	DECLARE("CPU_RTC_NANOTIME",
-		offsetof(cpu_data_t *,cpu_rtc_nanotime));
+        DECLARE("CPU_INT_STATE",
+		offsetof(cpu_data_t *, cpu_int_state));
+
+        DECLARE("CPU_HI_ISS",
+		offsetof(cpu_data_t *, cpu_hi_iss));
+        DECLARE("CPU_TASK_CR3",
+		offsetof(cpu_data_t *, cpu_task_cr3));
+        DECLARE("CPU_ACTIVE_CR3",
+		offsetof(cpu_data_t *, cpu_active_cr3));
+        DECLARE("CPU_KERNEL_CR3",
+		offsetof(cpu_data_t *, cpu_kernel_cr3));
+
+	DECLARE("CPU_IS64BIT",
+		offsetof(cpu_data_t *, cpu_is64bit));
+	DECLARE("CPU_TASK_MAP",
+		offsetof(cpu_data_t *, cpu_task_map));
+	DECLARE("TASK_MAP_32BIT",		TASK_MAP_32BIT); 
+	DECLARE("TASK_MAP_64BIT",		TASK_MAP_64BIT); 
+	DECLARE("TASK_MAP_64BIT_SHARED",	TASK_MAP_64BIT_SHARED); 
+	DECLARE("CPU_UBER_USER_GS_BASE",
+		offsetof(cpu_data_t *, cpu_uber.cu_user_gs_base));
+	DECLARE("CPU_UBER_ISF",
+		offsetof(cpu_data_t *, cpu_uber.cu_isf));
+	DECLARE("CPU_UBER_TMP",
+		offsetof(cpu_data_t *, cpu_uber.cu_tmp));
+	DECLARE("CPU_DR7",
+		offsetof(cpu_data_t *, cpu_dr7));
+
+	DECLARE("hwIntCnt", 	offsetof(cpu_data_t *,cpu_hwIntCnt));
+
+	DECLARE("enaExpTrace",	enaExpTrace);
+	DECLARE("enaExpTraceb",	enaExpTraceb);
+	DECLARE("enaUsrFCall",	enaUsrFCall);
+	DECLARE("enaUsrFCallb",	enaUsrFCallb);
+	DECLARE("enaUsrPhyMp",	enaUsrPhyMp);
+	DECLARE("enaUsrPhyMpb",	enaUsrPhyMpb);
+	DECLARE("enaDiagSCs",	enaDiagSCs);
+	DECLARE("enaDiagSCsb",	enaDiagSCsb);
+	DECLARE("enaDiagEM",	enaDiagEM);
+	DECLARE("enaDiagEMb",	enaDiagEMb);
+	DECLARE("enaNotifyEM",	enaNotifyEM);
+	DECLARE("enaNotifyEMb",	enaNotifyEMb);
+	DECLARE("dgLock",		offsetof(struct diagWork *, dgLock));
+	DECLARE("dgFlags",		offsetof(struct diagWork *, dgFlags));
+	DECLARE("dgMisc1",		offsetof(struct diagWork *, dgMisc1));
+	DECLARE("dgMisc2",		offsetof(struct diagWork *, dgMisc2));
+	DECLARE("dgMisc3",		offsetof(struct diagWork *, dgMisc3));
+	DECLARE("dgMisc4",		offsetof(struct diagWork *, dgMisc4));
+	DECLARE("dgMisc5",		offsetof(struct diagWork *, dgMisc5));
 
 	DECLARE("INTEL_PTE_KERNEL",	INTEL_PTE_VALID|INTEL_PTE_WRITE);
 	DECLARE("PTDPTDI",     PTDPTDI);
@@ -285,6 +434,8 @@ main(
 	DECLARE("PDESIZE",     PDESIZE);
 	DECLARE("PTESIZE",     PTESIZE);
 	DECLARE("APTDPTDI",     APTDPTDI);
+	DECLARE("HIGH_MEM_BASE", HIGH_MEM_BASE);
+	DECLARE("HIGH_IDT_BASE", pmap_index_to_virt(HIGH_FIXED_IDT));
 
 	DECLARE("KERNELBASEPDE",
 		(LINEAR_KERNEL_ADDRESS >> PDESHIFT) *
@@ -300,6 +451,7 @@ main(
 	DECLARE("K_TRAP_GATE",	ACC_P|ACC_PL_K|ACC_TRAP_GATE);
 	DECLARE("U_TRAP_GATE",	ACC_P|ACC_PL_U|ACC_TRAP_GATE);
 	DECLARE("K_INTR_GATE",	ACC_P|ACC_PL_K|ACC_INTR_GATE);
+	DECLARE("U_INTR_GATE",  ACC_P|ACC_PL_U|ACC_INTR_GATE);
 	DECLARE("K_TSS",	ACC_P|ACC_PL_K|ACC_TSS);
 
 	/*
@@ -308,33 +460,22 @@ main(
 	DECLARE("USL_INTERLOCK",	offsetof(usimple_lock_t, interlock));
 
 	DECLARE("INTSTACK_SIZE",	INTSTACK_SIZE);
-	DECLARE("MP_GDT",	   offsetof(struct mp_desc_table *, gdt[0]));
-	DECLARE("MP_IDT",	   offsetof(struct mp_desc_table *, idt[0]));
 	DECLARE("TIMER_LOW",	 	offsetof(struct timer *, low_bits));
 	DECLARE("TIMER_HIGH",		offsetof(struct timer *, high_bits));
 	DECLARE("TIMER_HIGHCHK",	offsetof(struct timer *, high_bits_check));
-	DECLARE("KADDR", offsetof(struct KernelBootArgs *, kaddr));
-	DECLARE("KSIZE", offsetof(struct KernelBootArgs *, ksize));
-
-	DECLARE("NANOTIME_BASE_TSC",
-		offsetof(commpage_nanotime_t*, nt_base_tsc));
-	DECLARE("NANOTIME_BASE_NS",
-		offsetof(commpage_nanotime_t*, nt_base_ns));
-	DECLARE("NANOTIME_SCALE",
-		offsetof(commpage_nanotime_t*, nt_scale));
-	DECLARE("NANOTIME_SHIFT",
-		offsetof(commpage_nanotime_t*, nt_shift));
-	DECLARE("NANOTIME_CHECK_TSC",
-		offsetof(commpage_nanotime_t*, nt_check_tsc));
-
-	DECLARE("RTN_TSC",
-		offsetof(rtc_nanotime_t *, rnt_tsc));
-	DECLARE("RTN_NANOS",
-		offsetof(rtc_nanotime_t *, rnt_nanos));
-	DECLARE("RTN_SCALE",
-		offsetof(rtc_nanotime_t *, rnt_scale));
-	DECLARE("RTN_SHIFT",
-		offsetof(rtc_nanotime_t *, rnt_shift));
+	DECLARE("KADDR", offsetof(struct boot_args *, kaddr));
+	DECLARE("KSIZE", offsetof(struct boot_args *, ksize));
+	DECLARE("MEMORYMAP", offsetof(struct boot_args *, MemoryMap));
+	DECLARE("DEVICETREEP", offsetof(struct boot_args *, deviceTreeP));
+
+	DECLARE("RNT_TSC_BASE",
+		offsetof(rtc_nanotime_t *, tsc_base));
+	DECLARE("RNT_NS_BASE",
+		offsetof(rtc_nanotime_t *, ns_base));
+	DECLARE("RNT_SCALE",
+		offsetof(rtc_nanotime_t *, scale));
+	DECLARE("RNT_SHIFT",
+		offsetof(rtc_nanotime_t *, shift));
 
 	/* values from kern/timer.h */
 	DECLARE("TIMER_LOW",
@@ -355,6 +496,22 @@ main(
 	DECLARE("USER_TIMER",
 		offsetof(struct thread *, user_timer));
 
+	DECLARE("OnProc", OnProc);
+
+
+	DECLARE("GCAP_ID",		offsetof(hpetReg_t *, GCAP_ID));
+	DECLARE("GEN_CONF",		offsetof(hpetReg_t *, GEN_CONF));
+	DECLARE("GINTR_STA",	offsetof(hpetReg_t *, GINTR_STA));
+	DECLARE("MAIN_CNT",		offsetof(hpetReg_t *, MAIN_CNT));
+	DECLARE("TIM0_CONF",	offsetof(hpetReg_t *, TIM0_CONF));
+	DECLARE("TIM_CONF",		TIM_CONF);
+	DECLARE("Tn_INT_ENB_CNF",	Tn_INT_ENB_CNF);
+	DECLARE("TIM0_COMP",	offsetof(hpetReg_t *, TIM0_COMP));
+	DECLARE("TIM_COMP",		TIM_COMP);
+	DECLARE("TIM1_CONF",	offsetof(hpetReg_t *, TIM1_CONF));
+	DECLARE("TIM1_COMP",	offsetof(hpetReg_t *, TIM1_COMP));
+	DECLARE("TIM2_CONF",	offsetof(hpetReg_t *, TIM2_CONF));
+	DECLARE("TIM2_COMP",	offsetof(hpetReg_t *, TIM2_COMP));
+
 	return (0);
 }
-
diff --git a/osfmk/i386/hibernate_i386.c b/osfmk/i386/hibernate_i386.c
index 6d47d1c96..c35bc5d4f 100644
--- a/osfmk/i386/hibernate_i386.c
+++ b/osfmk/i386/hibernate_i386.c
@@ -37,74 +37,90 @@
 #include <i386/pmap.h>
 #include <kern/cpu_data.h>
 #include <IOKit/IOPlatformExpert.h>
-#define KERNEL
+
+#include <pexpert/i386/efi.h>
 
 #include <IOKit/IOHibernatePrivate.h>
 #include <vm/vm_page.h>
+#include "i386_lowmem.h"
 
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-/* This assumes that
- * - we never will want to read or write memory below the start of kernel text
- * - kernel text and data isn't included in pmap memory regions
- */
+#define MAX_BANKS	32
 
-extern void *sectTEXTB;
-extern char		*first_avail;
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 hibernate_page_list_t *
 hibernate_page_list_allocate(void)
 {
-    vm_offset_t             base;
+    ppnum_t		    base, num;
     vm_size_t               size;
-    uint32_t                bank;
+    uint32_t                bank, num_banks;
     uint32_t		    pages, page_count;
     hibernate_page_list_t * list;
     hibernate_bitmap_t *    bitmap;
-    pmap_memory_region_t *  regions;
-    pmap_memory_region_t *  rp;
-    uint32_t                num_regions, num_alloc_regions;
-
-    page_count = 0;
-
-    /* Make a list of the maximum number of regions needed */
-    num_alloc_regions = 1 + pmap_memory_region_count;
-
-    /* Allocate our own list of memory regions so we can sort them in order. */
-    regions = (pmap_memory_region_t *)kalloc(sizeof(pmap_memory_region_t) * num_alloc_regions);
-    if (!regions)
-        return (0);
 
-    /* Fill in the actual regions we will be returning. */
-    rp = regions;
+    EfiMemoryRange *	    mptr;
+    uint32_t		    mcount, msize, i;
+    hibernate_bitmap_t	    dram_ranges[MAX_BANKS];
+    boot_args *		    args = (boot_args *) PE_state.bootArgs;
 
-    /* XXX should check for non-volatile memory region below kernel space. */
-    /* Kernel region is first. */
-    base = (vm_offset_t)(sectTEXTB) & 0x3FFFFFFF;
-    rp->base = atop_32(base);
-    rp->end = atop_32((vm_offset_t)first_avail) - 1;
-    rp->alloc = 0;
-    num_regions = 1;
+    mptr = args->MemoryMap;
+    if (args->MemoryMapDescriptorSize == 0)
+	panic("Invalid memory map descriptor size");
+    msize = args->MemoryMapDescriptorSize;
+    mcount = args->MemoryMapSize / msize;
 
-    /* Remaining memory regions.  Consolidate adjacent regions. */
-    for (bank = 0; bank < (uint32_t) pmap_memory_region_count; bank++)
+    num_banks = 0;
+    for (i = 0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize))
     {
-        if ((rp->end + 1) == pmap_memory_regions[bank].base) {
-            rp->end = pmap_memory_regions[bank].end;
-        } else {
-            ++rp;
-            ++num_regions;
-            rp->base = pmap_memory_regions[bank].base;
-            rp->end = pmap_memory_regions[bank].end;
-            rp->alloc = 0;
-        }
+	base = (ppnum_t) (mptr->PhysicalStart >> I386_PGSHIFT);
+	num = (ppnum_t) mptr->NumberOfPages;
+	if (!num)
+	    continue;
+
+	switch (mptr->Type)
+	{
+	    // any kind of dram
+	    case kEfiLoaderCode:
+	    case kEfiLoaderData:
+	    case kEfiBootServicesCode:
+	    case kEfiBootServicesData:
+	    case kEfiConventionalMemory:
+	    case kEfiACPIReclaimMemory:
+	    case kEfiACPIMemoryNVS:
+	    case kEfiPalCode:
+
+		if (!num_banks || (base != (1 + dram_ranges[num_banks - 1].last_page)))
+		{
+		    num_banks++;
+		    if (num_banks >= MAX_BANKS)
+			break;
+		    dram_ranges[num_banks - 1].first_page = base;
+		}
+		dram_ranges[num_banks - 1].last_page = base + num - 1;
+		break;
+
+	    // runtime services will be restarted, so no save
+	    case kEfiRuntimeServicesCode:
+	    case kEfiRuntimeServicesData:
+	    // non dram
+	    case kEfiReservedMemoryType:
+	    case kEfiUnusableMemory:
+	    case kEfiMemoryMappedIO:
+	    case kEfiMemoryMappedIOPortSpace:
+	    default:
+		break;
+	}
     }
 
-    /* Size the hibernation bitmap */
+    if (num_banks >= MAX_BANKS)
+	return (NULL);
+
+    // size the hibernation bitmap
+
     size = sizeof(hibernate_page_list_t);
     page_count = 0;
-    for (bank = 0, rp = regions; bank < num_regions; bank++, rp++) {
-	pages = rp->end + 1 - rp->base;
+    for (bank = 0; bank < num_banks; bank++) {
+	pages = dram_ranges[bank].last_page + 1 - dram_ranges[bank].first_page;
 	page_count += pages;
         size += sizeof(hibernate_bitmap_t) + ((pages + 31) >> 5) * sizeof(uint32_t);
     }
@@ -115,62 +131,75 @@ hibernate_page_list_allocate(void)
 	
     list->list_size  = size;
     list->page_count = page_count;
-    list->bank_count = num_regions;
+    list->bank_count = num_banks;
+
+    // convert to hibernation bitmap.
 
-    /* Convert to hibernation bitmap. */
-    /* This assumes that ranges are in order and do not overlap. */
     bitmap = &list->bank_bitmap[0];
-    for (bank = 0, rp = regions; bank < num_regions; bank++, rp++) {
-        bitmap->first_page = rp->base;
-        bitmap->last_page = rp->end;
+    for (bank = 0; bank < num_banks; bank++)
+    {
+        bitmap->first_page = dram_ranges[bank].first_page;
+        bitmap->last_page  = dram_ranges[bank].last_page;
         bitmap->bitmapwords = (bitmap->last_page + 1
                                - bitmap->first_page + 31) >> 5;
-        kprintf("HIB: Bank %d: 0x%x end 0x%x\n", bank,
-                ptoa_32(bitmap->first_page),
-                ptoa_32(bitmap->last_page));
+        kprintf("hib bank[%d]: 0x%x000 end 0x%xfff\n", bank,
+                bitmap->first_page,
+                bitmap->last_page);
 	bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
     }
 
-    kfree((void *)regions, sizeof(pmap_memory_region_t) * num_alloc_regions);
     return (list);
 }
 
+// mark pages not to be saved, but available for scratch usage during restore
+
+void
+hibernate_page_list_setall_machine( __unused hibernate_page_list_t * page_list,
+                                    __unused hibernate_page_list_t * page_list_wired,
+                                    __unused uint32_t * pagesOut)
+{
+}
+
+// mark pages not to be saved and not for scratch usage during restore
 void
-hibernate_page_list_setall_machine(hibernate_page_list_t * page_list,
-                                   hibernate_page_list_t * page_list_wired,
-                                   uint32_t * pagesOut)
+hibernate_page_list_set_volatile( hibernate_page_list_t * page_list,
+				  hibernate_page_list_t * page_list_wired,
+				  uint32_t * pagesOut)
 {
-    KernelBootArgs_t *      bootArgs = (KernelBootArgs_t *)PE_state.bootArgs;
-    MemoryRange *           mptr;
-    uint32_t                bank;
-    uint32_t                page, count;
-
-    for (bank = 0, mptr = bootArgs->memoryMap; bank < bootArgs->memoryMapCount; bank++, mptr++) {
-
-        if (kMemoryRangeNVS != mptr->type) continue;
-        kprintf("Base NVS region 0x%x + 0x%x\n", (vm_offset_t)mptr->base, (vm_size_t)mptr->length);
-        /* Round to page size.  Hopefully this does not overlap any reserved areas. */
-        page = atop_32(trunc_page((vm_offset_t)mptr->base));
-        count = atop_32(round_page((vm_offset_t)mptr->base + (vm_size_t)mptr->length)) - page;
-        kprintf("Rounded NVS region 0x%x size 0x%x\n", page, count);
-
-        hibernate_set_page_state(page_list, page_list_wired, page, count, 1);
-        pagesOut -= count;
+    boot_args * args = (boot_args *) PE_state.bootArgs;
+
+    hibernate_set_page_state(page_list, page_list_wired, 
+		I386_HIB_PAGETABLE, I386_HIB_PAGETABLE_COUNT, 
+		kIOHibernatePageStateFree);
+    *pagesOut -= I386_HIB_PAGETABLE_COUNT;
+
+    if (args->efiRuntimeServicesPageStart)
+    {
+	hibernate_set_page_state(page_list, page_list_wired, 
+		    args->efiRuntimeServicesPageStart, args->efiRuntimeServicesPageCount, 
+		    kIOHibernatePageStateFree);
+	*pagesOut -= args->efiRuntimeServicesPageCount;
     }
 }
 
 kern_return_t 
 hibernate_processor_setup(IOHibernateImageHeader * header)
 {
-    current_cpu_datap()->cpu_hibernate = 1;
+    boot_args * args = (boot_args *) PE_state.bootArgs;
+
+    cpu_datap(0)->cpu_hibernate = 1;
     header->processorFlags = 0;
+
+    header->runtimePages     = args->efiRuntimeServicesPageStart;
+    header->runtimePageCount = args->efiRuntimeServicesPageCount;
+
     return (KERN_SUCCESS);
 }
 
 void
 hibernate_vm_lock(void)
 {
-    if (FALSE /* getPerProc()->hibernate */)
+    if (current_cpu_datap()->cpu_hibernate)
     {
         vm_page_lock_queues();
         mutex_lock(&vm_page_queue_free_lock);
@@ -180,7 +209,7 @@ hibernate_vm_lock(void)
 void
 hibernate_vm_unlock(void)
 {
-    if (FALSE /* getPerProc()->hibernate */)
+    if (current_cpu_datap()->cpu_hibernate)
     {
         mutex_unlock(&vm_page_queue_free_lock);
         vm_page_unlock_queues();
diff --git a/osfmk/i386/hibernate_restore.s b/osfmk/i386/hibernate_restore.s
index 5f7507f94..772c49186 100644
--- a/osfmk/i386/hibernate_restore.s
+++ b/osfmk/i386/hibernate_restore.s
@@ -48,34 +48,12 @@ needs to be careful to only touch memory also in the "__HIB" section.
 	.long	address		;\
 	.word	segment
 	
-#define KVTOPHYS	(-KERNELBASE)
-#define	KVTOLINEAR	LINEAR_KERNELBASE
-
-#define	PA(addr)	((addr)+KVTOPHYS)
-#define	VA(addr)	((addr)-KVTOPHYS)
-
 /* Location of temporary page tables */
-#define HPTD        0x80000
-	
-#define KERNEL_MAP_SIZE (  4 * 1024 * 1024)
+#define HPTD        (0x13000)
+#define HPDPT       (0x17000)
 
-/*
- * fillkpt
- *	eax = page frame address
- *	ebx = index into page table
- *	ecx = how many pages to map
- * 	base = base address of page dir/table
- *	prot = protection bits
- */
-#define	fillkpt(base, prot)		  \
-	shll	$2,%ebx			; \
-	addl	base,%ebx		; \
-	orl	$(PTE_V), %eax          ; \
-	orl	prot,%eax		; \
-1:	movl	%eax,(%ebx)		; \
-	addl	$(PAGE_SIZE),%eax	; /* increment physical address */ \
-	addl	$4,%ebx			; /* next pte */ \
-	loop	1b
+#define LAST_PAGE	(0xFFE00000)
+#define LAST_PAGE_PDE   (0x7ff)
 
 /*
  * fillpse
@@ -86,26 +64,48 @@ needs to be careful to only touch memory also in the "__HIB" section.
  *	prot = protection bits
  */
 #define	fillpse(base, prot)		  \
-	shll	$2,%ebx			; \
+	shll	$3,%ebx			; \
 	addl	base,%ebx		; \
-	orl	$(PTE_V|PTE_PS), %eax   ; \
+	orl	$(PTE_V|PTE_PS|0x60), %eax   ; \
 	orl	prot,%eax		; \
-1:	movl	%eax,(%ebx)		; \
-	addl	$(1 << PDESHIFT),%eax	; /* increment physical address 4Mb */ \
+        xorl    %edx, %edx		; \
+1:	movl	%eax,(%ebx)		; /* low 32b */ \
+	addl	$4,%ebx			; \
+	movl	%edx,(%ebx)		; /* high 32b */ \
+	addl	$(1 << PDESHIFT),%eax	; /* increment physical address 2Mb */ \
 	addl	$4,%ebx			; /* next entry */ \
 	loop	1b
 	
-/*
- * fillkptphys(base, prot)
- *	eax = physical address
- *	ecx = how many pages to map
- *      base = base of page table
- *	prot = protection bits
+
+
+/*  Segment Descriptor
+ *
+ * 31          24         19   16                 7           0
+ * ------------------------------------------------------------
+ * |             | |B| |A|       | |   |1|0|E|W|A|            |
+ * | BASE 31..24 |G|/|0|V| LIMIT |P|DPL|  TYPE   | BASE 23:16 |
+ * |             | |D| |L| 19..16| |   |1|1|C|R|A|            |
+ * ------------------------------------------------------------
+ * |                             |                            |
+ * |        BASE 15..0           |       LIMIT 15..0          |
+ * |                             |                            |
+ * ------------------------------------------------------------
  */
-#define	fillkptphys(base, prot)		  \
-	movl	%eax, %ebx		; \
-	shrl	$(PAGE_SHIFT), %ebx	; \
-	fillkpt(base, prot)
+
+	.align	ALIGN
+ENTRY(hib_gdt)
+	.word	0, 0		/* 0x0  : null */
+	.byte	0, 0, 0, 0
+
+	.word	0xffff, 0x0000	/* 0x8  : code */
+	.byte	0, 0x9e, 0xcf, 0
+
+	.word	0xffff, 0x0000	/* 0x10 : data */
+	.byte	0, 0x92, 0xcf, 0
+
+ENTRY(hib_gdtr)
+	.word	24		/* limit (8*3 segs) */
+	.long	EXT(hib_gdt) 
 
 /*
  * Hibernation code restarts here.  Steal some pages from 0x10000
@@ -126,38 +126,69 @@ LEXT(hibernate_machine_entrypoint)
 	cli
 
         mov     %eax, %edi
-        
+
 	POSTCODE(0x1)
 
-	/* Map physical memory from zero to 0xC0000000 */
+	/* Map physical memory from zero to LAST_PAGE */
         xorl    %eax, %eax
         xorl    %ebx, %ebx
-        movl    $(KPTDI), %ecx
+        movl    $(LAST_PAGE_PDE), %ecx
         fillpse( $(HPTD), $(PTE_W) )
 
-        /* Map 0 again at 0xC0000000 */
-        xorl    %eax, %eax
-        movl    $(KPTDI), %ebx
-        movl    $(KERNEL_MAP_SIZE >> PDESHIFT), %ecx
-        fillpse( $(HPTD), $(PTE_W) )
-        	
-	movl	$(HPTD), %eax
+	movl	$(HPDPT), %ebx
+        movl    $(HPTD), %eax
+	orl	$(PTE_V), %eax
+
+        xorl    %edx, %edx		; \
+
+	movl	%eax,(%ebx)		; /* low 32b */ \
+	addl	$4,%ebx			; \
+	movl	%edx,(%ebx)		; /* high 32b */ \
+	addl	$4,%ebx			; \
+	addl	$(1 << 12),%eax		; /* increment physical address 1Gb */ \
+
+	movl	%eax,(%ebx)		; /* low 32b */ \
+	addl	$4,%ebx			; \
+	movl	%edx,(%ebx)		; /* high 32b */ \
+	addl	$4,%ebx			; \
+	addl	$(1 << 12),%eax		; /* increment physical address 1Gb */ \
+
+	movl	%eax,(%ebx)		; /* low 32b */ \
+	addl	$4,%ebx			; \
+	movl	%edx,(%ebx)		; /* high 32b */ \
+	addl	$4,%ebx			; \
+	addl	$(1 << 12),%eax		; /* increment physical address 1Gb */ \
+
+	movl	%eax,(%ebx)		; /* low 32b */
+	addl	$4,%ebx			; 
+	movl	%edx,(%ebx)		; /* high 32b */ \
+	addl	$4,%ebx			; \
+	addl	$(1 << 12),%eax		; /* increment physical address 1Gb */ \
+
+	/* set page dir ptr table addr */
+	movl	$(HPDPT), %eax
 	movl	%eax, %cr3
 
         POSTCODE(0x3)
         
 	movl    %cr4,%eax
-        orl     $(CR4_PSE),%eax
+        orl     $(CR4_PAE|CR4_PGE),%eax
         movl    %eax,%cr4               /* enable page size extensions */
+
+	movl	$(MSR_IA32_EFER), %ecx			/* MSR number in ecx */
+	rdmsr						/* MSR value return in edx: eax */
+	orl	$(MSR_IA32_EFER_NXE), %eax		/* Set NXE bit in low 32-bits */
+	wrmsr						/* Update Extended Feature Enable reg */
+
 	movl	%cr0, %eax
 	orl	$(CR0_PG|CR0_WP|CR0_PE), %eax
 	movl	%eax, %cr0	/* ready paging */
 	
         POSTCODE(0x4)
 
-	lgdt	PA(EXT(gdtptr))		/* load GDT */
-	lidt	PA(EXT(idtptr))		/* load IDT */
-	
+	lgdt	EXT(gdtptr)		/* load GDT */
+	lidt	EXT(idtptr)		/* load IDT */
+
         POSTCODE(0x5)
 
         LJMP	(KERNEL_CS,EXT(hstart))  /* paging on and go to correct vaddr */
@@ -181,23 +212,22 @@ LEXT(hstart)
 	
         xorl    %eax, %eax              /* Video memory - N/A */
         pushl   %eax
+        pushl   %eax
+        pushl   %eax
         mov     %edi, %eax              /* Pointer to hibernate header */
         pushl   %eax
         call    EXT(hibernate_kernel_entrypoint)
         /* NOTREACHED */
         hlt
-
-
         
 /*
 void 
 hibernate_restore_phys_page(uint64_t src, uint64_t dst, uint32_t len, uint32_t procFlags);
 */
 
-			.align	5
-			.globl	EXT(hibernate_restore_phys_page)
+	.align	5
+	.globl	EXT(hibernate_restore_phys_page)
 
-        /* XXX doesn't handle 64-bit addresses yet */
 	/* XXX can only deal with exactly one page */
 LEXT(hibernate_restore_phys_page)
 	pushl	%edi
@@ -205,35 +235,39 @@ LEXT(hibernate_restore_phys_page)
 
 	movl	8+ 4(%esp),%esi		/* source virtual address */
         addl    $0, %esi
-        jz      2f                      /* If source == 0, nothing to do */
+        jz      3f                      /* If source == 0, nothing to do */
         
+	movl    8+ 16(%esp),%eax        /* destination physical address, high 32 bits  */
+	movl    8+ 12(%esp),%edi        /* destination physical address, low 32 bits */
+        addl    $0, %eax
+        jne     1f                      /* need to map, above LAST_PAGE */
 
-	movl    8+ 12(%esp),%edi        /* destination physical address */
-        cmpl    $(LINEAR_KERNELBASE), %edi
-        jl      1f                      /* no need to map, below 0xC0000000 */
-
+        cmpl    $(LAST_PAGE), %edi
+        jb      2f                      /* no need to map, below LAST_PAGE */
+1:
+        /* Map physical address %eax:%edi to virt. address LAST_PAGE (4GB - 2MB) */
+        movl    %eax, (HPTD + (LAST_PAGE_PDE * 8) + 4)
         movl    %edi, %eax              /* destination physical address */
-        /* Map physical address to virt. address 0xffc00000 (4GB - 4MB) */
-        andl    $0xFFC00000, %eax
+        andl    $(LAST_PAGE), %eax
         orl     $(PTE_V | PTE_PS | PTE_W), %eax
-        movl    %eax, (HPTD + (0x3FF * 4))
-        orl     $0xFFC00000, %edi
+        movl    %eax, (HPTD + (LAST_PAGE_PDE * 8))
+        orl     $(LAST_PAGE), %edi
         invlpg  (%edi)
 
-1:      
+2:      
 	movl	8+ 20(%esp),%edx	/* number of bytes */
 	cld
-/* move longs*/
+	/* move longs*/
 	movl	%edx,%ecx
 	sarl	$2,%ecx
 	rep
 	movsl
-/* move bytes*/
+	/* move bytes*/
 	movl	%edx,%ecx
 	andl	$3,%ecx
 	rep
 	movsb
-2:
+3:
 	popl	%esi
 	popl	%edi
 	ret
diff --git a/osfmk/i386/hpet.c b/osfmk/i386/hpet.c
new file mode 100644
index 000000000..a3d45b877
--- /dev/null
+++ b/osfmk/i386/hpet.c
@@ -0,0 +1,434 @@
+/*
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <string.h>
+#include <mach/vm_param.h>
+#include <mach/vm_prot.h>
+#include <mach/machine.h>
+#include <mach/time_value.h>
+#include <kern/spl.h>
+#include <kern/assert.h>
+#include <kern/debug.h>
+#include <kern/misc_protos.h>
+#include <kern/startup.h>
+#include <kern/clock.h>
+#include <kern/cpu_data.h>
+#include <kern/processor.h>
+#include <vm/vm_page.h>
+#include <vm/pmap.h>
+#include <vm/vm_kern.h>
+#include <i386/pmap.h>
+#include <i386/misc_protos.h>
+#include <i386/cpuid.h>
+#include <i386/mp.h>
+#include <i386/machine_cpu.h>
+#include <i386/machine_routines.h>
+#include <i386/io_map_entries.h>
+#include <architecture/i386/pio.h>
+#include <i386/cpuid.h>
+#include <i386/apic.h>
+#include <i386/tsc.h>
+#include <i386/hpet.h>
+#include <i386/pmCPU.h>
+#include <pexpert/device_tree.h>
+#if	MACH_KDB
+#include <ddb/db_aout.h>
+#include <ddb/db_access.h>
+#include <ddb/db_sym.h>
+#include <ddb/db_variables.h>
+#include <ddb/db_command.h>
+#include <ddb/db_output.h>
+#include <ddb/db_expr.h>
+#endif /* MACH_KDB */
+#include <ddb/tr.h>
+
+/* Decimal powers: */
+#define kilo (1000ULL)
+#define Mega (kilo * kilo)
+#define Giga (kilo * Mega)
+#define Tera (kilo * Giga)
+#define Peta (kilo * Tera)
+
+uint32_t hpetArea = 0;			
+uint32_t hpetAreap = 0;			
+uint64_t hpetFemto = 0;
+uint64_t hpetFreq = 0;
+uint64_t hpetCvt = 0;			/* (TAKE OUT LATER)  */
+uint64_t hpetCvtt2n = 0;
+uint64_t hpetCvtn2t = 0;
+uint64_t tsc2hpet = 0;
+uint64_t hpet2tsc = 0;
+uint64_t bus2hpet = 0;
+uint64_t hpet2bus = 0;
+
+uint32_t rcbaArea = 0;			
+uint32_t rcbaAreap = 0;			
+
+#if DEBUG
+#define DBG(x...)	kprintf("DBG: " x)
+#else
+#define DBG(x...)
+#endif
+
+/*
+ * Map the RCBA area.
+ */
+static void
+map_rcbaArea(void)
+{
+	/*
+	 * Get RCBA area physical address and map it
+	 */
+	outl(cfgAdr, lpcCfg | (0xF0 & 0xFC));
+	rcbaAreap = inl(cfgDat | (0xF0 & 0x03));
+	rcbaArea = io_map_spec(rcbaAreap & -4096, PAGE_SIZE * 4, VM_WIMG_IO);
+	kprintf("RCBA: vaddr = %08X, paddr = %08X\n", rcbaArea, rcbaAreap);
+}
+
+/*
+ * Initialize the HPET
+ */
+void
+hpet_init(void)
+{
+	unsigned int	*xmod;
+
+	map_rcbaArea();
+
+	/*
+	 * Is the HPET memory already enabled?
+	 * If not, set address and enable.
+	 */
+	xmod = (uint32_t *)(rcbaArea + 0x3404);	/* Point to the HPTC */
+	uint32_t hptc = *xmod;			/* Get HPET config */
+	DBG("    current RCBA.HPTC:  %08X\n", *xmod);
+	if(!(hptc & hptcAE)) {
+		DBG("HPET memory is not enabled, "
+		    "enabling and assigning to 0xFED00000 (hope that's ok)\n");
+		*xmod = (hptc & ~3) | hptcAE;
+	}
+
+	/*
+	 * Get physical address of HPET and map it.
+	 */
+	hpetAreap = hpetAddr | ((hptc & 3) << 12);
+	hpetArea = io_map_spec(hpetAreap & -4096, PAGE_SIZE * 4, VM_WIMG_IO);
+	kprintf("HPET: vaddr = %08X, paddr = %08X\n", hpetArea, hpetAreap);
+
+	/*
+	 * Extract the HPET tick rate.
+	 * The period of the HPET is reported in femtoseconds (10**-15s)
+	 * and convert to frequency in hertz.
+	 */
+	hpetFemto = (uint32_t)(((hpetReg_t *)hpetArea)->GCAP_ID >> 32);
+	hpetFreq = (1 * Peta) / hpetFemto;
+
+	/*
+	 * The conversion factor is the number of nanoseconds per HPET tick
+	 * with about 32 bits of fraction.  The value is converted to a
+	 * base-2 fixed point number.  To convert from HPET to nanoseconds,
+	 * multiply the value by the conversion factor using 96-bit arithmetic,
+	 * then shift right 32 bits.  If the value is known to be small,
+	 * 64-bit arithmetic will work.
+	 */
+
+	/*
+	 * Begin conversion of base 10 femtoseconds to base 2, calculate:
+	 *  - HPET ticks to nanoseconds conversion in base 2 fraction (* 2**32)
+	 *  - nanoseconds to HPET ticks conversion
+	 */
+	hpetCvtt2n = (uint64_t)hpetFemto << 32;
+	hpetCvtt2n = hpetCvtt2n / 1000000ULL;
+	hpetCvtn2t = 0xFFFFFFFFFFFFFFFFULL / hpetCvtt2n;
+	kprintf("HPET: Frequency = %6d.%04dMHz, "
+		"cvtt2n = %08X.%08X, cvtn2t = %08X.%08X\n",
+		(uint32_t)(hpetFreq / Mega), (uint32_t)(hpetFreq % Mega), 
+		(uint32_t)(hpetCvtt2n >> 32), (uint32_t)hpetCvtt2n,
+		(uint32_t)(hpetCvtn2t >> 32), (uint32_t)hpetCvtn2t);
+
+
+	/* (TAKE OUT LATER)
+	 * Begin conversion of base 10 femtoseconds to base 2
+	 * HPET ticks to nanoseconds in base 2 fraction (times 1048576)
+	 */
+	hpetCvt = (uint64_t)hpetFemto << 20;
+	hpetCvt = hpetCvt / 1000000ULL;
+
+	/* Calculate conversion from TSC to HPET */
+	tsc2hpet = tmrCvt(tscFCvtt2n, hpetCvtn2t);
+	DBG(" CVT: TSC to HPET = %08X.%08X\n",
+	    (uint32_t)(tsc2hpet >> 32), (uint32_t)tsc2hpet);
+
+	/* Calculate conversion from HPET to TSC */
+	hpet2tsc = tmrCvt(hpetCvtt2n, tscFCvtn2t);
+	DBG(" CVT: HPET to TSC = %08X.%08X\n",
+	    (uint32_t)(hpet2tsc >> 32), (uint32_t)hpet2tsc);
+
+	/* Calculate conversion from BUS to HPET */
+	bus2hpet = tmrCvt(busFCvtt2n, hpetCvtn2t);
+	DBG(" CVT: BUS to HPET = %08X.%08X\n",
+	    (uint32_t)(bus2hpet >> 32), (uint32_t)bus2hpet);
+
+	/* Calculate conversion from HPET to BUS */
+	hpet2bus = tmrCvt(hpetCvtt2n, busFCvtn2t);
+	DBG(" CVT: HPET to BUS = %08X.%08X\n",
+	    (uint32_t)(hpet2bus >> 32), (uint32_t)hpet2bus);
+
+	/* Make sure the counter is off in the HPET configuration flags */
+	uint64_t hpetcon = ((hpetReg_t *)hpetArea)->GEN_CONF;
+	hpetcon = hpetcon & ~1;
+	((hpetReg_t *)hpetArea)->GEN_CONF = hpetcon;
+
+	/*
+	 * Convert current TSC to HPET value,
+	 * set it, and start it ticking.
+	 */
+	uint64_t currtsc = rdtsc64();
+	uint64_t tscInHPET = tmrCvt(currtsc, tsc2hpet);
+	((hpetReg_t *)hpetArea)->MAIN_CNT = tscInHPET;
+	hpetcon = hpetcon | 1;
+	((hpetReg_t *)hpetArea)->GEN_CONF = hpetcon;
+	kprintf("HPET started: TSC = %08X.%08X, HPET = %08X.%08X\n", 
+		(uint32_t)(currtsc >> 32), (uint32_t)currtsc,
+		(uint32_t)(tscInHPET >> 32), (uint32_t)tscInHPET);
+
+#if MACH_KDB
+	db_display_hpet((hpetReg_t *)hpetArea);	/* (BRINGUP) */
+#endif
+}
+
+/*
+ * This routine is used to get various information about the HPET
+ * without having to export gobs of globals.  It fills in a data
+ * structure with the info.
+ */
+void
+hpet_get_info(hpetInfo_t *info)
+{
+    info->hpetCvtt2n = hpetCvtt2n;
+    info->hpetCvtn2t = hpetCvtn2t;
+    info->tsc2hpet   = tsc2hpet;
+    info->hpet2tsc   = hpet2tsc;
+    info->bus2hpet   = bus2hpet;
+    info->hpet2bus   = hpet2bus;
+    info->rcbaArea   = rcbaArea;
+    info->rcbaAreap  = rcbaAreap;
+}
+
+
+/*
+ * This routine is called by the HPET driver
+ * when it assigns an HPET timer to a processor
+ */
+
+void
+ml_hpet_cfg(uint32_t cpu, uint32_t hpetVect)
+{
+	uint64_t *hpetVaddr;
+	uint64_t hpetcnf;
+	
+	if(cpu > 1) {
+		panic("ml_hpet_cfg: invalid cpu = %d\n", cpu);
+	}
+
+	/* Calculate address of the HPET for this processor */
+	hpetVaddr = (uint64_t *)(((uint32_t)&(((hpetReg_t *)hpetArea)->TIM1_CONF)) + (cpu << 5));
+
+	DBG("ml_hpet_cfg: HPET for cpu %d at %08X, vector = %d\n",
+	     cpu, hpetVaddr, hpetVect);
+	
+	/* Save the address and vector of the HPET for this processor */
+	cpu_data_ptr[cpu]->cpu_pmHpet = (uint64_t *)hpetVaddr;
+	cpu_data_ptr[cpu]->cpu_pmHpetVec = hpetVect;
+
+	/* Enable the interruptions now that we have a vector */
+	hpetcnf = *hpetVaddr;
+	hpetcnf = hpetcnf | Tn_INT_ENB_CNF;
+	*hpetVaddr = hpetcnf;
+
+	/* Save the configuration */
+	cpu_data_ptr[cpu]->cpu_pmStats.pmHpetCfg = hpetcnf;
+	cpu_data_ptr[cpu]->cpu_pmStats.pmHpetCmp = 0;
+
+	/* See if nap policy has changed now */
+	machine_nap_policy();
+
+}
+
+/*
+ * This is the HPET interrupt handler.
+ *
+ * We really don't want to be here, but so far, I haven't figured out
+ * a way to cancel the interrupt. Hopefully, some day we will figure out
+ * how to do that or switch all timers to the HPET.
+ */
+int
+HPETInterrupt(void)
+{
+
+	/* All we do here is to bump the count */
+	current_cpu_datap()->cpu_pmStats.pmHPETRupt++;
+
+	/* Return and show that the 'rupt has been handled... */
+	return 1;
+}
+
+
+static hpetReg_t saved_hpet;
+
+void hpet_save( void )
+{
+	hpetReg_t	*from = (hpetReg_t *) hpetArea;
+	hpetReg_t	*to = &saved_hpet;
+
+	to->GEN_CONF  = from->GEN_CONF;
+	to->TIM0_CONF = from->TIM0_CONF;
+	to->TIM0_COMP = from->TIM0_COMP;
+	to->TIM1_CONF = from->TIM1_CONF;
+	to->TIM1_COMP = from->TIM1_COMP;
+	to->TIM2_CONF = from->TIM2_CONF;
+	to->TIM2_COMP = from->TIM2_COMP;
+	to->MAIN_CNT  = from->MAIN_CNT;
+}
+
+void hpet_restore( void )
+{
+	hpetReg_t	*from = &saved_hpet;
+	hpetReg_t	*to = (hpetReg_t *) hpetArea;
+
+	/*
+	 * Is the HPET memory already enabled?
+	 * If not, set address and enable.
+	 */
+	uint32_t *hptcp = (uint32_t *)(rcbaArea + 0x3404);
+	uint32_t hptc = *hptcp;
+	if(!(hptc & hptcAE)) {
+		DBG("HPET memory is not enabled, "
+		    "enabling and assigning to 0xFED00000 (hope that's ok)\n");
+		*hptcp = (hptc & ~3) | hptcAE;
+	}
+
+	to->GEN_CONF  = from->GEN_CONF & ~1;
+
+	to->TIM0_CONF = from->TIM0_CONF;
+	to->TIM0_COMP = from->TIM0_COMP;
+	to->TIM1_CONF = from->TIM1_CONF;
+	to->TIM1_COMP = from->TIM1_COMP;
+	to->TIM2_CONF = from->TIM2_CONF;
+	to->TIM2_COMP = from->TIM2_COMP;
+	to->GINTR_STA = -1ULL;
+	to->MAIN_CNT  = from->MAIN_CNT;
+
+	to->GEN_CONF = from->GEN_CONF;
+}
+
+/*
+ *      Read the HPET timer
+ *
+ */
+uint64_t
+rdHPET(void)
+{
+	hpetReg_t		*hpetp = (hpetReg_t *) hpetArea;
+	volatile uint32_t	*regp = (uint32_t *) &hpetp->MAIN_CNT;
+	uint32_t		high;
+	uint32_t		low;
+
+	do {
+		high = *(regp + 1);
+		low = *regp;
+	} while (high != *(regp + 1));
+
+	return (((uint64_t) high) << 32) | low;
+}
+
+#if MACH_KDB
+
+#define HI32(x)	((uint32_t)(((x) >> 32) & 0xFFFFFFFF))
+#define LO32(x)	((uint32_t)((x) & 0xFFFFFFFF))
+
+/*
+ *	Displays HPET memory mapped area
+ *	hp
+ */
+void 
+db_hpet(__unused db_expr_t addr, __unused int have_addr, __unused db_expr_t count, __unused char *modif)
+{
+
+	db_display_hpet((hpetReg_t *) hpetArea);	/* Dump out the HPET
+							 * stuff */
+	return;
+}
+
+void
+db_display_hpet(hpetReg_t * hpt)
+{
+
+	uint64_t        cmain;
+
+	cmain = hpt->MAIN_CNT;	/* Get the main timer */
+
+	/* General capabilities */
+	db_printf("  GCAP_ID = %08X.%08X\n",
+		  HI32(hpt->GCAP_ID), LO32(hpt->GCAP_ID));
+	/* General configuration */
+	db_printf(" GEN_CONF = %08X.%08X\n",
+		  HI32(hpt->GEN_CONF), LO32(hpt->GEN_CONF));
+	/* General Interrupt status */
+	db_printf("GINTR_STA = %08X.%08X\n",
+		  HI32(hpt->GINTR_STA), LO32(hpt->GINTR_STA));
+	/* Main counter */
+	db_printf(" MAIN_CNT = %08X.%08X\n",
+		  HI32(cmain), LO32(cmain));
+	/* Timer 0 config and cap */
+	db_printf("TIM0_CONF = %08X.%08X\n",
+		  HI32(hpt->TIM0_CONF), LO32(hpt->TIM0_CONF));
+	/* Timer 0 comparator */
+	db_printf("TIM0_COMP = %08X.%08X\n",
+		  HI32(hpt->TIM0_COMP), LO32(hpt->TIM0_COMP));
+	/* Timer 1 config and cap */
+	db_printf("TIM0_CONF = %08X.%08X\n",
+		  HI32(hpt->TIM1_CONF), LO32(hpt->TIM1_CONF));
+	/* Timer 1 comparator */
+	db_printf("TIM1_COMP = %08X.%08X\n",
+		  HI32(hpt->TIM1_COMP), LO32(hpt->TIM1_COMP));
+	/* Timer 2 config and cap */
+	db_printf("TIM2_CONF = %08X.%08X\n",
+		  HI32(hpt->TIM2_CONF), LO32(hpt->TIM2_CONF));
+	/* Timer 2 comparator */
+	db_printf("TIM2_COMP = %08X.%08X\n",
+		  HI32(hpt->TIM2_COMP), LO32(hpt->TIM2_COMP));
+
+	db_printf("\nHPET Frequency = %d.%05dMHz\n",
+	  (uint32_t) (hpetFreq / 1000000), (uint32_t) (hpetFreq % 1000000));
+
+	return;
+
+}
+
+#endif
diff --git a/osfmk/i386/hpet.h b/osfmk/i386/hpet.h
new file mode 100644
index 000000000..475ace610
--- /dev/null
+++ b/osfmk/i386/hpet.h
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifdef KERNEL_PRIVATE
+#ifndef _I386_HPET_H_
+#define _I386_HPET_H_
+
+/*
+ * HPET kernel functions to support the HPET KEXT and the
+ * power management KEXT.
+ */
+
+
+/*
+ *	Memory mapped registers for the HPET
+ */
+typedef struct hpetReg {
+	uint64_t	GCAP_ID;		/* General capabilities */
+	uint64_t	rsv1;
+	uint64_t	GEN_CONF;		/* General configuration */
+	uint64_t	rsv2;
+	uint64_t	GINTR_STA;		/* General Interrupt status */
+	uint64_t	rsv3[25];
+	uint64_t	MAIN_CNT;		/* Main counter */
+	uint64_t	rsv4;
+	uint64_t	TIM0_CONF;		/* Timer 0 config and cap */
+#define			TIM_CONF 0
+#define			Tn_INT_ENB_CNF 4
+	uint64_t	TIM0_COMP;		/* Timer 0 comparator */
+#define			TIM_COMP 8
+	uint64_t	rsv5[2];
+	uint64_t	TIM1_CONF;		/* Timer 1 config and cap */
+	uint64_t	TIM1_COMP;		/* Timer 1 comparator */
+	uint64_t	rsv6[2];
+	uint64_t	TIM2_CONF;		/* Timer 2 config and cap */
+	uint64_t	TIM2_COMP;		/* Timer 2 comparator */
+	uint64_t	rsv7[2];
+} hpetReg;
+typedef struct 	hpetReg hpetReg_t;
+
+struct hpetInfo
+{
+    uint64_t	hpetCvtt2n;
+    uint64_t	hpetCvtn2t;
+    uint64_t	tsc2hpet;
+    uint64_t	hpet2tsc;
+    uint64_t	bus2hpet;
+    uint64_t	hpet2bus;
+    uint32_t	rcbaArea;
+    uint32_t	rcbaAreap;
+};
+typedef struct hpetInfo hpetInfo_t;
+
+extern uint64_t hpetFemto;
+extern uint64_t hpetFreq;
+extern uint64_t hpetCvtt2n;
+extern uint64_t hpetCvtn2t;
+extern uint64_t tsc2hpet;
+extern uint64_t hpet2tsc;
+extern uint64_t bus2hpet;
+extern uint64_t hpet2bus;
+
+extern uint32_t rcbaArea;			
+extern uint32_t rcbaAreap;
+
+extern void map_rcbaAread(void);
+extern void hpet_init(void);
+
+extern void hpet_save(void);
+extern void hpet_restore(void);
+
+#ifdef XNU_KERNEL_PRIVATE
+extern int HPETInterrupt(void);
+#endif
+
+extern uint64_t rdHPET(void);
+extern void hpet_get_info(hpetInfo_t *info);
+
+#define hpetAddr 	0xFED00000
+#define hptcAE 		0x80
+
+#endif	/* _I386_HPET_H_ */
+
+#endif	/* KERNEL_PRIVATE */
diff --git a/osfmk/i386/i386_init.c b/osfmk/i386/i386_init.c
index 07560b1e9..947f434dc 100644
--- a/osfmk/i386/i386_init.c
+++ b/osfmk/i386/i386_init.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -56,7 +56,6 @@
 
 #include <platforms.h>
 #include <mach_kdb.h>
-#include <himem.h>
 
 #include <mach/i386/vm_param.h>
 
@@ -71,120 +70,104 @@
 #include <kern/misc_protos.h>
 #include <kern/startup.h>
 #include <kern/clock.h>
+#include <kern/pms.h>
 #include <kern/xpr.h>
 #include <kern/cpu_data.h>
 #include <kern/processor.h>
+#include <console/serial_protos.h>
 #include <vm/vm_page.h>
 #include <vm/pmap.h>
 #include <vm/vm_kern.h>
 #include <i386/fpu.h>
 #include <i386/pmap.h>
 #include <i386/ipl.h>
-#include <i386/pio.h>
 #include <i386/misc_protos.h>
 #include <i386/cpuid.h>
 #include <i386/mp.h>
+#include <i386/mp_desc.h>
 #include <i386/machine_routines.h>
+#include <i386/machine_check.h>
 #include <i386/postcode.h>
+#include <i386/Diagnostics.h>
+#include <i386/pmCPU.h>
+#include <i386/tsc.h>
+#include <i386/hpet.h>
 #if	MACH_KDB
 #include <ddb/db_aout.h>
 #endif /* MACH_KDB */
 #include <ddb/tr.h>
-#ifdef __MACHO__
-#include <mach/thread_status.h>
 
-static KernelBootArgs_t *kernelBootArgs;
-#endif
+static boot_args *kernelBootArgs;
 
-vm_offset_t	boot_args_start = 0;	/* pointer to kernel arguments, set in start.s */
-
-#ifdef __MACHO__
-#include	<mach-o/loader.h>
-vm_offset_t     edata, etext, end;
-
-/* operations only against currently loaded 32 bit mach kernel */
-extern struct segment_command *getsegbyname(const char *);
-extern struct section *firstsect(struct segment_command *);
-extern struct section *nextsect(struct segment_command *, struct section *);
-
-/*
- * Called first for a mach-o kernel before paging is set up.
- * Returns the first available physical address in memory.
- */
+extern int disableConsoleOutput;
+extern const char version[];
+extern const char version_variant[];
+extern int nx_enabled;
 
-void
-i386_preinit(void)
-{
-	struct segment_command	*sgp;
-	struct section		*sp;
-	struct KernelBootArgs *pp;
-	int i;
-
-	sgp = getsegbyname("__DATA");
-	if (sgp) {
-		sp = firstsect(sgp);
-		if (sp) {
-			do {
-				if ((sp->flags & S_ZEROFILL))
-					bzero((char *) sp->addr, sp->size);
-			} while ((sp = nextsect(sgp, sp)));
-		}
-	}
+extern int noVMX;	/* if set, rosetta should not emulate altivec */
 
-	kernelBootArgs = (KernelBootArgs_t *)
-		ml_static_ptovirt(boot_args_start);
-	pp = (struct KernelBootArgs *) kernelBootArgs;
-	pp->configEnd = (char *)
-		ml_static_ptovirt((vm_offset_t) pp->configEnd);
-	for (i = 0; i < pp->numBootDrivers; i++) {
-		pp->driverConfig[i].address = (unsigned)
-			ml_static_ptovirt(pp->driverConfig[i].address);
-	}
-	return;
-}
-#endif
+void cpu_stack_set(void);
 
-extern const char version[];
-extern const char version_variant[];
 
 /*
  *	Cpu initialization.  Running virtual, but without MACH VM
- *	set up.  First C routine called, unless i386_preinit() was called first.
+ *	set up.  First C routine called.
  */
 void
-i386_init(void)
+i386_init(vm_offset_t boot_args_start)
 {
 	unsigned int	maxmem;
+	uint64_t	maxmemtouse;
 	unsigned int	cpus;
+	boolean_t	legacy_mode;
 
 	postcode(I386_INIT_ENTRY);
 
-	master_cpu = 0;
-	cpu_data_alloc(TRUE);
-	cpu_init();
-	postcode(CPU_INIT_D);
+	i386_macho_zerofill();
+
+	/* Initialize machine-check handling */
+	mca_cpu_init();
 
 	/*
-	 * Setup some processor related structures to satisfy funnels.
-	 * Must be done before using unparallelized device drivers.
+	 * Setup boot args given the physical start address.
 	 */
-	processor_bootstrap();
+	kernelBootArgs = (boot_args *)
+		ml_static_ptovirt(boot_args_start);
+        kernelBootArgs->MemoryMap = (uint32_t)
+		ml_static_ptovirt((vm_offset_t)kernelBootArgs->MemoryMap);
+        kernelBootArgs->deviceTreeP = (uint32_t)
+		ml_static_ptovirt((vm_offset_t)kernelBootArgs->deviceTreeP);
 
+	master_cpu = 0;
+	(void) cpu_data_alloc(TRUE);
+	cpu_init();
+	postcode(CPU_INIT_D);
+
+	/* init processor performance control */
+	pmsInit();
+	
 	PE_init_platform(FALSE, kernelBootArgs);
 	postcode(PE_INIT_PLATFORM_D);
 
-	/*
-	 * Set up initial thread so current_thread() works early on
-	 */
-	thread_bootstrap();
-	postcode(THREAD_BOOTSTRAP_D);
-
 	printf_init();			/* Init this in case we need debugger */
 	panic_init();			/* Init this in case we need debugger */
 
 	/* setup debugging output if one has been chosen */
 	PE_init_kprintf(FALSE);
 
+	if (!PE_parse_boot_arg("diag", &dgWork.dgFlags))
+		dgWork.dgFlags = 0;
+
+	serialmode = 0;
+	if(PE_parse_boot_arg("serial", &serialmode)) {
+		/* We want a serial keyboard and/or console */
+		kprintf("Serial mode specified: %08X\n", serialmode);
+	}
+	if(serialmode & 1) {
+		(void)switch_to_serial_console();
+		disableConsoleOutput = FALSE;	/* Allow printfs to happen */
+	}
+
 	/* setup console output */
 	PE_init_printf(FALSE);
 
@@ -196,22 +179,55 @@ i386_init(void)
 	 * The maximum number of cpus must be set beforehand.
 	 */
 	if (!PE_parse_boot_arg("maxmem", &maxmem))
-		maxmem=0;
+		maxmemtouse=0;
 	else
-		maxmem = maxmem * (1024 * 1024);
+	        maxmemtouse = ((uint64_t)maxmem) * (uint64_t)(1024 * 1024);
 
 	if (PE_parse_boot_arg("cpus", &cpus)) {
 		if ((0 < cpus) && (cpus < max_ncpus))
                         max_ncpus = cpus;
 	}
 
-	i386_vm_init(maxmem, kernelBootArgs);
+	/*
+	 * debug support for > 4G systems
+	 */
+	if (!PE_parse_boot_arg("himemory_mode", &vm_himemory_mode))
+	        vm_himemory_mode = 0;
+
+	/*
+	 * At this point we check whether we are a 64-bit processor
+	 * and that we're not restricted to legacy mode, 32-bit operation.
+	 */
+	boolean_t IA32e = FALSE;
+	if (cpuid_extfeatures() & CPUID_EXTFEATURE_EM64T) {
+		kprintf("EM64T supported");
+		if (PE_parse_boot_arg("-legacy", &legacy_mode)) {
+		 	kprintf(" but legacy mode forced\n");
+		} else {
+			IA32e = TRUE;
+			kprintf(" and will be enabled\n");
+		}
+	}
+	if (!(cpuid_extfeatures() & CPUID_EXTFEATURE_XD))
+		nx_enabled = 0;
+
+	i386_vm_init(maxmemtouse, IA32e, kernelBootArgs);
+
+	if ( ! PE_parse_boot_arg("novmx", &noVMX))
+		noVMX = 0;	/* OK to support Altivec in rosetta? */
+
+	tsc_init();
+	hpet_init();
+	power_management_init();
 
 	PE_init_platform(TRUE, kernelBootArgs);
 
 	/* create the console for verbose or pretty mode */
 	PE_create_console();
 
+	processor_bootstrap();
+	thread_bootstrap();
+
 	machine_startup();
 
 }
diff --git a/osfmk/i386/i386_lock.s b/osfmk/i386/i386_lock.s
index 4c3fb99e1..1dfa8e787 100644
--- a/osfmk/i386/i386_lock.s
+++ b/osfmk/i386/i386_lock.s
@@ -130,11 +130,6 @@
 /*
  *  Routines for general lock debugging.
  */
-#define	S_TYPE		SLOCK_TYPE(%edx)
-#define	S_PC		SLOCK_PC(%edx)
-#define	S_THREAD	SLOCK_THREAD(%edx)
-#define	S_DURATIONH	SLOCK_DURATIONH(%edx)
-#define	S_DURATIONL	SLOCK_DURATIONL(%edx)
 
 /* 
  * Checks for expected lock types and calls "panic" on
@@ -152,17 +147,6 @@
 	.text						;	\
 1:
 
-#define	CHECK_SIMPLE_LOCK_TYPE()				\
-	cmpl	$ USLOCK_TAG,S_TYPE 			;	\
-	je	1f					;	\
-	pushl	$2f					;	\
-	call	EXT(panic)				;	\
-	hlt						;	\
-	.data						;	\
-2:	String	"not a simple lock!"			;	\
-	.text						;	\
-1:
-
 /*
  * If one or more simplelocks are currently held by a thread,
  * an attempt to acquire a mutex will cause this check to fail
@@ -267,8 +251,9 @@ LEAF_ENTRY(hw_lock_init)
 LEAF_ENTRY(hw_lock_lock)
 	movl	L_ARG0,%edx		/* fetch lock pointer */
 
-	movl	L_PC,%ecx
-1:	DISABLE_PREEMPTION
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
+	DISABLE_PREEMPTION
+1:
 	movl	0(%edx), %eax
 	testl	%eax,%eax		/* lock locked? */
 	jne	3f			/* branch if so */
@@ -276,8 +261,7 @@ LEAF_ENTRY(hw_lock_lock)
 	jne	3f
 	movl	$1,%eax			/* In case this was a timeout call */
 	LEAF_RET			/* if yes, then nothing left to do */
-
-3:	ENABLE_PREEMPTION		/* no reason we can't be preemptable */
+3:
 	PAUSE				/* pause for hyper-threading */
 	jmp	1b			/* try again */
 
@@ -290,7 +274,7 @@ LEAF_ENTRY(hw_lock_lock)
 LEAF_ENTRY(hw_lock_to)
 1:
 	movl	L_ARG0,%edx		/* fetch lock pointer */
-	movl	L_PC,%ecx
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 	/*
 	 * Attempt to grab the lock immediately
 	 * - fastpath without timeout nonsense.
@@ -321,8 +305,6 @@ LEAF_ENTRY(hw_lock_to)
 	adcl	$0,%edx			/* add carry */
 	mov	%edx,%ecx
 	mov	%eax,%ebx		/* %ecx:%ebx is the timeout expiry */
-3:
-	ENABLE_PREEMPTION		/* no reason not to be preempted now */
 4:
 	/*
 	 * The inner-loop spin to look for the lock being freed.
@@ -343,7 +325,7 @@ LEAF_ENTRY(hw_lock_to)
 	cmpl	%ecx,%edx		/* compare high-order 32-bits */
 	jb	4b			/* continue spinning if less, or */
 	cmpl	%ebx,%eax		/* compare low-order 32-bits */ 
-	jb	5b			/* continue if less, else bail */
+	jb	4b			/* continue if less, else bail */
 	xor	%eax,%eax		/* with 0 return value */
 	pop	%ebx
 	pop	%edi
@@ -354,10 +336,9 @@ LEAF_ENTRY(hw_lock_to)
 	 * Here to try to grab the lock that now appears to be free
 	 * after contention.
 	 */
-	movl	8+L_PC,%edx		/* calling pc (8+ for pushed regs) */
-	DISABLE_PREEMPTION
+	movl	%gs:CPU_ACTIVE_THREAD,%edx
 	lock; cmpxchgl	%edx,0(%edi)	/* try to acquire the HW lock */
-	jne	3b			/* no - spin again */
+	jne	4b			/* no - spin again */
 	movl	$1,%eax			/* yes */
 	pop	%ebx
 	pop	%edi
@@ -375,6 +356,26 @@ LEAF_ENTRY(hw_lock_unlock)
 	ENABLE_PREEMPTION
 	LEAF_RET
 
+/*
+ *	void i386_lock_unlock_with_flush(hw_lock_t)
+ *
+ *	Unconditionally release lock, followed by a cacheline flush of
+ *	the line corresponding to the lock dword. This routine is currently
+ *	used with certain locks which are susceptible to lock starvation,
+ *	minimizing cache affinity for lock acquisitions. A queued spinlock
+ *	or other mechanism that ensures fairness would obviate the need
+ *	for this routine, but ideally few or no spinlocks should exhibit
+ *	enough contention to require such measures.
+ *	MACH_RT:  release preemption level.
+ */
+LEAF_ENTRY(i386_lock_unlock_with_flush)
+	movl	L_ARG0,%edx		/* Fetch lock pointer */
+	movl	$0,0(%edx)		/* Clear the lock */
+	mfence				/* Serialize prior stores */
+	clflush	0(%edx)			/* Write back and invalidate line */
+	ENABLE_PREEMPTION
+	LEAF_RET
+
 /*
  *	unsigned int hw_lock_try(hw_lock_t)
  *	MACH_RT:  returns with preemption disabled on success.
@@ -382,7 +383,7 @@ LEAF_ENTRY(hw_lock_unlock)
 LEAF_ENTRY(hw_lock_try)
 	movl	L_ARG0,%edx		/* fetch lock pointer */
 
-	movl	L_PC,%ecx
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 	DISABLE_PREEMPTION
 	movl	0(%edx),%eax
 	testl	%eax,%eax
@@ -393,7 +394,8 @@ LEAF_ENTRY(hw_lock_try)
 	movl	$1,%eax			/* success */
 	LEAF_RET
 
-1:	ENABLE_PREEMPTION		/* failure:  release preemption... */
+1:
+	ENABLE_PREEMPTION		/* failure:  release preemption... */
 	xorl	%eax,%eax		/* ...and return failure */
 	LEAF_RET
 
@@ -437,23 +439,21 @@ NONLEAF_ENTRY2(mutex_lock,_mutex_lock)
 
 	pushf				/* save interrupt state */
 	cli				/* disable interrupts */
+Lml_retry:
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
-ml_retry:
-	movl	B_PC,%ecx
-
-ml_get_hw:
+Lml_get_hw:
 	movl	M_ILK,%eax		/* read interlock */
 	testl	%eax,%eax		/* unlocked? */
-	je	1f			/* yes - attempt to lock it */
-	PAUSE				/* no  - pause */
-	jmp	ml_get_hw		/* try again */
-1:
+	jne	Lml_ilk_fail		/* no - take the slow path */
+
 	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
-	jne	ml_get_hw		/* branch on failure to retry */
+	jne	Lml_get_hw		/* branch on failure to retry */
 
 	movl	M_LOCKED,%ecx		/* get lock owner */
 	testl	%ecx,%ecx		/* is the mutex locked? */
-	jne	ml_fail			/* yes, we lose */
+	jne	Lml_fail		/* yes, we lose */
+Lml_acquire:
 	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 	movl	%ecx,M_LOCKED
 
@@ -463,28 +463,94 @@ ml_get_hw:
 	movl	%ecx,M_PC
 #endif
 
+	cmpw	$0,M_WAITERS		/* are there any waiters? */
+	jne	Lml_waiters		/* yes, more work to do */
+Lml_return:
+	xorl	%eax,%eax
+	movl	%eax,M_ILK
+
+	popf				/* restore interrupt state */
+
+	NONLEAF_RET
+
+Lml_waiters:
 	pushl	%edx			/* save mutex address */
 	pushl	%edx
 	call	EXT(lck_mtx_lock_acquire)
 	addl	$4,%esp
 	popl	%edx			/* restore mutex address */
+	jmp	Lml_return
+
+Lml_ilk_fail:
+	/*
+	 * Slow path: call out to do the spinning.
+	 */
+	pushl	%edx			/* lock address */
+	call	EXT(lck_mtx_interlock_spin)
+	popl	%edx			/* lock pointer */
+	jmp	Lml_retry		/* try again */
 
+Lml_fail:
+	/*
+	 n Check if the owner is on another processor and therefore
+	 * we should try to spin before blocking.
+	 */
+	testl	$(OnProc),ACT_SPF(%ecx)
+	jz	Lml_block
+
+	/*
+	 * Here if owner is on another processor:
+	 *  - release the interlock
+	 *  - spin on the holder until release or timeout
+	 *  - in either case re-acquire the interlock
+	 *  - if released, acquire it
+	 *  - otherwise drop thru to block.
+	 */
 	xorl	%eax,%eax
-	movl	%eax,M_ILK
+	movl	%eax,M_ILK		/* zero interlock */
+	popf
+	pushf				/* restore interrupt state */
 
-	popf				/* restore interrupt state */
+	push	%edx			/* lock address */
+	call	EXT(lck_mtx_lock_spin)	/* call out to do spinning */
+	addl	$4,%esp
+	movl	B_ARG0,%edx		/* refetch mutex address */
 
-	NONLEAF_RET
+	/* Re-acquire interlock */
+	cli				/* disable interrupts */
+Lml_reget_retry:
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
+
+Lml_reget_hw:
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	jne	Lml_ilk_refail		/* no - slow path */
+
+	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
+	jne	Lml_reget_hw		/* branch on failure to retry */
 
-ml_fail:
-ml_block:
+	movl	M_LOCKED,%ecx		/* get lock owner */
+	testl	%ecx,%ecx		/* is the mutex free? */
+	je	Lml_acquire		/* yes, acquire */
+	
+Lml_block:
 	CHECK_MYLOCK(M_THREAD)
 	pushl	M_LOCKED
 	pushl	%edx			/* push mutex address */
 	call	EXT(lck_mtx_lock_wait)	/* wait for the lock */
 	addl	$8,%esp
 	movl	B_ARG0,%edx		/* refetch mutex address */
-	jmp	ml_retry		/* and try again */
+	cli				/* ensure interrupts disabled */
+	jmp	Lml_retry		/* and try again */
+
+Lml_ilk_refail:
+	/*
+	 * Slow path: call out to do the spinning.
+	 */
+	pushl	%edx			/* lock address */
+	call	EXT(lck_mtx_interlock_spin)
+	popl	%edx			/* lock pointer */
+	jmp	Lml_reget_retry		/* try again */
 
 NONLEAF_ENTRY2(mutex_try,_mutex_try)	
 
@@ -493,24 +559,22 @@ NONLEAF_ENTRY2(mutex_try,_mutex_try)
 	CHECK_MUTEX_TYPE()
 	CHECK_NO_SIMPLELOCKS()
 
-	movl	B_PC,%ecx
-
 	pushf				/* save interrupt state */
 	cli				/* disable interrupts */
+Lmt_retry:
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
-mt_get_hw:
+Lmt_get_hw:
 	movl	M_ILK,%eax		/* read interlock */
 	testl	%eax,%eax		/* unlocked? */
-	je	1f			/* yes - attempt to lock it */
-	PAUSE				/* no  - pause */
-	jmp	mt_get_hw		/* try again */
-1:
+	jne	Lmt_ilk_fail		/* no - slow path */
+
 	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
-	jne	mt_get_hw		/* branch on failure to retry */
+	jne	Lmt_get_hw		/* branch on failure to retry */
 
 	movl	M_LOCKED,%ecx		/* get lock owner */
 	testl	%ecx,%ecx		/* is the mutex locked? */
-	jne	mt_fail			/* yes, we lose */
+	jne	Lmt_fail		/* yes, we lose */
 	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 	movl	%ecx,M_LOCKED
 
@@ -520,22 +584,35 @@ mt_get_hw:
 	movl	%ecx,M_PC
 #endif
 
-	pushl	%edx			/* save mutex address */
-	pushl	%edx
-	call	EXT(lck_mtx_lock_acquire)
-	addl	$4,%esp
-	popl	%edx			/* restore mutex address */
-
+	cmpl	$0,M_WAITERS		/* are there any waiters? */
+	jne	Lmt_waiters		/* yes, more work to do */
+Lmt_return:
 	xorl	%eax,%eax
 	movl	%eax,M_ILK
-
 	popf				/* restore interrupt state */
 
 	movl	$1,%eax
 
 	NONLEAF_RET
 
-mt_fail:
+Lmt_waiters:
+	pushl	%edx			/* save mutex address */
+	pushl	%edx
+	call	EXT(lck_mtx_lock_acquire)
+	addl	$4,%esp
+	popl	%edx			/* restore mutex address */
+	jmp	Lmt_return
+
+Lmt_ilk_fail:
+	/*
+	 * Slow path: call out to do the spinning.
+	 */
+	pushl	%edx			/* lock address */
+	call	EXT(lck_mtx_interlock_spin)
+	popl	%edx			/* lock pointer */
+	jmp	Lmt_retry		/* try again */
+
+Lmt_fail:
 	xorl	%eax,%eax
 	movl	%eax,M_ILK
 
@@ -551,25 +628,23 @@ NONLEAF_ENTRY(mutex_unlock)
 	CHECK_MUTEX_TYPE()
 	CHECK_THREAD(M_THREAD)
 
-	movl	B_PC,%ecx
-
 	pushf				/* save interrupt state */
 	cli				/* disable interrupts */
+Lmu_retry:
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
-mu_get_hw:
+Lmu_get_hw:
 	movl	M_ILK,%eax		/* read interlock */
 	testl	%eax,%eax		/* unlocked? */
-	je	1f			/* yes - attempt to lock it */
-	PAUSE				/* no  - pause */
-	jmp	mu_get_hw		/* try again */
-1:
+	jne	Lmu_ilk_fail		/* no - slow path */
+
 	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
-	jne	mu_get_hw		/* branch on failure to retry */
+	jne	Lmu_get_hw		/* branch on failure to retry */
 
 	cmpw	$0,M_WAITERS		/* are there any waiters? */
-	jne	mu_wakeup		/* yes, more work to do */
+	jne	Lmu_wakeup		/* yes, more work to do */
 
-mu_doit:
+Lmu_doit:
 
 #if	MACH_LDEBUG
 	movl	$0,M_THREAD		/* disown thread */
@@ -584,13 +659,22 @@ mu_doit:
 
 	NONLEAF_RET
 
-mu_wakeup:
+Lmu_ilk_fail:
+	/*
+	 * Slow path: call out to do the spinning.
+	 */
+	pushl	%edx			/* lock address */
+	call	EXT(lck_mtx_interlock_spin)
+	popl	%edx			/* lock pointer */
+	jmp	Lmu_retry		/* try again */
+
+Lmu_wakeup:
 	pushl	M_LOCKED
 	pushl	%edx			/* push mutex address */
 	call	EXT(lck_mtx_unlock_wakeup)/* yes, wake a thread */
 	addl	$8,%esp
 	movl	B_ARG0,%edx		/* restore lock pointer */
-	jmp	mu_doit
+	jmp	Lmu_doit
 
 /*
  * lck_mtx_lock()
@@ -611,40 +695,96 @@ NONLEAF_ENTRY(lck_mtx_lock)
 
 	pushf				/* save interrupt state */
 	cli				/* disable interrupts */
+Llml_retry:
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
-lml_retry:
-	movl	B_PC,%ecx
-
-lml_get_hw:
+Llml_get_hw:
 	movl	M_ILK,%eax		/* read interlock */
 	testl	%eax,%eax		/* unlocked? */
-	je	1f			/* yes - attempt to lock it */
-	PAUSE				/* no  - pause */
-	jmp	lml_get_hw		/* try again */
-1:
+	jne	Llml_ilk_fail		/* no - slow path */
+
 	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
-	jne	lml_get_hw		/* branch on failure to retry */
+	jne	Llml_get_hw		/* branch on failure to retry */
 
 	movl	M_LOCKED,%ecx		/* get lock owner */
 	testl	%ecx,%ecx		/* is the mutex locked? */
-	jne	lml_fail		/* yes, we lose */
+	jne	Llml_fail		/* yes, we lose */
+Llml_acquire:
 	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 	movl	%ecx,M_LOCKED
 
+	cmpl	$0,M_WAITERS		/* are there any waiters? */
+	jne	Llml_waiters		/* yes, more work to do */
+Llml_return:
+	xorl	%eax,%eax
+	movl	%eax,M_ILK
+
+	popf				/* restore interrupt state */
+
+	NONLEAF_RET
+
+Llml_waiters:
 	pushl	%edx			/* save mutex address */
 	pushl	%edx
 	call	EXT(lck_mtx_lock_acquire)
 	addl	$4,%esp
 	popl	%edx			/* restore mutex address */
+	jmp	Llml_return
+
+Llml_ilk_fail:
+	/*
+	 * Slow path: call out to do the spinning.
+	 */
+	pushl	%edx			/* lock address */
+	call	EXT(lck_mtx_interlock_spin)
+	popl	%edx			/* lock pointer */
+	jmp	Llml_retry		/* try again */
+
+Llml_fail:
+	/*
+	 * Check if the owner is on another processor and therefore
+	 * we should try to spin before blocking.
+	 */
+	testl	$(OnProc),ACT_SPF(%ecx)
+	jz	Llml_block
 
+	/*
+	 * Here if owner is on another processor:
+	 *  - release the interlock
+	 *  - spin on the holder until release or timeout
+	 *  - in either case re-acquire the interlock
+	 *  - if released, acquire it
+	 *  - otherwise drop thru to block.
+	 */
 	xorl	%eax,%eax
-	movl	%eax,M_ILK
+	movl	%eax,M_ILK		/* zero interlock */
+	popf
+	pushf				/* restore interrupt state */
 
-	popf				/* restore interrupt state */
+	pushl	%edx			/* save mutex address */
+	pushl	%edx
+	call	EXT(lck_mtx_lock_spin)
+	addl	$4,%esp
+	popl	%edx			/* restore mutex address */
 
-	NONLEAF_RET
+	/* Re-acquire interlock */
+	cli				/* disable interrupts */
+Llml_reget_retry:
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
+
+Llml_reget_hw:
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	jne	Llml_ilk_refail		/* no - slow path */
+
+	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
+	jne	Llml_reget_hw		/* branch on failure to retry */
 
-lml_fail:
+	movl	M_LOCKED,%ecx		/* get lock owner */
+	testl	%ecx,%ecx		/* is the mutex free? */
+	je	Llml_acquire		/* yes, acquire */
+	
+Llml_block:
 	CHECK_MYLOCK(M_THREAD)
 	pushl	%edx			/* save mutex address */
 	pushl	M_LOCKED
@@ -652,7 +792,17 @@ lml_fail:
 	call	EXT(lck_mtx_lock_wait)	/* wait for the lock */
 	addl	$8,%esp
 	popl	%edx			/* restore mutex address */
-	jmp	lml_retry		/* and try again */
+	cli				/* ensure interrupts disabled */
+	jmp	Llml_retry		/* and try again */
+
+Llml_ilk_refail:
+	/*
+	 * Slow path: call out to do the spinning.
+	 */
+	pushl	%edx			/* lock address */
+	call	EXT(lck_mtx_interlock_spin)
+	popl	%edx			/* lock pointer */
+	jmp	Llml_reget_retry	/* try again */
 
 NONLEAF_ENTRY(lck_mtx_try_lock)
 
@@ -663,33 +813,28 @@ NONLEAF_ENTRY(lck_mtx_try_lock)
 	CHECK_NO_SIMPLELOCKS()
 	CHECK_PREEMPTION_LEVEL()
 
-	movl	B_PC,%ecx
-
 	pushf				/* save interrupt state */
 	cli				/* disable interrupts */
+Llmt_retry:
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
-lmt_get_hw:
+Llmt_get_hw:
 	movl	M_ILK,%eax		/* read interlock */
 	testl	%eax,%eax		/* unlocked? */
-	je	1f			/* yes - attempt to lock it */
-	PAUSE				/* no  - pause */
-	jmp	lmt_get_hw		/* try again */
-1:
+	jne	Llmt_ilk_fail		/* no - slow path */
+
 	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
-	jne	lmt_get_hw		/* branch on failure to retry */
+	jne	Llmt_get_hw		/* branch on failure to retry */
 
 	movl	M_LOCKED,%ecx		/* get lock owner */
 	testl	%ecx,%ecx		/* is the mutex locked? */
-	jne	lmt_fail		/* yes, we lose */
+	jne	Llmt_fail		/* yes, we lose */
 	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 	movl	%ecx,M_LOCKED
 
-	pushl	%edx			/* save mutex address */
-	pushl	%edx
-	call	EXT(lck_mtx_lock_acquire)
-	addl	$4,%esp
-	popl	%edx			/* restore mutex address */
-
+	cmpl	$0,M_WAITERS		/* are there any waiters? */
+	jne	Llmt_waiters		/* yes, more work to do */
+Llmt_return:
 	xorl	%eax,%eax
 	movl	%eax,M_ILK
 
@@ -698,7 +843,24 @@ lmt_get_hw:
 	movl	$1,%eax			/* return success */
 	NONLEAF_RET
 
-lmt_fail:
+Llmt_waiters:
+	pushl	%edx			/* save mutex address */
+	pushl	%edx
+	call	EXT(lck_mtx_lock_acquire)
+	addl	$4,%esp
+	popl	%edx			/* restore mutex address */
+	jmp	Llmt_return
+
+Llmt_ilk_fail:
+	/*
+	 * Slow path: call out to do the spinning.
+	 */
+	pushl	%edx			/* lock address */
+	call	EXT(lck_mtx_interlock_spin)
+	popl	%edx			/* lock pointer */
+	jmp	Llmt_retry		/* try again */
+
+Llmt_fail:
 	xorl	%eax,%eax
 	movl	%eax,M_ILK
 
@@ -713,25 +875,23 @@ NONLEAF_ENTRY(lck_mtx_unlock)
 	cmpl	$(MUTEX_IND),M_ITAG	/* is this indirect? */
 	cmove	M_PTR,%edx		/* yes - take indirection */
 
-	movl	B_PC,%ecx
-
 	pushf				/* save interrupt state */
 	cli				/* disable interrupts */
+Llmu_retry:
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
-lmu_get_hw:
+Llmu_get_hw:
 	movl	M_ILK,%eax		/* read interlock */
 	testl	%eax,%eax		/* unlocked? */
-	je	1f			/* yes - attempt to lock it */
-	PAUSE				/* no  - pause */
-	jmp	lmu_get_hw		/* try again */
-1:
+	jne	Llmu_ilk_fail		/* no - slow path */
+
 	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
-	jne	lmu_get_hw		/* branch on failure to retry */
+	jne	Llmu_get_hw		/* branch on failure to retry */
 
 	cmpw	$0,M_WAITERS		/* are there any waiters? */
-	jne	lmu_wakeup		/* yes, more work to do */
+	jne	Llmu_wakeup		/* yes, more work to do */
 
-lmu_doit:
+Llmu_doit:
 	xorl	%ecx,%ecx
 	movl	%ecx,M_LOCKED		/* unlock the mutex */
 
@@ -741,14 +901,23 @@ lmu_doit:
 
 	NONLEAF_RET
 
-lmu_wakeup:
+Llmu_ilk_fail:
+	/*
+	 * Slow path: call out to do the spinning.
+	 */
+	pushl	%edx			/* lock address */
+	call	EXT(lck_mtx_interlock_spin)
+	popl	%edx			/* lock pointer */
+	jmp	Llmu_retry		/* try again */
+
+Llmu_wakeup:
 	pushl	%edx			/* save mutex address */
 	pushl	M_LOCKED
 	pushl	%edx			/* push mutex address */
 	call	EXT(lck_mtx_unlock_wakeup)/* yes, wake a thread */
 	addl	$8,%esp
 	popl	%edx			/* restore mutex pointer */
-	jmp	lmu_doit
+	jmp	Llmu_doit
 
 LEAF_ENTRY(lck_mtx_ilk_unlock)
 	movl	L_ARG0,%edx		/* no indirection here */
diff --git a/iokit/Drivers/platform/drvAppleIntelClock/AppleIntelClock.h b/osfmk/i386/i386_lowmem.h
similarity index 79%
rename from iokit/Drivers/platform/drvAppleIntelClock/AppleIntelClock.h
rename to osfmk/i386/i386_lowmem.h
index 8e293fc56..ffbb00b57 100644
--- a/iokit/Drivers/platform/drvAppleIntelClock/AppleIntelClock.h
+++ b/osfmk/i386/i386_lowmem.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -26,17 +26,17 @@
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-#ifndef _APPLEINTELCLOCK_H
-#define _APPLEINTELCLOCK_H
+#ifndef _I386_LOWMEM_H_
+#define _I386_LOWMEM_H_
 
-#include <IOKit/IOService.h>
 
-class AppleIntelClock : public IOService
-{
-  OSDeclareDefaultStructors(AppleIntelClock);
+#ifdef __APPLE_API_PRIVATE
 
-public:
-  virtual bool start(IOService * provider);
-};
+#define I386_LOWMEM_RESERVED	    0x18
 
-#endif /* _APPLEINTELCLOCK_H */
+#define I386_HIB_PAGETABLE	    0x13
+#define I386_HIB_PAGETABLE_COUNT    5
+
+#endif /* __APPLE_API_PRIVATE */
+
+#endif /* !_I386_LOWMEM_H_ */
diff --git a/osfmk/i386/i386_vm_init.c b/osfmk/i386/i386_vm_init.c
index b643a84fc..e8a9a8027 100644
--- a/osfmk/i386/i386_vm_init.c
+++ b/osfmk/i386/i386_vm_init.c
@@ -56,7 +56,6 @@
 
 #include <platforms.h>
 #include <mach_kdb.h>
-#include <himem.h>
 
 #include <mach/i386/vm_param.h>
 
@@ -76,32 +75,38 @@
 #include <vm/vm_kern.h>
 #include <i386/pmap.h>
 #include <i386/ipl.h>
-#include <i386/pio.h>
 #include <i386/misc_protos.h>
 #include <i386/mp_slave_boot.h>
 #include <i386/cpuid.h>
-#ifdef __MACHO__
 #include <mach/thread_status.h>
-#endif
+#include <pexpert/i386/efi.h>
+#include "i386_lowmem.h"
 
 vm_size_t	mem_size = 0; 
 vm_offset_t	first_avail = 0;/* first after page tables */
 vm_offset_t	last_addr;
 
-uint64_t        max_mem;
-uint64_t        sane_size = 0; /* we are going to use the booter memory
-				  table info to construct this */
+uint64_t	max_mem;        /* Size of physical memory (bytes), adjusted by maxmem */
+uint64_t        mem_actual;
+uint64_t	sane_size = 0;  /* Memory size to use for defaults calculations */
+
+#define MAXBOUNCEPOOL	(128 * 1024 * 1024)
+#define MAXLORESERVE	( 32 * 1024 * 1024)
+
+extern int bsd_mbuf_cluster_reserve(void);
+
+
+uint32_t	bounce_pool_base = 0;
+uint32_t	bounce_pool_size = 0;
 
-pmap_paddr_t     avail_start, avail_end;
+static void	reserve_bouncepool(uint32_t);
+
+
+pmap_paddr_t	avail_start, avail_end;
 vm_offset_t	virtual_avail, virtual_end;
-pmap_paddr_t	avail_remaining;
+static pmap_paddr_t	avail_remaining;
 vm_offset_t     static_memory_end = 0;
 
-#ifndef __MACHO__
-extern char	edata, end;
-#endif
-
-#ifdef __MACHO__
 #include	<mach-o/loader.h>
 vm_offset_t	edata, etext, end;
 
@@ -118,22 +123,55 @@ void *sectPRELINKB; int sectSizePRELINK;
 void *sectHIBB; int sectSizeHIB;
 
 extern void *getsegdatafromheader(struct mach_header *, const char *, int *);
-#endif
+extern struct segment_command *getsegbyname(const char *);
+extern struct section *firstsect(struct segment_command *);
+extern struct section *nextsect(struct segment_command *, struct section *);
+
+
+void
+i386_macho_zerofill(void)
+{
+	struct segment_command	*sgp;
+	struct section		*sp;
+
+	sgp = getsegbyname("__DATA");
+	if (sgp) {
+		sp = firstsect(sgp);
+		if (sp) {
+			do {
+				if ((sp->flags & S_ZEROFILL))
+					bzero((char *) sp->addr, sp->size);
+			} while ((sp = nextsect(sgp, sp)));
+		}
+	}
+
+	return;
+}
 
 /*
  * Basic VM initialization.
  */
 void
-i386_vm_init(unsigned int maxmem, KernelBootArgs_t *args)
+i386_vm_init(uint64_t	maxmem,
+	     boolean_t	IA32e,
+	     boot_args	*args)
 {
 	pmap_memory_region_t *pmptr;
-	MemoryRange *mptr;
+        pmap_memory_region_t *prev_pmptr;
+	EfiMemoryRange *mptr;
+        unsigned int mcount;
+        unsigned int msize;
 	ppnum_t fap;
 	unsigned int i;
-	ppnum_t maxpg = (maxmem >> I386_PGSHIFT);
+	unsigned int safeboot;
+	ppnum_t maxpg = 0;
+        uint32_t pmap_type;
+	uint32_t maxbouncepoolsize;
+	uint32_t maxloreserve;
+	uint32_t maxdmaaddr;
 
-#ifdef	__MACHO__
-	/* Now retrieve addresses for end, edata, and etext 
+	/*
+	 * Now retrieve addresses for end, edata, and etext 
 	 * from MACH-O headers.
 	 */
 
@@ -152,279 +190,336 @@ i386_vm_init(unsigned int maxmem, KernelBootArgs_t *args)
 
 	etext = (vm_offset_t) sectTEXTB + sectSizeTEXT;
 	edata = (vm_offset_t) sectDATAB + sectSizeDATA;
-#endif
-#ifndef	__MACHO__
-	/*
-	 * Zero the BSS.
-	 */
-
-	bzero((char *)&edata,(unsigned)(&end - &edata));
-#endif
-
-	/*
-	 * Initialize the pic prior to any possible call to an spl.
-	 */
 
-	set_cpu_model();
 	vm_set_page_size();
 
 	/*
 	 * Compute the memory size.
 	 */
 
+	if ((1 == vm_himemory_mode) || PE_parse_boot_arg("-x", &safeboot)) {
+	        maxpg = 1 << (32 - I386_PGSHIFT);
+	}
 	avail_remaining = 0;
 	avail_end = 0;
 	pmptr = pmap_memory_regions;
+        prev_pmptr = 0;
 	pmap_memory_region_count = pmap_memory_region_current = 0;
 	fap = (ppnum_t) i386_btop(first_avail);
-	mptr = args->memoryMap;
 
-#ifdef PAE
-#define FOURGIG 0x0000000100000000ULL
-	for (i=0; i < args->memoryMapCount; i++,mptr++) {
-	  ppnum_t base, top;
-
-	  base = (ppnum_t) (mptr->base >> I386_PGSHIFT);
-	  top = (ppnum_t) ((mptr->base + mptr->length) >> I386_PGSHIFT) - 1;
-
-	  if (maxmem) {
-	    if (base >= maxpg) break;
-	    top = (top > maxpg)? maxpg : top;
-	  }
-
-	  if (kMemoryRangeUsable != mptr->type) continue;
-	  sane_size += (uint64_t)(mptr->length);
-#ifdef DEVICES_HANDLE_64BIT_IO  /* XXX enable else clause  when I/O to high memory works */
-	  if (top < fap) {
-	    /* entire range below first_avail */
-	    continue;
-	  } else if (mptr->base >= FOURGIG) {
-	    /* entire range above 4GB (pre PAE) */
-	    continue;
-	  } else if ( (base < fap) &&
-		      (top > fap)) {
-	    /* spans first_avail */
-	    /*  put mem below first avail in table but
-		mark already allocated */
-	    pmptr->base = base;
-	    pmptr->alloc = pmptr->end = (fap - 1);
-            pmptr->type = mptr->type;
-	    /* we bump these here inline so the accounting below works
-	       correctly */
-	    pmptr++;
-	    pmap_memory_region_count++;
-	    pmptr->alloc = pmptr->base = fap;
-            pmptr->type = mptr->type;
-	    pmptr->end = top;
-	  } else if ( (mptr->base < FOURGIG) &&
-		      ((mptr->base+mptr->length) > FOURGIG) ) {
-	    /* spans across 4GB (pre PAE) */
-	    pmptr->alloc = pmptr->base = base;
-            pmptr->type = mptr->type;
-	    pmptr->end = (FOURGIG >> I386_PGSHIFT) - 1;
-	  } else {
-	    /* entire range useable */
-	    pmptr->alloc = pmptr->base = base;
-            pmptr->type = mptr->type;
-	    pmptr->end = top;
-	  }
-#else
-	  if (top < fap) {
-	    /* entire range below first_avail */
-	    continue;
-	  } else if ( (base < fap) &&
-		      (top > fap)) {
-	    /* spans first_avail */
-	    pmptr->alloc = pmptr->base = fap;
-            pmptr->type = mptr->type;
-	    pmptr->end = top;
-	  } else {
-	    /* entire range useable */
-	    pmptr->alloc = pmptr->base = base;
-            pmptr->type = mptr->type;
-	    pmptr->end = top;
-	  }
-#endif
-	  if (i386_ptob(pmptr->end) > avail_end ) {
-	    avail_end = i386_ptob(pmptr->end);
-	  }
-	  avail_remaining += (pmptr->end - pmptr->base);
-	  pmap_memory_region_count++;
-	  pmptr++;
-	}
-#else  /* non PAE follows */
+	mptr = (EfiMemoryRange *)args->MemoryMap;
+        if (args->MemoryMapDescriptorSize == 0)
+	        panic("Invalid memory map descriptor size");
+        msize = args->MemoryMapDescriptorSize;
+        mcount = args->MemoryMapSize / msize;
+
 #define FOURGIG 0x0000000100000000ULL
-	for (i=0; i < args->memoryMapCount; i++,mptr++) {
-	  ppnum_t base, top;
-
-	  base = (ppnum_t) (mptr->base >> I386_PGSHIFT);
-	  top = (ppnum_t) ((mptr->base + mptr->length) >> I386_PGSHIFT) - 1;
-
-	  if (maxmem) {
-	    if (base >= maxpg) break;
-	    top = (top > maxpg)? maxpg : top;
-	  }
-
-	  if (kMemoryRangeUsable != mptr->type) continue;
-
-          // save other regions
-          if (kMemoryRangeNVS == mptr->type) {
-              // Mark this as a memory range (for hibernation),
-              // but don't count as usable memory
-              pmptr->base = base;
-              pmptr->end = ((mptr->base + mptr->length + I386_PGBYTES - 1) >> I386_PGSHIFT) - 1;
-              pmptr->alloc = pmptr->end;
-              pmptr->type = mptr->type;
-              kprintf("NVS region: 0x%x ->0x%x\n", pmptr->base, pmptr->end);
-          } else if (kMemoryRangeUsable != mptr->type) {
-              continue;
-          } else {
-              // Usable memory region
-	  sane_size += (uint64_t)(mptr->length);
-	  if (top < fap) {
-	    /* entire range below first_avail */
-	    /* salvage some low memory pages */
-	    /* we use some very low memory at startup */
-	    /* mark as already allocated here */
-	    pmptr->base = 0x18; /* PAE and HIB use below this */
-	    pmptr->alloc = pmptr->end = top;  /* mark as already mapped */
-	    pmptr->type = mptr->type;
-	  } else if (mptr->base >= FOURGIG) {
-	    /* entire range above 4GB (pre PAE) */
-	    continue;
-	  } else if ( (base < fap) &&
-		      (top > fap)) {
-	    /* spans first_avail */
-	    /*  put mem below first avail in table but
-		mark already allocated */
-	    pmptr->base = base;
-	    pmptr->alloc = pmptr->end = (fap - 1);
-            pmptr->type = mptr->type;
-	    /* we bump these here inline so the accounting below works
-	       correctly */
-	    pmptr++;
-	    pmap_memory_region_count++;
-	    pmptr->alloc = pmptr->base = fap;
-            pmptr->type = mptr->type;
-	    pmptr->end = top;
-	  } else if ( (mptr->base < FOURGIG) &&
-		      ((mptr->base+mptr->length) > FOURGIG) ) {
-	    /* spans across 4GB (pre PAE) */
-	    pmptr->alloc = pmptr->base = base;
-            pmptr->type = mptr->type;
-	    pmptr->end = (FOURGIG >> I386_PGSHIFT) - 1;
-	  } else {
-	    /* entire range useable */
-	    pmptr->alloc = pmptr->base = base;
-            pmptr->type = mptr->type;
-	    pmptr->end = top;
-	  }
-
-	  if (i386_ptob(pmptr->end) > avail_end ) {
-	    avail_end = i386_ptob(pmptr->end);
-	  }
-
-	  avail_remaining += (pmptr->end - pmptr->base);
-	  pmap_memory_region_count++;
-	  pmptr++;
-	  }
+
+	for (i = 0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
+	        ppnum_t base, top;
+
+		if (pmap_memory_region_count >= PMAP_MEMORY_REGIONS_SIZE) {
+		        kprintf("WARNING: truncating memory region count at %d\n", pmap_memory_region_count);
+			break;
+		}
+		base = (ppnum_t) (mptr->PhysicalStart >> I386_PGSHIFT);
+		top = (ppnum_t) ((mptr->PhysicalStart) >> I386_PGSHIFT) + mptr->NumberOfPages - 1;
+
+		switch (mptr->Type) {
+		case kEfiLoaderCode:
+		case kEfiLoaderData:
+		case kEfiBootServicesCode:
+		case kEfiBootServicesData:
+		case kEfiConventionalMemory:
+		        /*
+			 * Consolidate usable memory types into one.
+			 */
+		        pmap_type = kEfiConventionalMemory;
+		        sane_size += (uint64_t)(mptr->NumberOfPages << I386_PGSHIFT);
+			break;
+
+		case kEfiRuntimeServicesCode:
+		case kEfiRuntimeServicesData:
+		case kEfiACPIReclaimMemory:
+		case kEfiACPIMemoryNVS:
+		case kEfiPalCode:
+			/*
+			 * sane_size should reflect the total amount of physical ram
+			 * in the system, not just the amount that is available for
+			 * the OS to use
+			 */
+		        sane_size += (uint64_t)(mptr->NumberOfPages << I386_PGSHIFT);
+			/* fall thru */
+
+		case kEfiUnusableMemory:
+		case kEfiMemoryMappedIO:
+		case kEfiMemoryMappedIOPortSpace:
+		case kEfiReservedMemoryType:
+		default:
+		        pmap_type = mptr->Type;
+		}
+
+		kprintf("EFI region: type = %d/%d,  base = 0x%x,  top = 0x%x\n", mptr->Type, pmap_type, base, top);
+
+		if (maxpg) {
+		        if (base >= maxpg)
+				break;
+		        top = (top > maxpg) ? maxpg : top;
+		}
+
+		/*
+		 * handle each region
+		 */
+		if (kEfiACPIMemoryNVS == pmap_type) {
+		        prev_pmptr = 0;
+			continue;
+		} else if ((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME ||
+			   pmap_type != kEfiConventionalMemory) {
+		        prev_pmptr = 0;
+			continue;
+		} else {
+		        /*
+			 * Usable memory region
+			 */
+		        if (top < I386_LOWMEM_RESERVED) {
+			        prev_pmptr = 0;
+				continue;
+			}
+			if (top < fap) {
+			        /*
+				 * entire range below first_avail
+			         * salvage some low memory pages
+				 * we use some very low memory at startup
+				 * mark as already allocated here
+				 */
+			        if (base >= I386_LOWMEM_RESERVED)
+				        pmptr->base = base;
+				else
+				        pmptr->base = I386_LOWMEM_RESERVED;
+				/*
+				 * mark as already mapped
+				 */
+				pmptr->alloc = pmptr->end = top;
+				pmptr->type = pmap_type;
+			}
+			else if ( (base < fap) && (top > fap) ) {
+			        /*
+				 * spans first_avail
+				 * put mem below first avail in table but
+				 * mark already allocated
+				 */
+			        pmptr->base = base;
+				pmptr->alloc = pmptr->end = (fap - 1);
+				pmptr->type = pmap_type;
+				/*
+				 * we bump these here inline so the accounting
+				 * below works correctly
+				 */
+				pmptr++;
+				pmap_memory_region_count++;
+				pmptr->alloc = pmptr->base = fap;
+				pmptr->type = pmap_type;
+				pmptr->end = top;
+			}
+			else {
+			        /*
+				 * entire range useable
+				 */
+			        pmptr->alloc = pmptr->base = base;
+				pmptr->type = pmap_type;
+				pmptr->end = top;
+			}
+
+			if (i386_ptob(pmptr->end) > avail_end )
+			        avail_end = i386_ptob(pmptr->end);
+
+			avail_remaining += (pmptr->end - pmptr->base);
+
+			/*
+			 * Consolidate contiguous memory regions, if possible
+			 */
+			if (prev_pmptr &&
+			    pmptr->type == prev_pmptr->type &&
+			    pmptr->base == pmptr->alloc &&
+			    pmptr->base == (prev_pmptr->end + 1)) {
+			        prev_pmptr->end = pmptr->end;
+			} else {
+			        pmap_memory_region_count++;
+				prev_pmptr = pmptr;
+				pmptr++;
+			}
+		}
 	}
-#endif
+
 
 #ifdef PRINT_PMAP_MEMORY_TABLE
- {
-   unsigned int j;
-  pmap_memory_region_t *p = pmap_memory_regions;
-   for (j=0;j<pmap_memory_region_count;j++, p++) {
-     kprintf("%d base 0x%x alloc 0x%x top 0x%x\n",j,
-	     p->base, p->alloc, p->end);
-   }
- }
+	{
+        unsigned int j;
+        pmap_memory_region_t *p = pmap_memory_regions;
+        vm_offset_t region_start, region_end;
+        vm_offset_t efi_start, efi_end;
+        for (j=0;j<pmap_memory_region_count;j++, p++) {
+            kprintf("type %d base 0x%x alloc 0x%x top 0x%x\n", p->type,
+                    p->base << I386_PGSHIFT, p->alloc << I386_PGSHIFT, p->end << I386_PGSHIFT);
+            region_start = p->base << I386_PGSHIFT;
+            region_end = (p->end << I386_PGSHIFT) - 1;
+            mptr = args->MemoryMap;
+            for (i=0; i<mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
+                if (mptr->Type != kEfiLoaderCode &&
+                    mptr->Type != kEfiLoaderData &&
+                    mptr->Type != kEfiBootServicesCode &&
+                    mptr->Type != kEfiBootServicesData &&
+                    mptr->Type != kEfiConventionalMemory) {
+                efi_start = (vm_offset_t)mptr->PhysicalStart;
+                efi_end = efi_start + ((vm_offset_t)mptr->NumberOfPages << I386_PGSHIFT) - 1;
+                if ((efi_start >= region_start && efi_start <= region_end) ||
+                    (efi_end >= region_start && efi_end <= region_end)) {
+                    kprintf(" *** Overlapping region with EFI runtime region %d\n", i);
+                }
+                }
+                
+            }
+        }
+	}
 #endif
 
 	avail_start = first_avail;
+	mem_actual = sane_size;
 
-	if (maxmem) {  /* if user set maxmem try to use it */
-	  uint64_t  tmp = (uint64_t)maxmem;
-	  /* can't set below first_avail or above actual memory */
-	  if ( (maxmem > first_avail) && (tmp < sane_size) ) {
-	    sane_size = tmp;
-	    avail_end = maxmem;
-	  }
+#define MEG		(1024*1024)
+
+	/*
+	 * For user visible memory size, round up to 128 Mb - accounting for the various stolen memory
+	 * not reported by EFI.
+	 */
+
+	sane_size = (sane_size + 128 * MEG - 1) & ~((uint64_t)(128 * MEG - 1));
+
+	/*
+	 * if user set maxmem, reduce memory sizes
+	 */
+	if ( (maxmem > (uint64_t)first_avail) && (maxmem < sane_size)) {
+		ppnum_t discarded_pages  = (sane_size - maxmem) >> I386_PGSHIFT;
+		sane_size                = maxmem;
+		if (avail_remaining > discarded_pages)
+			avail_remaining -= discarded_pages;
+		else
+			avail_remaining = 0;
 	}
-	// round up to a megabyte - mostly accounting for the
-	// low mem madness
-	sane_size += ( 0x100000ULL - 1);
-	sane_size &=  ~0xFFFFFULL;
-
-#ifndef PAE
-	if (sane_size < FOURGIG)
-	  mem_size = (unsigned long) sane_size;
-	else
-	  mem_size = (unsigned long) (FOURGIG >> 1);
-#else
-	  mem_size = (unsigned long) sane_size;
-#endif
 
+	/*
+	 * mem_size is only a 32 bit container... follow the PPC route
+	 * and pin it to a 2 Gbyte maximum
+	 */
+	if (sane_size > (FOURGIG >> 1))
+	        mem_size = (vm_size_t)(FOURGIG >> 1);
+	else
+	        mem_size = (vm_size_t)sane_size;
 	max_mem = sane_size;
 
-	/* now make sane size sane */
-#define MIN(a,b)	(((a)<(b))?(a):(b))
-#define MEG		(1024*1024)
-	sane_size = MIN(sane_size, 256*MEG);
+	kprintf("Physical memory %d MB\n", sane_size/MEG);
 
-	kprintf("Physical memory %d MB\n",
-		mem_size/MEG);
+	if (!PE_parse_boot_arg("max_valid_dma_addr", &maxdmaaddr))
+	        max_valid_dma_address = 1024ULL * 1024ULL * 4096ULL;
+	else
+	        max_valid_dma_address = ((uint64_t) maxdmaaddr) * 1024ULL * 1024ULL;
+
+	if (!PE_parse_boot_arg("maxbouncepool", &maxbouncepoolsize))
+	        maxbouncepoolsize = MAXBOUNCEPOOL;
+	else
+	        maxbouncepoolsize = maxbouncepoolsize * (1024 * 1024);
 
 	/*
-	 *	Initialize kernel physical map.
-	 *	Kernel virtual address starts at VM_KERNEL_MIN_ADDRESS.
+	 * bsd_mbuf_cluster_reserve depends on sane_size being set
+	 * in order to correctly determine the size of the mbuf pool
+	 * that will be reserved
 	 */
-	pmap_bootstrap(0);
+	if (!PE_parse_boot_arg("maxloreserve", &maxloreserve))
+	        maxloreserve = MAXLORESERVE + bsd_mbuf_cluster_reserve();
+	else
+	        maxloreserve = maxloreserve * (1024 * 1024);
+
 
+	if (avail_end >= max_valid_dma_address) {
+	        if (maxbouncepoolsize)
+		        reserve_bouncepool(maxbouncepoolsize);
+
+		if (maxloreserve)
+			vm_lopage_poolsize = maxloreserve / PAGE_SIZE;
+	}
 
+	/*
+	 *	Initialize kernel physical map.
+	 *	Kernel virtual address starts at VM_KERNEL_MIN_ADDRESS.
+	 */
+	pmap_bootstrap(0, IA32e);
 }
 
+
 unsigned int
 pmap_free_pages(void)
 {
 	return avail_remaining;
 }
 
+
 boolean_t
 pmap_next_page(
 	       ppnum_t *pn)
 {
-
-	while (pmap_memory_region_current < pmap_memory_region_count) {
-	  if (pmap_memory_regions[pmap_memory_region_current].alloc ==
-	      pmap_memory_regions[pmap_memory_region_current].end) {
-	    pmap_memory_region_current++;
-	    continue;
-	  }
-	  *pn = pmap_memory_regions[pmap_memory_region_current].alloc++;
-	  avail_remaining--;
-
-	  return TRUE;
+	
+	if (avail_remaining) while (pmap_memory_region_current < pmap_memory_region_count) {
+	        if (pmap_memory_regions[pmap_memory_region_current].alloc ==
+		    pmap_memory_regions[pmap_memory_region_current].end) {
+		        pmap_memory_region_current++;
+			continue;
+		}
+		*pn = pmap_memory_regions[pmap_memory_region_current].alloc++;
+		avail_remaining--;
+
+		return TRUE;
 	}
 	return FALSE;
 }
 
+
 boolean_t
 pmap_valid_page(
 	ppnum_t pn)
 {
-  unsigned int i;
-  pmap_memory_region_t *pmptr = pmap_memory_regions;
-
-  assert(pn);
-  for (i=0; i<pmap_memory_region_count; i++, pmptr++) {
-    if ( (pn >= pmptr->base) && (pn <= pmptr->end) ) {
-        if (pmptr->type == kMemoryRangeUsable)
-            return TRUE;
-        else
-            return FALSE;
-    }
-  }
-  return FALSE;
+        unsigned int i;
+	pmap_memory_region_t *pmptr = pmap_memory_regions;
+
+	assert(pn);
+	for (i = 0; i < pmap_memory_region_count; i++, pmptr++) {
+                if ( (pn >= pmptr->base) && (pn <= pmptr->end) && pmptr->type == kEfiConventionalMemory )
+	                return TRUE;
+	}
+	return FALSE;
+}
+
+
+static void
+reserve_bouncepool(uint32_t bounce_pool_wanted)
+{
+	pmap_memory_region_t *pmptr  = pmap_memory_regions;
+	pmap_memory_region_t *lowest = NULL;
+        unsigned int i;
+	unsigned int pages_needed;
+
+	pages_needed = bounce_pool_wanted / PAGE_SIZE;
+
+	for (i = 0; i < pmap_memory_region_count; i++, pmptr++) {
+	        if ( (pmptr->type == kEfiConventionalMemory) && ((pmptr->end - pmptr->alloc) >= pages_needed) ) {
+		        if ( (lowest == NULL) || (pmptr->alloc < lowest->alloc) )
+			        lowest = pmptr;
+		}
+	}
+	if ( (lowest != NULL) ) {
+	        bounce_pool_base = lowest->alloc * PAGE_SIZE;
+		bounce_pool_size = bounce_pool_wanted;
+
+		lowest->alloc += pages_needed;
+		avail_remaining -= pages_needed;
+	}
 }
diff --git a/osfmk/i386/idt.s b/osfmk/i386/idt.s
index 2fb2a37dc..2b1a0af77 100644
--- a/osfmk/i386/idt.s
+++ b/osfmk/i386/idt.s
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -58,84 +58,131 @@
 #include <i386/asm.h>
 #include <assym.s>
 #include <mach_kdb.h>
+#include <i386/eflags.h>
+#include <i386/trap.h>
+
+#define HI_DATA(lo_addr)	( (EXT(lo_addr) - EXT(hi_remap_data)) + HIGH_IDT_BASE )
+#define HI_TEXT(lo_text)	( (EXT(lo_text) - EXT(hi_remap_text)) + HIGH_MEM_BASE )
 
 /*
  * Interrupt descriptor table and code vectors for it.
  */
 #define	IDT_BASE_ENTRY(vec,seg,type) \
 	.data			;\
-	.long	vec		;\
+	.long	EXT(vec) - EXT(hi_remap_text) + HIGH_MEM_BASE ;\
+	.word	seg		;\
+	.byte	0		;\
+	.byte	type		;\
+	.text
+
+#define	IDT_BASE_ENTRY_INT(vec,seg,type) \
+	.data			;\
+	.long	vec - EXT(hi_remap_text) + HIGH_MEM_BASE ;\
+	.word	seg		;\
+	.byte	0		;\
+	.byte	type		;\
+	.text
+
+#define	IDT_BASE_ENTRY_TG(vec,seg,type) \
+	.data			;\
+	.long	0		;\
 	.word	seg		;\
 	.byte	0		;\
 	.byte	type		;\
 	.text
 
 #define	IDT_ENTRY(vec,type)	IDT_BASE_ENTRY(vec,KERNEL_CS,type)
+#define	IDT_ENTRY_INT(vec,type)	IDT_BASE_ENTRY_INT(vec,KERNEL_CS,type)
 
 /*
  * No error code.  Clear error code and push trap number.
  */
 #define	EXCEPTION(n,name) \
-	IDT_ENTRY(EXT(name),K_TRAP_GATE);\
+	IDT_ENTRY(name,K_INTR_GATE);\
 Entry(name)				;\
 	pushl	$0			;\
 	pushl	$(n)			;\
-	jmp	EXT(alltraps)
+	pusha				;\
+	movl	$ EXT(lo_alltraps),%ebx	;\
+	jmp	enter_lohandler
 
+	
 /*
  * Interrupt from user.  Clear error code and push trap number.
  */
 #define	EXCEP_USR(n,name) \
-	IDT_ENTRY(EXT(name),U_TRAP_GATE);\
+	IDT_ENTRY(name,U_INTR_GATE);\
 Entry(name)				;\
 	pushl	$0			;\
 	pushl	$(n)			;\
-	jmp	EXT(alltraps)
+	pusha				;\
+	movl	$ EXT(lo_alltraps),%ebx	;\
+	jmp	enter_lohandler
+	
 
 /*
  * Special interrupt code.
  */
 #define	EXCEP_SPC(n,name)  \
-	IDT_ENTRY(EXT(name),K_TRAP_GATE)
-
+	IDT_ENTRY(name,K_INTR_GATE) 
+	
 /*
  * Special interrupt code from user.
  */
 #define EXCEP_SPC_USR(n,name)  \
-	IDT_ENTRY(EXT(name),U_TRAP_GATE)
+	IDT_ENTRY(name,U_INTR_GATE) 
+
 
 /*
  * Extra-special interrupt code.  Note that no offset may be
  * specified in a task gate descriptor, so name is ignored.
  */
 #define	EXCEP_TASK(n,name)  \
-	IDT_BASE_ENTRY(0,DEBUG_TSS,K_TASK_GATE)
+	IDT_BASE_ENTRY_TG(0,DEBUG_TSS,K_TASK_GATE)
+
+/* Double-fault fatal handler */
+#define DF_FATAL_TASK(n,name)  \
+	IDT_BASE_ENTRY_TG(0,DF_TSS,K_TASK_GATE)
+
+/* machine-check handler */
+#define MC_FATAL_TASK(n,name)  \
+	IDT_BASE_ENTRY_TG(0,MC_TSS,K_TASK_GATE)
 
 /*
  * Error code has been pushed.  Push trap number.
  */
 #define	EXCEP_ERR(n,name) \
-	IDT_ENTRY(EXT(name),K_TRAP_GATE);\
+	IDT_ENTRY(name,K_INTR_GATE);\
 Entry(name)				;\
 	pushl	$(n)			;\
-	jmp	EXT(alltraps)
+	pusha				;\
+	movl	$ EXT(lo_alltraps),%ebx	;\
+	jmp	enter_lohandler
 
+	
 /*
  * Interrupt.
  */
 #define	INTERRUPT(n) \
-	IDT_ENTRY(0f,K_INTR_GATE)	;\
-0:					;\
-	pushl	%eax			;\
-	movl	$(n),%eax		;\
-	jmp	EXT(all_intrs)
+	IDT_ENTRY_INT(L_ ## n,K_INTR_GATE)	;\
+	.align FALIGN				;\
+L_ ## n:					;\
+	pushl	$0				;\
+	pushl	$(n)				;\
+	pusha					;\
+	movl	$ EXT(lo_allintrs),%ebx		;\
+	jmp	enter_lohandler
+
 
 	.data
-Entry(idt)
+	.align 12
+Entry(master_idt)
+Entry(hi_remap_data)
 	.text
+Entry(hi_remap_text)
 
 EXCEPTION(0x00,t_zero_div)
-EXCEP_SPC(0x01,t_debug)
+EXCEP_SPC(0x01,hi_debug)
 INTERRUPT(0x02)			/* NMI */
 EXCEP_USR(0x03,t_int3)
 EXCEP_USR(0x04,t_into)
@@ -145,23 +192,23 @@ EXCEPTION(0x07,t_nofpu)
 #if	MACH_KDB
 EXCEP_TASK(0x08,db_task_dbl_fault)
 #else
-EXCEPTION(0x08,a_dbl_fault)
+DF_FATAL_TASK(0x08,df_task_start)
 #endif
 EXCEPTION(0x09,a_fpu_over)
 EXCEPTION(0x0a,a_inv_tss)
-EXCEP_SPC(0x0b,t_segnp)
+EXCEP_SPC(0x0b,hi_segnp)
 #if	MACH_KDB
 EXCEP_TASK(0x0c,db_task_stk_fault)
 #else
 EXCEP_ERR(0x0c,t_stack_fault)
 #endif
-EXCEP_SPC(0x0d,t_gen_prot)
-EXCEP_SPC(0x0e,t_page_fault)
+EXCEP_SPC(0x0d,hi_gen_prot)
+EXCEP_SPC(0x0e,hi_page_fault)
 EXCEPTION(0x0f,t_trap_0f)
 EXCEPTION(0x10,t_fpu_err)
 EXCEPTION(0x11,t_trap_11)
-EXCEPTION(0x12,t_trap_12)
-EXCEPTION(0x13,t_trap_13)
+MC_FATAL_TASK(0x12,mc_task_start)
+EXCEPTION(0x13,t_sse_err)
 EXCEPTION(0x14,t_trap_14)
 EXCEPTION(0x15,t_trap_15)
 EXCEPTION(0x16,t_trap_16)
@@ -277,10 +324,11 @@ INTERRUPT(0x7d)
 INTERRUPT(0x7e)
 INTERRUPT(0x7f)
 
-EXCEP_SPC_USR(0x80,syscall_int80)
-INTERRUPT(0x81)
-INTERRUPT(0x82)
-INTERRUPT(0x83)
+EXCEP_SPC_USR(0x80,hi_unix_scall)
+EXCEP_SPC_USR(0x81,hi_mach_scall)
+EXCEP_SPC_USR(0x82,hi_mdep_scall)
+EXCEP_SPC_USR(0x83,hi_diag_scall)
+
 INTERRUPT(0x84)
 INTERRUPT(0x85)
 INTERRUPT(0x86)
@@ -413,3 +461,399 @@ INTERRUPT(0xfd)
 INTERRUPT(0xfe)
 EXCEPTION(0xff,t_preempt)
 
+
+	.data
+Entry(lo_kernel_cr3)
+	.long 0
+	.long 0
+	
+        .text
+
+	
+/*******************************************************************************************************
+ *
+ * Trap/interrupt entry points.
+ *
+ * All traps must create the following save area on the PCB "stack":
+ *
+ *	gs
+ *	fs
+ *	es
+ *	ds
+ *	edi
+ *	esi
+ *	ebp
+ *	cr2 if page fault - otherwise unused
+ *	ebx
+ *	edx
+ *	ecx
+ *	eax
+ *	trap number
+ *	error code
+ *	eip
+ *	cs
+ *	eflags
+ *	user esp - if from user
+ *	user ss  - if from user
+ */
+
+	
+Entry(hi_ret_to_user)
+	movl	%esp,%ebx
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
+	subl	ACT_PCB_ISS(%ecx),%ebx
+	movl	$(WINDOWS_CLEAN),ACT_COPYIO_STATE(%ecx)
+
+	movl	ACT_PCB_IDS(%ecx),%eax	/* get debug state struct */
+	cmpl	$0,%eax			/* is there a debug state */
+	je	1f 			/* branch if not */
+	movl	DS_DR0(%eax), %ecx	/* Load the 32 bit debug registers */
+	movl	%ecx, %db0
+	movl	DS_DR1(%eax), %ecx
+	movl	%ecx, %db1
+	movl	DS_DR2(%eax), %ecx
+	movl	%ecx, %db2
+	movl	DS_DR3(%eax), %ecx
+	movl	%ecx, %db3
+	movl	DS_DR7(%eax), %eax
+1:
+	addl	%gs:CPU_HI_ISS,%ebx	/* rebase PCB save area to high addr */
+	movl	%gs:CPU_TASK_CR3,%ecx
+	movl	%ecx,%gs:CPU_ACTIVE_CR3
+	movl	%ebx,%esp		/* switch to hi based PCB stack */
+	movl    %ecx,%cr3               /* switch to user's address space */
+
+	cmpl	$0,%eax			/* is dr7 set to something? */
+	je	2f 			/* branch if not */
+	movl	%eax,%db7		/* Set dr7 */
+2:
+
+Entry(hi_ret_to_kernel)
+
+	popl	%eax			/* ignore flavor of saved state */
+EXT(ret_popl_gs):
+	popl	%gs			/* restore segment registers */
+EXT(ret_popl_fs):
+	popl	%fs
+EXT(ret_popl_es):
+	popl	%es
+EXT(ret_popl_ds):	
+	popl	%ds
+
+        popa                            /* restore general registers */
+        addl    $8,%esp                 /* discard trap number and error code */
+
+        cmpl    $(SYSENTER_CS),4(%esp)  /* test for fast entry/exit */
+        je      fast_exit
+EXT(ret_iret):
+        iret                            /* return from interrupt */
+fast_exit:
+	popl    %edx                    /* user return eip */
+        popl    %ecx                    /* pop and toss cs */
+	andl	$(~EFL_IF),(%esp)	/* clear intrs enabled, see sti below */
+        popf                            /* flags - carry denotes failure */
+        popl    %ecx                    /* user return esp */
+	sti				/* interrupts enabled after sysexit */
+        sysexit
+
+/*******************************************************************************************************/
+
+		
+Entry(hi_unix_scall)
+	pushl   %eax                    /* save system call number */
+        pushl   $0                      /* clear trap number slot */
+        pusha                           /* save the general registers */
+	movl	$ EXT(lo_unix_scall),%ebx
+	jmp	enter_lohandler
+
+	
+Entry(hi_mach_scall)
+	pushl   %eax                    /* save system call number */
+        pushl   $0                      /* clear trap number slot */
+        pusha                           /* save the general registers */
+	movl	$ EXT(lo_mach_scall),%ebx
+	jmp	enter_lohandler
+
+	
+Entry(hi_mdep_scall)
+	pushl   %eax                    /* save system call number */
+        pushl   $0                      /* clear trap number slot */
+        pusha                           /* save the general registers */
+	movl	$ EXT(lo_mdep_scall),%ebx
+	jmp	enter_lohandler
+
+	
+Entry(hi_diag_scall)
+	pushl   %eax                    // Save sselector
+        pushl   $0                      // Clear trap number slot
+        pusha                           // save the general registers
+	movl	$EXT(lo_diag_scall),%ebx	// Get the function down low to transfer to
+	jmp	enter_lohandler			// Leap to it...
+
+	
+/*
+ * sysenter entry point
+ * Requires user code to set up:
+ *	edx: user instruction pointer (return address)
+ *	ecx: user stack pointer
+ *		on which is pushed stub ret addr and saved ebx
+ * Return to user-space is made using sysexit.
+ * Note: sysenter/sysexit cannot be used for calls returning a value in edx,
+ *       or requiring ecx to be preserved.
+ */
+Entry(hi_sysenter)
+	movl	(%esp), %esp		/* switch from intr stack to pcb */
+	/*
+	 * Push values on to the PCB stack
+	 * to cons up the saved state.
+	 */
+	pushl	$(USER_DS)		/* ss */
+	pushl	%ecx			/* uesp */
+	pushf				/* flags */
+	/*
+	* Clear, among others, the Nested Task (NT) flags bit;
+	* This is cleared by INT, but not by sysenter, which only
+	* clears RF, VM and IF.
+	*/
+	pushl	$0
+	popfl
+	pushl	$(SYSENTER_CS)		/* cs */
+hi_sysenter_2:
+	pushl	%edx			/* eip */
+	pushl	%eax			/* err/eax - syscall code */
+	pushl	$0			/* clear trap number slot */
+	pusha				/* save the general registers */
+	orl	$(EFL_IF),R_EFLAGS-R_EDI(%esp)	/* (edi was last reg pushed) */
+	movl	$ EXT(lo_sysenter),%ebx
+enter_lohandler:
+	pushl   %ds
+	pushl   %es
+        pushl   %fs
+        pushl   %gs
+enter_lohandler1:
+	pushl	$(SS_32)		/* 32-bit state flavor */
+	mov	%ss,%eax
+	mov	%eax,%ds
+	mov	%eax,%fs
+	mov	%eax,%es		/* switch to kernel data seg */
+	mov	$(CPU_DATA_GS),%eax
+	mov	%eax,%gs
+	cld				/* clear direction flag */
+	/*
+	 * Switch to kernel's address space if necessary
+	 */
+	movl    HI_DATA(lo_kernel_cr3),%ecx
+	movl	%cr3,%eax
+	cmpl	%eax,%ecx
+	je	1f
+	movl	%ecx,%cr3
+	movl	%ecx,%gs:CPU_ACTIVE_CR3
+1:
+	testb	$3,R_CS(%esp)
+	jz	2f
+	movl	%esp,%edx			/* came from user mode */
+	subl	%gs:CPU_HI_ISS,%edx
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
+	addl	ACT_PCB_ISS(%ecx),%edx		/* rebase the high stack to a low address */
+	movl	%edx,%esp
+	cmpl	$0, ACT_PCB_IDS(%ecx)	/* Is there a debug register state? */
+	je	2f
+	movl	$0, %ecx		/* If so, reset DR7 (the control) */
+	movl	%ecx, %dr7
+2:
+	movl	R_TRAPNO(%esp),%ecx			// Get the interrupt vector
+	addl	$1,%gs:hwIntCnt(,%ecx,4)	// Bump the count
+	jmp	*%ebx
+
+	
+/*
+ * Page fault traps save cr2.
+ */
+Entry(hi_page_fault)
+	pushl	$(T_PAGE_FAULT)		/* mark a page fault trap */
+	pusha				/* save the general registers */
+	movl	%cr2,%eax		/* get the faulting address */
+	movl	%eax,R_CR2-R_EDI(%esp)	/* save in esp save slot */
+
+	movl	$ EXT(lo_alltraps),%ebx
+	jmp	enter_lohandler
+
+
+
+/*
+ * Debug trap.  Check for single-stepping across system call into
+ * kernel.  If this is the case, taking the debug trap has turned
+ * off single-stepping - save the flags register with the trace
+ * bit set.
+ */
+Entry(hi_debug)
+	testb	$3,4(%esp)
+	jnz	hi_debug_trap
+					/* trap came from kernel mode */
+	cmpl	$(HI_TEXT(hi_mach_scall)),(%esp)
+	jne	6f
+	addl	$12,%esp		/* remove eip/cs/eflags from debug_trap */
+	jmp	EXT(hi_mach_scall)	/* continue system call entry */
+6:
+	cmpl	$(HI_TEXT(hi_mdep_scall)),(%esp)
+	jne	5f
+	addl	$12,%esp		/* remove eip/cs/eflags from debug_trap */
+	jmp	EXT(hi_mdep_scall)	/* continue system call entry */
+5:
+	cmpl	$(HI_TEXT(hi_unix_scall)),(%esp)
+	jne	4f
+	addl	$12,%esp		/* remove eip/cs/eflags from debug_trap */
+	jmp	EXT(hi_unix_scall)	/* continue system call entry */
+4:
+	cmpl	$(HI_TEXT(hi_sysenter)),(%esp)
+	jne	hi_debug_trap
+	/*
+	 * eip/cs/flags have been pushed on intr stack
+	 * We have to switch to pcb stack and copy eflags.
+	 * Note: setting the cs selector to SYSENTER_TF_CS
+	 * will cause the return to user path to take the iret path so
+	 * that eflags (containing the trap bit) is set atomically.
+	 * In unix_syscall this is tested so that we'll rewind the pc
+	 * to account for with sysenter or int entry.
+	 */ 
+	addl	$8,%esp			/* remove eip/cs */
+	pushl	%ecx			/* save %ecx */
+	movl	8(%esp),%ecx		/* top of intr stack -> pcb stack */
+	xchgl	%ecx,%esp		/* switch to pcb stack */
+	pushl	$(USER_DS)		/* ss */
+	pushl	%ss:(%ecx)		/* %ecx into uesp slot */
+	pushl	%ss:4(%ecx)		/* eflags */
+	movl	%ss:(%ecx),%ecx		/* restore %ecx */
+	pushl	$(SYSENTER_TF_CS)	/* cs - not SYSENTER_CS for iret path */
+	jmp	hi_sysenter_2		/* continue sysenter entry */
+hi_debug_trap:
+	pushl	$0
+	pushl	$(T_DEBUG)		/* handle as user trap */
+	pusha				/* save the general registers */
+	movl	$ EXT(lo_alltraps),%ebx
+	jmp	enter_lohandler	
+
+
+
+/*
+ * General protection or segment-not-present fault.
+ * Check for a GP/NP fault in the kernel_return
+ * sequence; if there, report it as a GP/NP fault on the user's instruction.
+ *
+ * esp->     0:	trap code (NP or GP)
+ *	     4:	segment number in error
+ *	     8	eip
+ *	    12	cs
+ *	    16	eflags 
+ *	    20	old registers (trap is from kernel)
+ */
+Entry(hi_gen_prot)
+	pushl	$(T_GENERAL_PROTECTION)	/* indicate fault type */
+	jmp	trap_check_kernel_exit	/* check for kernel exit sequence */
+
+Entry(hi_segnp)
+	pushl	$(T_SEGMENT_NOT_PRESENT)
+					/* indicate fault type */
+trap_check_kernel_exit:
+	testb	$3,12(%esp)
+	jnz	hi_take_trap
+					/* trap was from kernel mode, so */
+					/* check for the kernel exit sequence */
+	cmpl	$(HI_TEXT(ret_iret)),8(%esp)	/* on IRET? */
+	je	fault_iret
+	cmpl	$(HI_TEXT(ret_popl_ds)),8(%esp)	/* popping DS? */
+	je	fault_popl_ds
+	cmpl	$(HI_TEXT(ret_popl_es)),8(%esp)	/* popping ES? */
+	je	fault_popl_es
+	cmpl	$(HI_TEXT(ret_popl_fs)),8(%esp)	/* popping FS? */
+	je	fault_popl_fs
+	cmpl	$(HI_TEXT(ret_popl_gs)),8(%esp)	/* popping GS? */
+	je	fault_popl_gs
+hi_take_trap:
+	pusha				/* save the general registers */
+	movl	$ EXT(lo_alltraps),%ebx
+	jmp	enter_lohandler
+
+		
+/*
+ * GP/NP fault on IRET: CS or SS is in error.
+ * All registers contain the user's values.
+ *
+ * on SP is
+ *  0	trap number
+ *  4	errcode
+ *  8	eip
+ * 12	cs		--> trapno
+ * 16	efl		--> errcode
+ * 20	user eip
+ * 24	user cs
+ * 28	user eflags
+ * 32	user esp
+ * 36	user ss
+ */
+fault_iret:
+	movl	%eax,8(%esp)		/* save eax (we don`t need saved eip) */
+	popl	%eax			/* get trap number */
+	movl	%eax,12-4(%esp)		/* put in user trap number */
+	popl	%eax			/* get error code */
+	movl	%eax,16-8(%esp)		/* put in user errcode */
+	popl	%eax			/* restore eax */
+					/* now treat as fault from user */
+	pusha				/* save the general registers */
+	movl	$ EXT(lo_alltraps),%ebx
+	jmp	enter_lohandler
+
+/*
+ * Fault restoring a segment register.  The user's registers are still
+ * saved on the stack.  The offending segment register has not been
+ * popped.
+ */
+fault_popl_ds:
+	popl	%eax			/* get trap number */
+	popl	%edx			/* get error code */
+	addl	$12,%esp		/* pop stack to user regs */
+	jmp	push_es			/* (DS on top of stack) */
+fault_popl_es:
+	popl	%eax			/* get trap number */
+	popl	%edx			/* get error code */
+	addl	$12,%esp		/* pop stack to user regs */
+	jmp	push_fs			/* (ES on top of stack) */
+fault_popl_fs:
+	popl	%eax			/* get trap number */
+	popl	%edx			/* get error code */
+	addl	$12,%esp		/* pop stack to user regs */
+	jmp	push_gs			/* (FS on top of stack) */
+fault_popl_gs:
+	popl	%eax			/* get trap number */
+	popl	%edx			/* get error code */
+	addl	$12,%esp		/* pop stack to user regs */
+	jmp	push_none		/* (GS on top of stack) */
+
+push_es:
+	pushl	%es			/* restore es, */
+push_fs:
+	pushl	%fs			/* restore fs, */
+push_gs:
+	pushl	%gs			/* restore gs. */
+push_none:
+	movl	%eax,R_TRAPNO(%esp)	/* set trap number */
+	movl	%edx,R_ERR(%esp)	/* set error code */
+					/* now treat as fault from user */
+					/* except that segment registers are */
+					/* already pushed */
+	movl	$ EXT(lo_alltraps),%ebx
+	jmp	enter_lohandler1
+
+	
+        .text
+
+
+Entry(lo_ret_to_user)
+	jmp *1f
+1:	.long HI_TEXT(hi_ret_to_user) 
+
+Entry(lo_ret_to_kernel)
+	jmp *1f
+1:	.long HI_TEXT(hi_ret_to_kernel)
+
+Entry(hi_remap_etext)
diff --git a/osfmk/i386/idt64.s b/osfmk/i386/idt64.s
new file mode 100644
index 000000000..99cb3b8f2
--- /dev/null
+++ b/osfmk/i386/idt64.s
@@ -0,0 +1,1167 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <i386/asm.h>
+#include <i386/asm64.h>
+#include <assym.s>
+#include <mach_kdb.h>
+#include <i386/eflags.h>
+#include <i386/trap.h>
+#define _ARCH_I386_ASM_HELP_H_		/* Prevent inclusion of user header */
+#include <mach/i386/syscall_sw.h>
+#include <i386/postcode.h>
+#include <i386/proc_reg.h>
+
+/*
+ * Locore handlers.
+ */
+#define	LO_ALLINTRS		EXT(lo_allintrs)
+#define	LO_ALLTRAPS		EXT(lo_alltraps)
+#define	LO_SYSENTER		EXT(lo_sysenter)
+#define	LO_SYSCALL		EXT(lo_syscall)
+#define	LO_UNIX_SCALL		EXT(lo_unix_scall)
+#define	LO_MACH_SCALL		EXT(lo_mach_scall)
+#define	LO_MDEP_SCALL		EXT(lo_mdep_scall)
+#define	LO_DIAG_SCALL		EXT(lo_diag_scall)
+#define	LO_DOUBLE_FAULT		EXT(lo_df64)
+#define	LO_MACHINE_CHECK	EXT(lo_mc64)
+
+/*
+ * Interrupt descriptor table and code vectors for it.
+ *
+ * The IDT64_BASE_ENTRY macro lays down a fake descriptor that must be
+ * reformatted ("fixed") before use. 
+ * All vector are rebased in uber-space.
+ * Special vectors (e.g. double-fault) use a non-0 IST.
+ */
+#define	IDT64_BASE_ENTRY(vec,seg,ist,type)		 \
+	.data						;\
+	.long	vec					;\
+	.long	KERNEL_UBER_BASE_HI32			;\
+	.word	seg					;\
+	.byte	ist*16					;\
+	.byte	type					;\
+	.long	0					;\
+	.text
+
+#define	IDT64_ENTRY(vec,ist,type)			\
+	IDT64_BASE_ENTRY(EXT(vec),KERNEL64_CS,ist,type)
+#define	IDT64_ENTRY_LOCAL(vec,ist,type)			\
+	IDT64_BASE_ENTRY(vec,KERNEL64_CS,ist,type)
+
+/*
+ * Push trap number and address of compatibility mode handler,
+ * then branch to common trampoline. Error already pushed.
+ */
+#define	EXCEP64_ERR(n,name)				 \
+	IDT64_ENTRY(name,0,K_INTR_GATE)			;\
+Entry(name)						;\
+	push	$(n)					;\
+	movl	$(LO_ALLTRAPS), 4(%rsp)			;\
+	jmp	L_enter_lohandler
+
+
+/*
+ * Push error(0), trap number and address of compatibility mode handler,
+ * then branch to common trampoline.
+ */
+#define	EXCEPTION64(n,name)				 \
+	IDT64_ENTRY(name,0,K_INTR_GATE)			;\
+Entry(name)						;\
+	push	$0					;\
+	push	$(n)					;\
+	movl	$(LO_ALLTRAPS), 4(%rsp)			;\
+	jmp	L_enter_lohandler
+
+	
+/*
+ * Interrupt from user.
+ * Push error (0), trap number and address of compatibility mode handler,
+ * then branch to common trampoline.
+ */
+#define	EXCEP64_USR(n,name)				 \
+	IDT64_ENTRY(name,0,U_INTR_GATE)			;\
+Entry(name)						;\
+	push	$0					;\
+	push	$(n)					;\
+	movl	$(LO_ALLTRAPS), 4(%rsp)			;\
+	jmp	L_enter_lohandler
+
+
+/*
+ * Special interrupt code from user.
+ */
+#define EXCEP64_SPC_USR(n,name) 			\
+	IDT64_ENTRY(name,0,U_INTR_GATE) 
+
+
+/*
+ * Special interrupt code. 
+ * In 64-bit mode we may use an IST slot instead of task gates.
+ */
+#define	EXCEP64_IST(n,name,ist) 			\
+	IDT64_ENTRY(name,ist,K_INTR_GATE)
+#define	EXCEP64_SPC(n,name)	 			\
+	IDT64_ENTRY(name,0,K_INTR_GATE)
+
+	
+/*
+ * Interrupt.
+ * Push zero err, interrupt vector and address of compatibility mode handler,
+ * then branch to common trampoline.
+ */
+#define	INTERRUPT64(n)					 \
+	IDT64_ENTRY_LOCAL(L_ ## n,0,K_INTR_GATE)	;\
+	.align FALIGN					;\
+L_ ## n:						;\
+	push	$0					;\
+	push	$(n)					;\
+	movl	$(LO_ALLINTRS), 4(%rsp)			;\
+	jmp	L_enter_lohandler
+
+
+	.data
+	.align 12
+Entry(master_idt64)
+Entry(hi64_data_base)
+	.text
+	.code64
+Entry(hi64_text_base)
+
+EXCEPTION64(0x00,t64_zero_div)
+EXCEP64_SPC(0x01,hi64_debug)
+INTERRUPT64(0x02)			/* NMI */
+EXCEP64_USR(0x03,t64_int3)
+EXCEP64_USR(0x04,t64_into)
+EXCEP64_USR(0x05,t64_bounds)
+EXCEPTION64(0x06,t64_invop)
+EXCEPTION64(0x07,t64_nofpu)
+#if	MACH_KDB
+EXCEP64_IST(0x08,db_task_dbl_fault64,1)
+#else
+EXCEP64_IST(0x08,hi64_double_fault,1)
+#endif
+EXCEPTION64(0x09,a64_fpu_over)
+EXCEPTION64(0x0a,a64_inv_tss)
+EXCEP64_SPC(0x0b,hi64_segnp)
+#if	MACH_KDB
+EXCEP64_IST(0x0c,db_task_stk_fault64,1)
+#else
+EXCEP64_SPC(0x0c,hi64_stack_fault)
+#endif
+EXCEP64_SPC(0x0d,hi64_gen_prot)
+EXCEP64_ERR(0x0e,t64_page_fault)
+EXCEPTION64(0x0f,t64_trap_0f)
+EXCEPTION64(0x10,t64_fpu_err)
+EXCEPTION64(0x11,t64_trap_11)
+EXCEP64_IST(0x12,mc64,1)
+EXCEPTION64(0x13,t64_sse_err)
+EXCEPTION64(0x14,t64_trap_14)
+EXCEPTION64(0x15,t64_trap_15)
+EXCEPTION64(0x16,t64_trap_16)
+EXCEPTION64(0x17,t64_trap_17)
+EXCEPTION64(0x18,t64_trap_18)
+EXCEPTION64(0x19,t64_trap_19)
+EXCEPTION64(0x1a,t64_trap_1a)
+EXCEPTION64(0x1b,t64_trap_1b)
+EXCEPTION64(0x1c,t64_trap_1c)
+EXCEPTION64(0x1d,t64_trap_1d)
+EXCEPTION64(0x1e,t64_trap_1e)
+EXCEPTION64(0x1f,t64_trap_1f)
+
+INTERRUPT64(0x20)
+INTERRUPT64(0x21)
+INTERRUPT64(0x22)
+INTERRUPT64(0x23)
+INTERRUPT64(0x24)
+INTERRUPT64(0x25)
+INTERRUPT64(0x26)
+INTERRUPT64(0x27)
+INTERRUPT64(0x28)
+INTERRUPT64(0x29)
+INTERRUPT64(0x2a)
+INTERRUPT64(0x2b)
+INTERRUPT64(0x2c)
+INTERRUPT64(0x2d)
+INTERRUPT64(0x2e)
+INTERRUPT64(0x2f)
+
+INTERRUPT64(0x30)
+INTERRUPT64(0x31)
+INTERRUPT64(0x32)
+INTERRUPT64(0x33)
+INTERRUPT64(0x34)
+INTERRUPT64(0x35)
+INTERRUPT64(0x36)
+INTERRUPT64(0x37)
+INTERRUPT64(0x38)
+INTERRUPT64(0x39)
+INTERRUPT64(0x3a)
+INTERRUPT64(0x3b)
+INTERRUPT64(0x3c)
+INTERRUPT64(0x3d)
+INTERRUPT64(0x3e)
+INTERRUPT64(0x3f)
+
+INTERRUPT64(0x40)
+INTERRUPT64(0x41)
+INTERRUPT64(0x42)
+INTERRUPT64(0x43)
+INTERRUPT64(0x44)
+INTERRUPT64(0x45)
+INTERRUPT64(0x46)
+INTERRUPT64(0x47)
+INTERRUPT64(0x48)
+INTERRUPT64(0x49)
+INTERRUPT64(0x4a)
+INTERRUPT64(0x4b)
+INTERRUPT64(0x4c)
+INTERRUPT64(0x4d)
+INTERRUPT64(0x4e)
+INTERRUPT64(0x4f)
+
+INTERRUPT64(0x50)
+INTERRUPT64(0x51)
+INTERRUPT64(0x52)
+INTERRUPT64(0x53)
+INTERRUPT64(0x54)
+INTERRUPT64(0x55)
+INTERRUPT64(0x56)
+INTERRUPT64(0x57)
+INTERRUPT64(0x58)
+INTERRUPT64(0x59)
+INTERRUPT64(0x5a)
+INTERRUPT64(0x5b)
+INTERRUPT64(0x5c)
+INTERRUPT64(0x5d)
+INTERRUPT64(0x5e)
+INTERRUPT64(0x5f)
+
+INTERRUPT64(0x60)
+INTERRUPT64(0x61)
+INTERRUPT64(0x62)
+INTERRUPT64(0x63)
+INTERRUPT64(0x64)
+INTERRUPT64(0x65)
+INTERRUPT64(0x66)
+INTERRUPT64(0x67)
+INTERRUPT64(0x68)
+INTERRUPT64(0x69)
+INTERRUPT64(0x6a)
+INTERRUPT64(0x6b)
+INTERRUPT64(0x6c)
+INTERRUPT64(0x6d)
+INTERRUPT64(0x6e)
+INTERRUPT64(0x6f)
+
+INTERRUPT64(0x70)
+INTERRUPT64(0x71)
+INTERRUPT64(0x72)
+INTERRUPT64(0x73)
+INTERRUPT64(0x74)
+INTERRUPT64(0x75)
+INTERRUPT64(0x76)
+INTERRUPT64(0x77)
+INTERRUPT64(0x78)
+INTERRUPT64(0x79)
+INTERRUPT64(0x7a)
+INTERRUPT64(0x7b)
+INTERRUPT64(0x7c)
+INTERRUPT64(0x7d)
+INTERRUPT64(0x7e)
+INTERRUPT64(0x7f)
+
+EXCEP64_SPC_USR(0x80,hi64_unix_scall)
+EXCEP64_SPC_USR(0x81,hi64_mach_scall)
+EXCEP64_SPC_USR(0x82,hi64_mdep_scall)
+EXCEP64_SPC_USR(0x83,hi64_diag_scall)
+
+INTERRUPT64(0x84)
+INTERRUPT64(0x85)
+INTERRUPT64(0x86)
+INTERRUPT64(0x87)
+INTERRUPT64(0x88)
+INTERRUPT64(0x89)
+INTERRUPT64(0x8a)
+INTERRUPT64(0x8b)
+INTERRUPT64(0x8c)
+INTERRUPT64(0x8d)
+INTERRUPT64(0x8e)
+INTERRUPT64(0x8f)
+
+INTERRUPT64(0x90)
+INTERRUPT64(0x91)
+INTERRUPT64(0x92)
+INTERRUPT64(0x93)
+INTERRUPT64(0x94)
+INTERRUPT64(0x95)
+INTERRUPT64(0x96)
+INTERRUPT64(0x97)
+INTERRUPT64(0x98)
+INTERRUPT64(0x99)
+INTERRUPT64(0x9a)
+INTERRUPT64(0x9b)
+INTERRUPT64(0x9c)
+INTERRUPT64(0x9d)
+INTERRUPT64(0x9e)
+INTERRUPT64(0x9f)
+
+INTERRUPT64(0xa0)
+INTERRUPT64(0xa1)
+INTERRUPT64(0xa2)
+INTERRUPT64(0xa3)
+INTERRUPT64(0xa4)
+INTERRUPT64(0xa5)
+INTERRUPT64(0xa6)
+INTERRUPT64(0xa7)
+INTERRUPT64(0xa8)
+INTERRUPT64(0xa9)
+INTERRUPT64(0xaa)
+INTERRUPT64(0xab)
+INTERRUPT64(0xac)
+INTERRUPT64(0xad)
+INTERRUPT64(0xae)
+INTERRUPT64(0xaf)
+
+INTERRUPT64(0xb0)
+INTERRUPT64(0xb1)
+INTERRUPT64(0xb2)
+INTERRUPT64(0xb3)
+INTERRUPT64(0xb4)
+INTERRUPT64(0xb5)
+INTERRUPT64(0xb6)
+INTERRUPT64(0xb7)
+INTERRUPT64(0xb8)
+INTERRUPT64(0xb9)
+INTERRUPT64(0xba)
+INTERRUPT64(0xbb)
+INTERRUPT64(0xbc)
+INTERRUPT64(0xbd)
+INTERRUPT64(0xbe)
+INTERRUPT64(0xbf)
+
+INTERRUPT64(0xc0)
+INTERRUPT64(0xc1)
+INTERRUPT64(0xc2)
+INTERRUPT64(0xc3)
+INTERRUPT64(0xc4)
+INTERRUPT64(0xc5)
+INTERRUPT64(0xc6)
+INTERRUPT64(0xc7)
+INTERRUPT64(0xc8)
+INTERRUPT64(0xc9)
+INTERRUPT64(0xca)
+INTERRUPT64(0xcb)
+INTERRUPT64(0xcc)
+INTERRUPT64(0xcd)
+INTERRUPT64(0xce)
+INTERRUPT64(0xcf)
+
+INTERRUPT64(0xd0)
+INTERRUPT64(0xd1)
+INTERRUPT64(0xd2)
+INTERRUPT64(0xd3)
+INTERRUPT64(0xd4)
+INTERRUPT64(0xd5)
+INTERRUPT64(0xd6)
+INTERRUPT64(0xd7)
+INTERRUPT64(0xd8)
+INTERRUPT64(0xd9)
+INTERRUPT64(0xda)
+INTERRUPT64(0xdb)
+INTERRUPT64(0xdc)
+INTERRUPT64(0xdd)
+INTERRUPT64(0xde)
+INTERRUPT64(0xdf)
+
+INTERRUPT64(0xe0)
+INTERRUPT64(0xe1)
+INTERRUPT64(0xe2)
+INTERRUPT64(0xe3)
+INTERRUPT64(0xe4)
+INTERRUPT64(0xe5)
+INTERRUPT64(0xe6)
+INTERRUPT64(0xe7)
+INTERRUPT64(0xe8)
+INTERRUPT64(0xe9)
+INTERRUPT64(0xea)
+INTERRUPT64(0xeb)
+INTERRUPT64(0xec)
+INTERRUPT64(0xed)
+INTERRUPT64(0xee)
+INTERRUPT64(0xef)
+
+INTERRUPT64(0xf0)
+INTERRUPT64(0xf1)
+INTERRUPT64(0xf2)
+INTERRUPT64(0xf3)
+INTERRUPT64(0xf4)
+INTERRUPT64(0xf5)
+INTERRUPT64(0xf6)
+INTERRUPT64(0xf7)
+INTERRUPT64(0xf8)
+INTERRUPT64(0xf9)
+INTERRUPT64(0xfa)
+INTERRUPT64(0xfb)
+INTERRUPT64(0xfc)
+INTERRUPT64(0xfd)
+INTERRUPT64(0xfe)
+EXCEPTION64(0xff,t64_preempt)
+
+
+        .text
+/*
+ *
+ * Trap/interrupt entry points.
+ *
+ * All traps must create the following 32-bit save area on the PCB "stack"
+ * - this is identical to the legacy mode 32-bit case:
+ *
+ *	gs
+ *	fs
+ *	es
+ *	ds
+ *	edi
+ *	esi
+ *	ebp
+ *	cr2 (defined only for page fault)
+ *	ebx
+ *	edx
+ *	ecx
+ *	eax
+ *	trap number
+ *	error code
+ *	eip
+ *	cs
+ *	eflags
+ *	user esp - if from user
+ *	user ss  - if from user
+ *
+ * Above this is the trap number and compatibility mode handler address
+ * (packed into an 8-byte stack entry) and the 64-bit interrupt stack frame:
+ *
+ *	(trapno, trapfn)
+ *	err
+ *	rip
+ *	cs
+ *	rflags
+ *	rsp
+ *	ss
+ *	
+ */
+
+	.code32
+/*
+ * Control is passed here to return to the compatibility mode user.
+ * At this stage we're in kernel space in compatibility mode
+ * but we need to switch into 64-bit mode in the 4G-based trampoline
+ * space before performing the iret.
+ */ 
+Entry(lo64_ret_to_user)
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
+
+	movl	ACT_PCB_IDS(%ecx),%eax	/* Obtain this thread's debug state */
+	cmpl	$0,%eax			/* Is there a debug register context? */
+	je	2f 			/* branch if not */
+	cmpl	$(TASK_MAP_32BIT), %gs:CPU_TASK_MAP	/* Are we a 64-bit task? */
+	jne	1f
+	movl	DS_DR0(%eax), %ecx	/* If not, load the 32 bit DRs */
+	movl	%ecx, %db0
+	movl	DS_DR1(%eax), %ecx
+	movl	%ecx, %db1
+	movl	DS_DR2(%eax), %ecx
+	movl	%ecx, %db2
+	movl	DS_DR3(%eax), %ecx
+	movl	%ecx, %db3
+	movl	DS_DR7(%eax), %ecx
+	movl 	%ecx, %gs:CPU_DR7
+	movl	$0, %gs:CPU_DR7 + 4
+	jmp 	2f
+1:
+	ENTER_64BIT_MODE()		/* Enter long mode */
+	mov	DS64_DR0(%eax), %rcx	/* Load the full width DRs*/
+	mov	%rcx, %dr0
+	mov	DS64_DR1(%eax), %rcx
+	mov	%rcx, %dr1
+	mov	DS64_DR2(%eax), %rcx
+	mov	%rcx, %dr2
+	mov	DS64_DR3(%eax), %rcx
+	mov	%rcx, %dr3
+	mov	DS64_DR7(%eax), %rcx
+	mov 	%rcx, %gs:CPU_DR7
+	jmp	3f			/* Enter uberspace */
+2:
+	ENTER_64BIT_MODE()
+3:
+	ENTER_UBERSPACE()	
+
+	/*
+	 * Now switch %cr3, if necessary.
+	 */
+	swapgs				/* switch back to uber-kernel gs base */
+	mov	%gs:CPU_TASK_CR3,%rcx
+	mov	%rcx,%gs:CPU_ACTIVE_CR3
+	mov	%cr3, %rax
+	cmp	%rcx, %rax
+	je	1f
+	/* flag the copyio engine state as WINDOWS_CLEAN */
+	mov	%gs:CPU_ACTIVE_THREAD,%eax
+	movl	$(WINDOWS_CLEAN),ACT_COPYIO_STATE(%eax)
+	mov	%rcx,%cr3               /* switch to user's address space */
+1:
+
+	mov	%gs:CPU_DR7, %rax	/* Is there a debug control register?*/
+	cmp	$0, %rax
+	je	1f
+	mov	%rax, %dr7		/* Set DR7 */
+	movq	$0, %gs:CPU_DR7
+1:
+
+	/*
+	 * Adjust stack to use uber-space.
+	 */
+	mov	$(KERNEL_UBER_BASE_HI32), %rax
+	shl	$32, %rsp
+	shrd	$32, %rax, %rsp			/* relocate into uber-space */
+
+	cmpl	$(SS_32), SS_FLAVOR(%rsp)	/* 32-bit state? */
+	jne	L_64bit_return
+	jmp	L_32bit_return
+
+Entry(lo64_ret_to_kernel)
+	ENTER_64BIT_MODE()
+	ENTER_UBERSPACE()	
+
+	swapgs				/* switch back to uber-kernel gs base */
+
+	/*
+	 * Adjust stack to use uber-space.
+	 */
+	mov	$(KERNEL_UBER_BASE_HI32), %rax
+	shl	$32, %rsp
+	shrd	$32, %rax, %rsp			/* relocate into uber-space */
+
+	/* Check for return to 64-bit kernel space (EFI today) */
+	cmpl	$(SS_32), SS_FLAVOR(%rsp)	/* 32-bit state? */
+	jne	L_64bit_return
+	/* fall through for 32-bit return */
+
+L_32bit_return:
+	/*
+	 * Restore registers into the machine state for iret.
+	 */
+	movl	R_EIP(%rsp), %eax
+	movl	%eax, ISC32_RIP(%rsp)
+	movl	R_EFLAGS(%rsp), %eax
+	movl	%eax, ISC32_RFLAGS(%rsp)
+	movl	R_CS(%rsp), %eax
+	movl	%eax, ISC32_CS(%rsp)
+	movl	R_UESP(%rsp), %eax
+	movl	%eax, ISC32_RSP(%rsp)
+	movl	R_SS(%rsp), %eax
+	movl	%eax, ISC32_SS(%rsp)
+
+	/*
+	 * Restore general 32-bit registers
+	 */
+	movl	R_EAX(%rsp), %eax
+	movl	R_EBX(%rsp), %ebx
+	movl	R_ECX(%rsp), %ecx
+	movl	R_EDX(%rsp), %edx
+	movl	R_EBP(%rsp), %ebp
+	movl	R_ESI(%rsp), %esi
+	movl	R_EDI(%rsp), %edi
+
+	/*
+	 * Restore segment registers. We make take an exception here but
+	 * we've got enough space left in the save frame area to absorb
+         * a hardware frame plus the trapfn and trapno
+	 */
+	swapgs
+EXT(ret32_set_ds):	
+	movw	R_DS(%rsp), %ds
+EXT(ret32_set_es):
+	movw	R_ES(%rsp), %es
+EXT(ret32_set_fs):
+	movw	R_FS(%rsp), %fs
+EXT(ret32_set_gs):
+	movw	R_GS(%rsp), %gs
+
+	add	$(ISC32_OFFSET)+8+8, %rsp	/* pop compat frame +
+						   trapno/trapfn and error */	
+        cmp	$(SYSENTER_CS),ISF64_CS-8-8(%rsp)
+					/* test for fast entry/exit */
+        je      L_fast_exit
+EXT(ret32_iret):
+        iretq				/* return from interrupt */
+
+L_fast_exit:
+	pop	%rdx                    /* user return eip */
+        pop	%rcx                    /* pop and toss cs */
+	andl	$(~EFL_IF), (%rsp)	/* clear interrupts enable, sti below */
+        popf                            /* flags - carry denotes failure */
+        pop	%rcx                    /* user return esp */
+	.code32
+	sti				/* interrupts enabled after sysexit */
+        sysexit				/* 32-bit sysexit */
+	.code64
+
+L_64bit_return:
+	/*
+	 * Set the GS Base MSR with the user's gs base.
+	 */
+	movl	%gs:CPU_UBER_USER_GS_BASE, %eax
+	movl	%gs:CPU_UBER_USER_GS_BASE+4, %edx
+	movl	$(MSR_IA32_GS_BASE), %ecx
+	swapgs
+	testb	$3, R64_CS(%rsp)		/* returning to user-space? */
+	jz	1f
+	wrmsr					/* set 64-bit base */
+1:
+
+	/*
+	 * Restore general 64-bit registers
+	 */
+	mov	R64_R15(%rsp), %r15
+	mov	R64_R14(%rsp), %r14
+	mov	R64_R13(%rsp), %r13
+	mov	R64_R12(%rsp), %r12
+	mov	R64_R11(%rsp), %r11
+	mov	R64_R10(%rsp), %r10
+	mov	R64_R9(%rsp),  %r9
+	mov	R64_R8(%rsp),  %r8
+	mov	R64_RSI(%rsp), %rsi
+	mov	R64_RDI(%rsp), %rdi
+	mov	R64_RBP(%rsp), %rbp
+	mov	R64_RDX(%rsp), %rdx
+	mov	R64_RBX(%rsp), %rbx
+	mov	R64_RCX(%rsp), %rcx
+	mov	R64_RAX(%rsp), %rax
+
+	add	$(ISS64_OFFSET)+8+8, %rsp	/* pop saved state frame +
+						   trapno/trapfn and error */	
+        cmpl	$(SYSCALL_CS),ISF64_CS-8-8(%rsp)
+					/* test for fast entry/exit */
+        je      L_sysret
+EXT(ret64_iret):
+        iretq				/* return from interrupt */
+
+L_sysret:
+	/*
+	 * Here to load rcx/r11/rsp and perform the sysret back to user-space.
+	 * 	rcx	user rip
+	 *	r1	user rflags
+	 *	rsp	user stack pointer
+	 */
+	mov	ISF64_RIP-16(%rsp), %rcx
+	mov	ISF64_RFLAGS-16(%rsp), %r11
+	mov	ISF64_RSP-16(%rsp), %rsp
+        sysretq				/* return from system call */
+
+/*
+ * Common path to enter locore handlers.
+ */
+L_enter_lohandler:
+	swapgs				/* switch to kernel gs (cpu_data) */
+L_enter_lohandler_continue:
+	cmpl	$(USER64_CS), ISF64_CS(%rsp)
+	je	L_64bit_enter		/* this is a 64-bit user task */
+	cmpl	$(KERNEL64_CS), ISF64_CS(%rsp)
+	je	L_64bit_enter		/* we're in 64-bit (EFI) code */
+	jmp	L_32bit_enter
+
+/*
+ * System call handlers.
+ * These are entered via a syscall interrupt. The system call number in %rax
+ * is saved to the error code slot in the stack frame. We then branch to the
+ * common state saving code.
+ */
+		
+Entry(hi64_unix_scall)
+	swapgs				/* switch to kernel gs (cpu_data) */
+L_unix_scall_continue:
+	push	%rax			/* save system call number */
+	push	$(UNIX_INT)
+	movl	$(LO_UNIX_SCALL), 4(%rsp)
+	jmp	L_32bit_enter_check
+
+	
+Entry(hi64_mach_scall)
+	swapgs				/* switch to kernel gs (cpu_data) */
+L_mach_scall_continue:
+	push	%rax			/* save system call number */
+	push	$(MACH_INT)
+	movl	$(LO_MACH_SCALL), 4(%rsp)
+	jmp	L_32bit_enter_check
+
+	
+Entry(hi64_mdep_scall)
+	swapgs				/* switch to kernel gs (cpu_data) */
+L_mdep_scall_continue:
+	push	%rax			/* save system call number */
+	push	$(MACHDEP_INT)
+	movl	$(LO_MDEP_SCALL), 4(%rsp)
+	jmp	L_32bit_enter_check
+
+	
+Entry(hi64_diag_scall)
+	swapgs				/* switch to kernel gs (cpu_data) */
+L_diag_scall_continue:
+	push	%rax			/* save system call number */
+	push	$(DIAG_INT)
+	movl	$(LO_DIAG_SCALL), 4(%rsp)
+	jmp	L_32bit_enter_check
+
+Entry(hi64_syscall)
+	swapgs				/* Kapow! get per-cpu data area */
+L_syscall_continue:
+	mov	%rsp, %gs:CPU_UBER_TMP	/* save user stack */
+	mov	%gs:CPU_UBER_ISF, %rsp	/* switch stack to pcb */
+
+	/*
+	 * Save values in the ISF frame in the PCB
+	 * to cons up the saved machine state.
+	 */
+	movl	$(USER_DS), ISF64_SS(%rsp)	
+	movl	$(SYSCALL_CS), ISF64_CS(%rsp)	/* cs - a pseudo-segment */
+	mov	%r11, ISF64_RFLAGS(%rsp)	/* rflags */
+	mov	%rcx, ISF64_RIP(%rsp)		/* rip */
+	mov	%gs:CPU_UBER_TMP, %rcx
+	mov	%rcx, ISF64_RSP(%rsp)		/* user stack */
+	mov	%rax, ISF64_ERR(%rsp)		/* err/rax - syscall code */
+	movl	$(0), ISF64_TRAPNO(%rsp)	/* trapno */
+	movl	$(LO_SYSCALL), ISF64_TRAPFN(%rsp)
+	jmp	L_64bit_enter		/* this can only be a 64-bit task */
+	
+/*
+ * sysenter entry point
+ * Requires user code to set up:
+ *	edx: user instruction pointer (return address)
+ *	ecx: user stack pointer
+ *		on which is pushed stub ret addr and saved ebx
+ * Return to user-space is made using sysexit.
+ * Note: sysenter/sysexit cannot be used for calls returning a value in edx,
+ *       or requiring ecx to be preserved.
+ */
+Entry(hi64_sysenter)
+	mov	(%rsp), %rsp		/* switch from temporary stack to pcb */
+	/*
+	 * Push values on to the PCB stack
+	 * to cons up the saved machine state.
+	 */
+	push	$(USER_DS)		/* ss */
+	push	%rcx			/* uesp */
+	pushf				/* flags */
+	/*
+	* Clear, among others, the Nested Task (NT) flags bit;
+	* This is cleared by INT, but not by sysenter, which only
+	* clears RF, VM and IF.
+	*/
+	push	$0
+	popf
+	push	$(SYSENTER_CS)		/* cs */
+	swapgs				/* switch to kernel gs (cpu_data) */
+L_sysenter_continue:
+	push	%rdx			/* eip */
+	push	%rax			/* err/eax - syscall code */
+	push	$(0)
+	movl	$(LO_SYSENTER), ISF64_TRAPFN(%rsp)
+	orl	$(EFL_IF), ISF64_RFLAGS(%rsp)
+
+L_32bit_enter_check:
+	/*
+	 * Check we're not a confused 64-bit user.
+	 */
+	cmpl	$(TASK_MAP_32BIT), %gs:CPU_TASK_MAP
+	jne	L_64bit_entry_reject
+	/* fall through to 32-bit handler: */
+
+L_32bit_enter:
+	/*
+	 * Make space for the compatibility save area.
+	 */
+	sub	$(ISC32_OFFSET), %rsp
+	movl	$(SS_32), SS_FLAVOR(%rsp)
+
+	/*
+	 * Save segment regs
+	 */
+	mov	%ds, R_DS(%rsp)
+	mov	%es, R_ES(%rsp)
+	mov	%fs, R_FS(%rsp)
+	mov	%gs, R_GS(%rsp)
+
+	/*
+	 * Save general 32-bit registers
+	 */
+	mov	%eax, R_EAX(%rsp)
+	mov	%ebx, R_EBX(%rsp)
+	mov	%ecx, R_ECX(%rsp)
+	mov	%edx, R_EDX(%rsp)
+	mov	%ebp, R_EBP(%rsp)
+	mov	%esi, R_ESI(%rsp)
+	mov	%edi, R_EDI(%rsp)
+
+	/* Unconditionally save cr2; only meaningful on page faults */
+	mov	%cr2, %rax
+	mov	%eax, R_CR2(%rsp)
+
+	/*
+	 * Copy registers already saved in the machine state 
+	 * (in the interrupt stack frame) into the compat save area.
+	 */
+	mov	ISC32_RIP(%rsp), %eax
+	mov	%eax, R_EIP(%rsp)
+	mov	ISC32_RFLAGS(%rsp), %eax
+	mov	%eax, R_EFLAGS(%rsp)
+	mov	ISC32_CS(%rsp), %eax
+	mov	%eax, R_CS(%rsp)
+	mov	ISC32_RSP(%rsp), %eax
+	mov	%eax, R_UESP(%rsp)
+	mov	ISC32_SS(%rsp), %eax
+	mov	%eax, R_SS(%rsp)
+L_32bit_enter_after_fault:
+	mov	ISC32_TRAPNO(%rsp), %ebx	/* %ebx := trapno for later */
+	mov	%ebx, R_TRAPNO(%rsp)
+	mov	ISC32_ERR(%rsp), %eax
+	mov	%eax, R_ERR(%rsp)
+	mov	ISC32_TRAPFN(%rsp), %edx
+
+/*
+ * Common point to enter lo_handler in compatibilty mode:
+ *	%ebx	trapno
+ *	%edx	locore handler address 
+ */
+L_enter_lohandler2:
+	/*
+	 * Switch address space to kernel
+	 * if not shared space and not already mapped.
+	 * Note: cpu_task_map is valid only if cpu_task_cr3 is loaded in cr3.
+	 */
+	mov	%cr3, %rax
+	mov	%gs:CPU_TASK_CR3, %rcx
+	cmp	%rax, %rcx			/* is the task's cr3 loaded? */
+	jne	1f
+	cmpl	$(TASK_MAP_64BIT_SHARED), %gs:CPU_TASK_MAP
+	je	2f
+1:
+	mov	%gs:CPU_KERNEL_CR3, %rcx
+	cmp	%rax, %rcx
+	je	2f
+	mov	%rcx, %cr3
+	mov	%rcx, %gs:CPU_ACTIVE_CR3
+2:
+	/*
+	 * Switch to compatibility mode.
+	 * Then establish kernel segments.
+	 */
+	swapgs					/* Done with uber-kernel gs */
+	ENTER_COMPAT_MODE()
+
+	/*
+	 * Now in compatibility mode and running in compatibility space
+	 * prepare to enter the locore handler.
+	 * 	%ebx		trapno
+	 *	%edx		lo_handler pointer
+	 * Note: the stack pointer (now 32-bit) is now directly addressing the
+	 * the kernel below 4G and therefore is automagically re-based.
+	 */
+	mov	$(KERNEL_DS), %eax
+	mov	%eax, %ss
+	mov	%eax, %ds
+	mov	%eax, %es
+	mov	%eax, %fs
+	mov	$(CPU_DATA_GS), %eax
+	mov	%eax, %gs
+
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* Get the active thread */
+	cmpl	$0, ACT_PCB_IDS(%ecx)	/* Is there a debug register state? */
+	je	1f
+	movl	$0, %ecx		/* If so, reset DR7 (the control) */
+	movl	%ecx, %dr7
+1:
+	addl	$1,%gs:hwIntCnt(,%ebx,4)	// Bump the trap/intr count
+	
+	/* Dispatch the designated lo handler */
+	jmp	*%edx
+
+	.code64
+L_64bit_entry_reject:
+	/*
+	 * Here for a 64-bit user attempting an invalid kernel entry.
+	 */
+	movl	$(LO_ALLTRAPS), ISF64_TRAPFN(%rsp)
+	movl	$(T_INVALID_OPCODE), ISF64_TRAPNO(%rsp)
+	/* Fall through... */
+	
+L_64bit_enter:
+	/*
+	 * Here for a 64-bit user task, or special 64-bit kernel code.
+	 * Make space for the save area.
+	 */
+	sub	$(ISS64_OFFSET), %rsp
+	movl	$(SS_64), SS_FLAVOR(%rsp)
+
+	/*
+	 * Save segment regs
+	 */
+	mov	%fs, R64_FS(%rsp)
+	mov	%gs, R64_GS(%rsp)
+
+	/* Save general-purpose registers */
+	mov	%rax, R64_RAX(%rsp)
+	mov	%rcx, R64_RCX(%rsp)
+	mov	%rbx, R64_RBX(%rsp)
+	mov	%rbp, R64_RBP(%rsp)
+	mov	%r11, R64_R11(%rsp)
+	mov	%r12, R64_R12(%rsp)
+	mov	%r13, R64_R13(%rsp)
+	mov	%r14, R64_R14(%rsp)
+	mov	%r15, R64_R15(%rsp)
+
+	/* cr2 is significant only for page-faults */
+	mov	%cr2, %rax
+	mov	%rax, R64_CR2(%rsp)
+
+	/* Other registers (which may contain syscall args) */
+	mov	%rdi, R64_RDI(%rsp)	/* arg0 .. */
+	mov	%rsi, R64_RSI(%rsp)
+	mov	%rdx, R64_RDX(%rsp)
+	mov	%r10, R64_R10(%rsp)
+	mov	%r8, R64_R8(%rsp)
+	mov	%r9, R64_R9(%rsp)	/* .. arg5 */
+
+L_64bit_enter_after_fault:
+	/*
+	 * At this point we're almost ready to join the common lo-entry code.
+	 */ 
+	mov	R64_TRAPNO(%rsp), %ebx
+	mov	R64_TRAPFN(%rsp), %edx
+
+	jmp	L_enter_lohandler2
+
+/*
+ * Debug trap.  Check for single-stepping across system call into
+ * kernel.  If this is the case, taking the debug trap has turned
+ * off single-stepping - save the flags register with the trace
+ * bit set.
+ */
+Entry(hi64_debug)
+	swapgs				/* set %gs for cpu data */
+	push	$0			/* error code */
+	push	$(T_DEBUG)
+	movl	$(LO_ALLTRAPS), ISF64_TRAPFN(%rsp)
+
+	testb	$3, ISF64_CS(%rsp)
+	jnz	L_enter_lohandler_continue
+
+	/*
+	 * trap came from kernel mode
+	 */
+	cmpl	$(KERNEL_UBER_BASE_HI32), ISF64_RIP+4(%rsp)
+	jne	L_enter_lohandler_continue	/* trap not in uber-space */
+
+	cmpl	$(EXT(hi64_mach_scall)), ISF64_RIP(%rsp)
+	jne	6f
+	add	$(ISF64_SIZE),%rsp	/* remove entire intr stack frame */
+	jmp	L_mach_scall_continue	/* continue system call entry */
+6:
+	cmpl	$(EXT(hi64_mdep_scall)), ISF64_RIP(%rsp)
+	jne	5f
+	add	$(ISF64_SIZE),%rsp	/* remove entire intr stack frame */
+	jmp	L_mdep_scall_continue	/* continue system call entry */
+5:
+	cmpl	$(EXT(hi64_unix_scall)), ISF64_RIP(%rsp)
+	jne	4f
+	add	$(ISF64_SIZE),%rsp	/* remove entire intr stack frame */
+	jmp	L_unix_scall_continue	/* continue system call entry */
+4:
+	cmpl	$(EXT(hi64_sysenter)), ISF64_RIP(%rsp)
+	jne	L_enter_lohandler_continue	
+	/*
+	 * Interrupt stack frame has been pushed on the temporary stack.
+	 * We have to switch to pcb stack and copy eflags.
+	 */ 
+	add	$32,%rsp		/* remove trapno/trapfn/err/rip/cs */
+	push	%rcx			/* save %rcx - user stack pointer */
+	mov	32(%rsp),%rcx		/* top of intr stack -> pcb stack */
+	xchg	%rcx,%rsp		/* switch to pcb stack */
+	push	$(USER_DS)		/* ss */
+	push	(%rcx)			/* saved %rcx into rsp slot */
+	push	8(%rcx)			/* rflags */
+	mov	(%rcx),%rcx		/* restore %rcx */
+	push	$(SYSENTER_TF_CS)	/* cs - not SYSENTER_CS for iret path */
+	jmp	L_sysenter_continue	/* continue sysenter entry */
+
+
+Entry(hi64_double_fault)
+	swapgs				/* set %gs for cpu data */
+	push	$(T_DOUBLE_FAULT)
+	movl	$(LO_DOUBLE_FAULT), ISF64_TRAPFN(%rsp)
+
+	cmpl	$(KERNEL_UBER_BASE_HI32), ISF64_RIP+4(%rsp)
+	jne	L_enter_lohandler_continue	/* trap not in uber-space */
+
+	cmpl	$(EXT(hi64_syscall)), ISF64_RIP(%rsp)
+	jne	L_enter_lohandler_continue
+
+	mov	ISF64_RSP(%rsp), %rsp
+	jmp	L_syscall_continue
+	
+
+/*
+ * General protection or segment-not-present fault.
+ * Check for a GP/NP fault in the kernel_return
+ * sequence; if there, report it as a GP/NP fault on the user's instruction.
+ *
+ * rsp->     0:	trap code (NP or GP) and trap function
+ *	     8:	segment number in error (error code)
+ *	    16	rip
+ *	    24	cs
+ *	    32	rflags 
+ *	    40	rsp
+ *	    48	ss
+ *	    56	old registers (trap is from kernel)
+ */
+Entry(hi64_gen_prot)
+	push	$(T_GENERAL_PROTECTION)
+	jmp	trap_check_kernel_exit	/* check for kernel exit sequence */
+
+Entry(hi64_stack_fault)
+	push	$(T_STACK_FAULT)
+	jmp	trap_check_kernel_exit	/* check for kernel exit sequence */
+
+Entry(hi64_segnp)
+	push	$(T_SEGMENT_NOT_PRESENT)
+					/* indicate fault type */
+trap_check_kernel_exit:
+	movl	$(LO_ALLTRAPS), 4(%rsp)
+	testb	$3,24(%rsp)
+	jnz	hi64_take_trap
+					/* trap was from kernel mode, so */
+					/* check for the kernel exit sequence */
+	cmpl	$(KERNEL_UBER_BASE_HI32), 16+4(%rsp)
+	jne	hi64_take_trap		/* trap not in uber-space */
+
+	cmpl	$(EXT(ret32_iret)), 16(%rsp)
+	je	L_fault_iret32
+	cmpl	$(EXT(ret32_set_ds)), 16(%rsp)
+	je	L_32bit_fault_set_seg
+	cmpl	$(EXT(ret32_set_es)), 16(%rsp)
+	je	L_32bit_fault_set_seg
+	cmpl	$(EXT(ret32_set_fs)), 16(%rsp)
+	je	L_32bit_fault_set_seg
+	cmpl	$(EXT(ret32_set_gs)), 16(%rsp)
+	je	L_32bit_fault_set_seg
+
+	cmpl	$(EXT(ret64_iret)), 16(%rsp)
+	je	L_fault_iret64
+
+hi64_take_trap:
+	jmp	L_enter_lohandler
+
+		
+/*
+ * GP/NP fault on IRET: CS or SS is in error.
+ * All registers contain the user's values.
+ *
+ * on SP is
+ *   0	trap number/function
+ *   8	errcode
+ *  16	rip
+ *  24	cs
+ *  32	rflags
+ *  40	rsp
+ *  48	ss			--> new trapno/trapfn
+ *  56  (16-byte padding)	--> new errcode
+ *  64	user rip
+ *  72	user cs
+ *  80	user rflags
+ *  88	user rsp
+ *  96  user ss
+ */
+L_fault_iret32:
+	mov	%rax, 16(%rsp)		/* save rax (we don`t need saved rip) */
+	mov	0(%rsp), %rax		/* get trap number */
+	mov	%rax, 48(%rsp)		/* put in user trap number */
+	mov	8(%rsp), %rax		/* get error code */
+	mov	%rax, 56(%rsp)		/* put in user errcode */
+	mov	16(%rsp), %rax		/* restore rax */
+	add	$48, %rsp		/* reset to original frame */
+					/* now treat as fault from user */
+	swapgs
+	jmp	L_32bit_enter
+
+L_fault_iret64:
+	mov	%rax, 16(%rsp)		/* save rax (we don`t need saved rip) */
+	mov	0(%rsp), %rax		/* get trap number */
+	mov	%rax, 48(%rsp)		/* put in user trap number */
+	mov	8(%rsp), %rax		/* get error code */
+	mov	%rax, 56(%rsp)		/* put in user errcode */
+	mov	16(%rsp), %rax		/* restore rax */
+	add	$48, %rsp		/* reset to original frame */
+					/* now treat as fault from user */
+	swapgs
+	jmp	L_64bit_enter
+
+/*
+ * Fault restoring a segment register.  All of the saved state is still
+ * on the stack untouched since we didn't move the stack pointer.
+ */
+L_32bit_fault_set_seg:
+	mov	0(%rsp), %rax		/* get trap number/function */
+	mov	8(%rsp), %rdx		/* get error code */
+	mov	40(%rsp), %rsp		/* reload stack prior to fault */
+	mov	%rax,ISC32_TRAPNO(%rsp)
+	mov	%rdx,ISC32_ERR(%rsp)
+					/* now treat as fault from user */
+					/* except that all the state is */
+					/* already saved - we just have to */
+					/* move the trapno and error into */
+					/* the compatibility frame */
+	swapgs
+	jmp	L_32bit_enter_after_fault
+
+
+/*
+ * Fatal exception handlers:
+ */
+Entry(db_task_dbl_fault64)
+	push	$(T_DOUBLE_FAULT)
+	movl	$(LO_DOUBLE_FAULT), ISF64_TRAPFN(%rsp)
+	jmp	L_enter_lohandler	
+
+Entry(db_task_stk_fault64)
+	push	$(T_STACK_FAULT)
+	movl	$(LO_DOUBLE_FAULT), ISF64_TRAPFN(%rsp)
+	jmp	L_enter_lohandler	
+
+Entry(mc64)
+	push	$(0)			/* Error */
+	push	$(T_MACHINE_CHECK)
+	movl	$(LO_MACHINE_CHECK), ISF64_TRAPFN(%rsp)
+	jmp	L_enter_lohandler	
diff --git a/osfmk/i386/io_emulate.c b/osfmk/i386/io_emulate.c
index 039f6a46a..a2320f877 100644
--- a/osfmk/i386/io_emulate.c
+++ b/osfmk/i386/io_emulate.c
@@ -76,7 +76,7 @@
 #if 1
 int
 emulate_io(
-	__unused struct i386_saved_state	*regs,
+	__unused x86_saved_state32_t		*regs,
 	__unused int				opcode,
 	__unused int				io_port)
 {
diff --git a/osfmk/i386/io_emulate.h b/osfmk/i386/io_emulate.h
index f523d90d4..977b6834a 100644
--- a/osfmk/i386/io_emulate.h
+++ b/osfmk/i386/io_emulate.h
@@ -64,7 +64,7 @@
  * Return codes from IO emulation.
  */
 extern int	emulate_io(
-			struct i386_saved_state	*regs,
+			x86_saved_state32_t	*regs,
 			int			opcode,
 			int			io_port);
 
diff --git a/osfmk/i386/io_map.c b/osfmk/i386/io_map.c
index 0c27621c6..40892578b 100644
--- a/osfmk/i386/io_map.c
+++ b/osfmk/i386/io_map.c
@@ -69,9 +69,7 @@ extern vm_offset_t	virtual_avail;
  * Mach VM is running.
  */
 vm_offset_t
-io_map(phys_addr, size)
-	vm_offset_t	phys_addr;
-	vm_size_t	size;
+io_map(vm_offset_t phys_addr, vm_size_t size, unsigned int flags)
 {
 	vm_offset_t	start;
 
@@ -83,19 +81,21 @@ io_map(phys_addr, size)
 	    virtual_avail += round_page(size);
 
 	    (void) pmap_map_bd(start, phys_addr, phys_addr + round_page(size),
-			       VM_PROT_READ|VM_PROT_WRITE);
+			       VM_PROT_READ|VM_PROT_WRITE,
+			       flags);
 	}
 	else {
 	    (void) kmem_alloc_pageable(kernel_map, &start, round_page(size));
 	    (void) pmap_map(start, phys_addr, phys_addr + round_page(size),
-			    VM_PROT_READ|VM_PROT_WRITE);
+			    VM_PROT_READ|VM_PROT_WRITE,
+			    flags);
 	}
 	return (start);
 }
 
 /* just wrap this since io_map handles it */
 
-vm_offset_t io_map_spec(vm_offset_t phys_addr, vm_size_t size)
+vm_offset_t io_map_spec(vm_offset_t phys_addr, vm_size_t size, unsigned int flags)
 {
-  return (io_map(phys_addr, size));
+  return (io_map(phys_addr, size, flags));
 }
diff --git a/osfmk/i386/io_map_entries.h b/osfmk/i386/io_map_entries.h
index e8c470f54..0913ba060 100644
--- a/osfmk/i386/io_map_entries.h
+++ b/osfmk/i386/io_map_entries.h
@@ -40,8 +40,9 @@
 __BEGIN_DECLS
 extern vm_offset_t	io_map(
 				vm_offset_t		phys_addr,
-				vm_size_t		size);
-extern vm_offset_t io_map_spec(vm_offset_t phys_addr, vm_size_t size);
+				vm_size_t		size,
+				unsigned int            flags);
+extern vm_offset_t io_map_spec(vm_offset_t phys_addr, vm_size_t size, unsigned int flags);
 __END_DECLS
 #endif	/* __APPLE_API_PRIVATE */
 
diff --git a/osfmk/i386/iopb.c b/osfmk/i386/iopb.c
index 8e86de4b2..1b3b0cb5d 100644
--- a/osfmk/i386/iopb.c
+++ b/osfmk/i386/iopb.c
@@ -400,7 +400,7 @@ i386_io_port_add(
 	/* Make sure the thread has a TSS. */
 
 	simple_lock(&pcb->lock);
-	io_tss = pcb->ims.io_tss;
+	io_tss = pcb->io_tss;
 	if (io_tss == 0) {
 	    if (new_io_tss == 0) {
 		/*
@@ -415,7 +415,7 @@ i386_io_port_add(
 		goto Retry;
 	    }
 	    io_tss = new_io_tss;
-	    pcb->ims.io_tss = io_tss;
+	    pcb->io_tss = io_tss;
 	    new_io_tss = 0;
 	}
 
@@ -489,7 +489,7 @@ i386_io_port_remove(
 	}
 
 	simple_lock(&pcb->lock);
-	io_tss = pcb->ims.io_tss;
+	io_tss = pcb->io_tss;
 	if (io_tss == 0) {
 	    simple_unlock(&pcb->lock);
 	    simple_unlock(&iopb_lock);
@@ -569,7 +569,7 @@ i386_io_port_list(thread, list, list_count)
 
 	    simple_lock(&iopb_lock);
 	    simple_lock(&pcb->lock);
-	    io_tss = pcb->ims.io_tss;
+	    io_tss = pcb->io_tss;
 	    if (io_tss != 0) {
 		register io_use_t iu;
 
@@ -654,7 +654,7 @@ iopb_check_mapping(
 	/* Look up the mapping in the device`s mapping list. */
 
 	queue_iterate(&io_port->io_use_list, iu, io_use_t, psq) {
-	    if (iu->ts == pcb->ims.io_tss) {
+	    if (iu->ts == pcb->io_tss) {
 		/*
 		 * Device is mapped.
 		 */
diff --git a/osfmk/i386/ktss.c b/osfmk/i386/ktss.c
index df65efe02..a947de6e6 100644
--- a/osfmk/i386/ktss.c
+++ b/osfmk/i386/ktss.c
@@ -62,13 +62,14 @@
  * We don't use the i386 task switch mechanism.  We need a TSS
  * only to hold the kernel stack pointer for the current thread.
  *
- * XXX multiprocessor??
  */
 #include <i386/tss.h>
 #include <i386/seg.h>
 #include <mach_kdb.h>
 
-struct i386_tss	ktss = {
+struct i386_tss	master_ktss
+	__attribute__ ((section ("__DESC, master_ktss")))
+	__attribute__ ((aligned (4096))) = {
 	0,				/* back link */
 	0,				/* esp0 */
 	KERNEL_DS,			/* ss0 */
@@ -100,9 +101,104 @@ struct i386_tss	ktss = {
 					   so no bitmap */
 };
 
+/*
+ * The transient stack for sysenter.
+ * At its top is a 32-bit link to the PCB in legacy mode, 64-bit otherwise.
+ * NB: it also must be large enough to contain a interrupt stack frame
+ * due to a single-step trace trap at system call entry.
+ */
+struct sysenter_stack master_sstk
+	__attribute__ ((section ("__DESC, master_sstk")))
+	__attribute__ ((aligned (16)))  = { {0}, 0 };
+
+#ifdef X86_64
+struct x86_64_tss master_ktss64 __attribute__ ((aligned (4096))) = {
+	.io_bit_map_offset = 0x0FFF,
+};
+#endif	/* X86_64 */
+ 
+
+
+/*
+ * Task structure for double-fault handler:
+ */
+struct i386_tss	master_dftss
+	__attribute__ ((section ("__DESC, master_dftss")))
+	__attribute__ ((aligned (4096))) = {
+	0,				/* back link */
+	(int) &df_task_stack_end - 4,	/* esp0 */
+	KERNEL_DS,			/* ss0 */
+	0,				/* esp1 */
+	0,				/* ss1 */
+	0,				/* esp2 */
+	0,				/* ss2 */
+	0,				/* cr3 */
+	(int) &df_task_start,		/* eip */
+	0,				/* eflags */
+	0,				/* eax */
+	0,				/* ecx */
+	0,				/* edx */
+	0,				/* ebx */
+	(int) &df_task_stack_end - 4,	/* esp */
+	0,				/* ebp */
+	0,				/* esi */
+	0,				/* edi */
+	KERNEL_DS,			/* es */
+	KERNEL_CS,			/* cs */
+	KERNEL_DS,			/* ss */
+	KERNEL_DS,			/* ds */
+	KERNEL_DS,			/* fs */
+	CPU_DATA_GS,			/* gs */
+	KERNEL_LDT,			/* ldt */
+	0,				/* trace_trap */
+	0x0FFF				/* IO bitmap offset -
+					   beyond end of TSS segment,
+					   so no bitmap */
+};
+
+
+/*
+ * Task structure for machine_check handler:
+ */
+struct i386_tss	master_mctss
+	__attribute__ ((section ("__DESC, master_mctss")))
+	__attribute__ ((aligned (4096))) = {
+	0,				/* back link */
+	(int) &mc_task_stack_end - 4,	/* esp0 */
+	KERNEL_DS,			/* ss0 */
+	0,				/* esp1 */
+	0,				/* ss1 */
+	0,				/* esp2 */
+	0,				/* ss2 */
+	0,				/* cr3 */
+	(int) &mc_task_start,		/* eip */
+	0,				/* eflags */
+	0,				/* eax */
+	0,				/* ecx */
+	0,				/* edx */
+	0,				/* ebx */
+	(int) &mc_task_stack_end - 4,	/* esp */
+	0,				/* ebp */
+	0,				/* esi */
+	0,				/* edi */
+	KERNEL_DS,			/* es */
+	KERNEL_CS,			/* cs */
+	KERNEL_DS,			/* ss */
+	KERNEL_DS,			/* ds */
+	KERNEL_DS,			/* fs */
+	CPU_DATA_GS,			/* gs */
+	KERNEL_LDT,			/* ldt */
+	0,				/* trace_trap */
+	0x0FFF				/* IO bitmap offset -
+					   beyond end of TSS segment,
+					   so no bitmap */
+};
+
 #if	MACH_KDB
 
-struct i386_tss	dbtss = {
+struct i386_tss	master_dbtss
+	__attribute__ ((section ("__DESC, master_dbtss")))
+	__attribute__ ((aligned (4096))) = {
 	0,				/* back link */
 	0,				/* esp0 */
 	KERNEL_DS,			/* ss0 */
diff --git a/osfmk/i386/ldt.c b/osfmk/i386/ldt.c
index e2d0ee1c2..13ed8cd32 100644
--- a/osfmk/i386/ldt.c
+++ b/osfmk/i386/ldt.c
@@ -65,46 +65,42 @@
 #include <mach/i386/vm_types.h>
 #include <mach/i386/vm_param.h>
 
-extern int	syscall(void);
-extern int	mach_rpc(void);
-
-struct fake_descriptor	ldt[LDTSZ] = {
-/*007*/	{ (unsigned int)&syscall,
-	  KERNEL_CS,
-	  0,				/* no parameters */
-	  ACC_P|ACC_PL_U|ACC_CALL_GATE
-	},				/* call gate for system calls */
-/*00F*/	{ (unsigned int)&mach_rpc,
-          KERNEL_CS,
-          0,                            /* no parameters */
-          ACC_P|ACC_PL_U|ACC_CALL_GATE
-        },				/* call gate for mach rpc */
-/*017*/	{ 0,
-	  0xfffff,
- 	  SZ_32|SZ_G,
-	  ACC_P|ACC_PL_U|ACC_CODE_R
-	},				/* user code segment */
-/*01F*/	{ 0,
-	  0xfffff,
-	  SZ_32|SZ_G,
-	  ACC_P|ACC_PL_U|ACC_DATA_W
-	},				/* user data segment */
-/*027*/	{ 0,
-	  0xfffff,
-	  SZ_32|SZ_G,
-	  ACC_P|ACC_PL_U|ACC_DATA_W
-	},				/* user cthread segment */
-// Storage space for user ldt entries we will make room for 10 entries initially
-// as we will probably never need many more than that
-/*02F*/	{ 0, 0, 0, 0},
-/*037*/	{ 0, 0, 0, 0},
-/*03F*/	{ 0, 0, 0, 0},
-/*047*/	{ 0, 0, 0, 0},
-/*04F*/	{ 0, 0, 0, 0},
-/*057*/	{ 0, 0, 0, 0},
-/*05F*/	{ 0, 0, 0, 0},
-/*067*/	{ 0, 0, 0, 0},
-/*06F*/	{ 0, 0, 0, 0},
-/*077*/	{ 0, 0, 0, 0},
 
+struct fake_descriptor	master_ldt[LDTSZ] __attribute__ ((aligned (4096))) = {
+	[SEL_TO_INDEX(SYSENTER_CS)] {	/* kernel code (sysenter) */
+		0,
+		0xfffff,
+		SZ_32|SZ_G,
+		ACC_P|ACC_PL_K|ACC_CODE_R
+	},
+	[SEL_TO_INDEX(SYSENTER_DS)] {	/* kernel data (sysenter) */
+		0,
+		0xfffff,
+		SZ_32|SZ_G,
+		ACC_P|ACC_PL_K|ACC_DATA_W
+	},
+	[SEL_TO_INDEX(USER_CS)] {	/* user code segment */
+		0,
+		0xfffff,
+		SZ_32|SZ_G,
+		ACC_P|ACC_PL_U|ACC_CODE_R
+	},
+	[SEL_TO_INDEX(USER_DS)] {	/* user data segment */
+		0,
+		0xfffff,
+		SZ_32|SZ_G,
+		ACC_P|ACC_PL_U|ACC_DATA_W
+	},
+	[SEL_TO_INDEX(USER64_CS)] {	/* user 64-bit code segment */
+		0,
+		0xfffff,
+		SZ_64|SZ_G,
+		ACC_P|ACC_PL_U|ACC_CODE_R
+	},
+	[SEL_TO_INDEX(USER_CTHREAD)] {	/* user cthread segment */
+		0,
+		0xfffff,
+		SZ_32|SZ_G,
+		ACC_P|ACC_PL_U|ACC_DATA_W
+	},
 };
diff --git a/osfmk/i386/lock.h b/osfmk/i386/lock.h
index 277ce8563..ef9f4cff3 100644
--- a/osfmk/i386/lock.h
+++ b/osfmk/i386/lock.h
@@ -93,15 +93,7 @@ typedef struct {
 #endif  /* MACH_LDEBUG */
 } mutex_t;
 
-typedef struct {
-	decl_simple_lock_data(,interlock) /* "hardware" interlock field */
-	volatile unsigned int
-		read_count:16,	/* No. of accepted readers */
-		want_upgrade:1,	/* Read-to-write upgrade waiting */
-		want_write:1,	/* Writer is waiting, or locked for write */ 
-		waiting:1,		/* Someone is sleeping on lock */
-	can_sleep:1;		/* Can attempts to lock go to sleep? */
-} lock_t;
+typedef lck_rw_t lock_t;
 
 extern unsigned int LockTimeOut;			/* Number of hardware ticks of a lock timeout */
 
@@ -158,21 +150,21 @@ static inline unsigned long i_bit_isset(unsigned int test, volatile unsigned lon
 
 static inline char	xchgb(volatile char * cp, char new);
 
-static inline void	atomic_incl(long * p, long delta);
-static inline void	atomic_incs(short * p, short delta);
-static inline void	atomic_incb(char * p, char delta);
+static inline void	atomic_incl(volatile long * p, long delta);
+static inline void	atomic_incs(volatile short * p, short delta);
+static inline void	atomic_incb(volatile char * p, char delta);
 
-static inline void	atomic_decl(long * p, long delta);
-static inline void	atomic_decs(short * p, short delta);
-static inline void	atomic_decb(char * p, char delta);
+static inline void	atomic_decl(volatile long * p, long delta);
+static inline void	atomic_decs(volatile short * p, short delta);
+static inline void	atomic_decb(volatile char * p, char delta);
 
-static inline long	atomic_getl(long * p);
-static inline short	atomic_gets(short * p);
-static inline char	atomic_getb(char * p);
+static inline long	atomic_getl(const volatile long * p);
+static inline short	atomic_gets(const volatile short * p);
+static inline char	atomic_getb(const volatile char * p);
 
-static inline void	atomic_setl(long * p, long value);
-static inline void	atomic_sets(short * p, short value);
-static inline void	atomic_setb(char * p, char value);
+static inline void	atomic_setl(volatile long * p, long value);
+static inline void	atomic_sets(volatile short * p, short value);
+static inline void	atomic_setb(volatile char * p, char value);
 
 static inline char	xchgb(volatile char * cp, char new)
 {
@@ -205,35 +197,7 @@ atomic_cmpxchg(uint32_t *p, uint32_t old, uint32_t new)
 	return (res);
 }
 
-static inline uint64_t
-atomic_load64(uint64_t *quadp)
-{
-	uint64_t ret;
-
-	asm volatile(
-		"	lock; cmpxchg8b	%1"
-		: "=A" (ret)
-		: "m" (*quadp), "a" (0), "d" (0), "b" (0), "c" (0));
-	return (ret);
-}
-
-static inline uint64_t
-atomic_loadstore64(uint64_t *quadp, uint64_t new)
-{
-	uint64_t ret;
-
-	ret = *quadp;
-	asm volatile(
-		"1:				\n\t"
-		"	lock; cmpxchg8b	%1	\n\t"
-		"	jnz	1b"
-		: "+A" (ret)
-		: "m" (*quadp),
-		  "b" ((uint32_t)new), "c" ((uint32_t)(new >> 32)));
-	return (ret);
-}
-
-static inline void	atomic_incl(long * p, long delta)
+static inline void	atomic_incl(volatile long * p, long delta)
 {
 	__asm__ volatile ("	lock		\n		\
 				addl    %0,%1"		:	\
@@ -241,7 +205,7 @@ static inline void	atomic_incl(long * p, long delta)
 				"r" (delta), "m" (*(volatile long *)p));
 }
 
-static inline void	atomic_incs(short * p, short delta)
+static inline void	atomic_incs(volatile short * p, short delta)
 {
 	__asm__ volatile ("	lock		\n		\
 				addw    %0,%1"		:	\
@@ -249,7 +213,7 @@ static inline void	atomic_incs(short * p, short delta)
 				"q" (delta), "m" (*(volatile short *)p));
 }
 
-static inline void	atomic_incb(char * p, char delta)
+static inline void	atomic_incb(volatile char * p, char delta)
 {
 	__asm__ volatile ("	lock		\n		\
 				addb    %0,%1"		:	\
@@ -257,7 +221,7 @@ static inline void	atomic_incb(char * p, char delta)
 				"q" (delta), "m" (*(volatile char *)p));
 }
 
-static inline void	atomic_decl(long * p, long delta)
+static inline void	atomic_decl(volatile long * p, long delta)
 {
 	__asm__ volatile ("	lock		\n		\
 				subl	%0,%1"		:	\
@@ -265,7 +229,7 @@ static inline void	atomic_decl(long * p, long delta)
 				"r" (delta), "m" (*(volatile long *)p));
 }
 
-static inline int	atomic_decl_and_test(long * p, long delta)
+static inline int	atomic_decl_and_test(volatile long * p, long delta)
 {
 	uint8_t	ret;
 	asm volatile (
@@ -277,7 +241,7 @@ static inline int	atomic_decl_and_test(long * p, long delta)
 	return ret;
 }
 
-static inline void	atomic_decs(short * p, short delta)
+static inline void	atomic_decs(volatile short * p, short delta)
 {
 	__asm__ volatile ("	lock		\n		\
 				subw    %0,%1"		:	\
@@ -285,7 +249,7 @@ static inline void	atomic_decs(short * p, short delta)
 				"q" (delta), "m" (*(volatile short *)p));
 }
 
-static inline void	atomic_decb(char * p, char delta)
+static inline void	atomic_decb(volatile char * p, char delta)
 {
 	__asm__ volatile ("	lock		\n		\
 				subb    %0,%1"		:	\
@@ -293,32 +257,32 @@ static inline void	atomic_decb(char * p, char delta)
 				"q" (delta), "m" (*(volatile char *)p));
 }
 
-static inline long	atomic_getl(long * p)
+static inline long	atomic_getl(const volatile long * p)
 {
 	return (*p);
 }
 
-static inline short	atomic_gets(short * p)
+static inline short	atomic_gets(const volatile short * p)
 {
 	return (*p);
 }
 
-static inline char	atomic_getb(char * p)
+static inline char	atomic_getb(const volatile char * p)
 {
 	return (*p);
 }
 
-static inline void	atomic_setl(long * p, long value)
+static inline void	atomic_setl(volatile long * p, long value)
 {
 	*p = value;
 }
 
-static inline void	atomic_sets(short * p, short value)
+static inline void	atomic_sets(volatile short * p, short value)
 {
 	*p = value;
 }
 
-static inline void	atomic_setb(char * p, char value)
+static inline void	atomic_setb(volatile char * p, char value)
 {
 	*p = value;
 }
diff --git a/osfmk/i386/locks.h b/osfmk/i386/locks.h
index 5bb341ab9..3129d79bd 100644
--- a/osfmk/i386/locks.h
+++ b/osfmk/i386/locks.h
@@ -88,6 +88,12 @@ typedef struct _lck_mtx_ {
 #define	LCK_MTX_TAG_INDIRECT			0x00001007	/* lock marked as Indirect  */
 #define	LCK_MTX_TAG_DESTROYED			0x00002007	/* lock marked as Destroyed */
 
+/* Adaptive spin before blocking */
+extern unsigned int	MutexSpin;
+extern void		lck_mtx_lock_spin(lck_mtx_t *lck);
+
+extern void		lck_mtx_interlock_spin(lck_mtx_t *lck);
+
 typedef struct {
 	unsigned int		type;
 	vm_offset_t		pc;
@@ -109,9 +115,9 @@ typedef struct _lck_mtx_ext_ {
 } lck_mtx_ext_t;
 
 #define	LCK_MTX_ATTR_DEBUG	0x1
-#define	LCK_MTX_ATTR_DEBUGb	31
+#define	LCK_MTX_ATTR_DEBUGb	0
 #define	LCK_MTX_ATTR_STAT	0x2
-#define	LCK_MTX_ATTR_STATb	30
+#define	LCK_MTX_ATTR_STATb	1
 
 #else
 #ifdef	KERNEL_PRIVATE
@@ -129,12 +135,24 @@ typedef struct {
 	volatile unsigned int
 						read_count:16,	/* No. of accepted readers */
 						want_upgrade:1,	/* Read-to-write upgrade waiting */
-						want_write:1,	/* Writer is waiting, or locked for write */
+			want_write:1,	/* Writer waiting or locked for write */
 						waiting:1,		/* Someone is sleeping on lock */
-						can_sleep:1;	/* Can attempts to lock go to sleep? */
+			can_sleep:1,	/* Can attempts to lock go to sleep? */
+			read_priority:1;/* New read takes piority over write */
 	unsigned int		lck_rw_tag;
 } lck_rw_t;
 
+#define	LCK_RW_ATTR_DEBUG	0x1
+#define	LCK_RW_ATTR_DEBUGb	0
+#define	LCK_RW_ATTR_STAT	0x2
+#define	LCK_RW_ATTR_STATb	1
+#define LCK_RW_ATTR_READ_PRI	0x3
+#define LCK_RW_ATTR_READ_PRIb	2
+#define	LCK_RW_ATTR_DIS_THREAD	0x40000000
+#define	LCK_RW_ATTR_DIS_THREADb	30
+#define	LCK_RW_ATTR_DIS_MYLOCK	0x10000000
+#define	LCK_RW_ATTR_DIS_MYLOCKb	28
+
 #define	LCK_RW_TAG_DESTROYED		0x00002007	/* lock marked as Destroyed */
 
 #else
diff --git a/osfmk/i386/locks_i386.c b/osfmk/i386/locks_i386.c
index 846038183..860aa64b8 100644
--- a/osfmk/i386/locks_i386.c
+++ b/osfmk/i386/locks_i386.c
@@ -84,9 +84,7 @@
 #include <ddb/db_print.h>
 #endif	/* MACH_KDB */
 
-#ifdef __ppc__
-#include <ppc/Firmware.h>
-#endif
+#include <i386/machine_cpu.h>
 
 #include <sys/kdebug.h>
 
@@ -97,6 +95,7 @@
 #define	LCK_RW_LCK_SH_TO_EX1_CODE	0x104
 #define	LCK_RW_LCK_EX_TO_SH_CODE	0x105
 
+#define	LCK_MTX_LCK_SPIN		0x200
 
 #define	ANY_LOCK_DEBUG	(USLOCK_DEBUG || LOCK_DEBUG || MUTEX_DEBUG)
 
@@ -255,7 +254,7 @@ boolean_t
 lck_spin_try_lock(
 	lck_spin_t	*lck)
 {
-	usimple_lock_try((usimple_lock_t) lck);
+	return(usimple_lock_try((usimple_lock_t) lck));
 }
 
 /*
@@ -671,15 +670,14 @@ lock_init(
 	lock_t		*l,
 	boolean_t	can_sleep,
 	__unused unsigned short	tag,
-	unsigned short	tag1)
+	__unused unsigned short	tag1)
 {
-	(void) memset((void *) l, 0, sizeof(lock_t));
-
-	simple_lock_init(&l->interlock, tag1);
+	hw_lock_init(&l->interlock);
 	l->want_write = FALSE;
 	l->want_upgrade = FALSE;
 	l->read_count = 0;
 	l->can_sleep = can_sleep;
+	l->lck_rw_tag = tag;
 }
 
 
@@ -695,162 +693,21 @@ void
 lock_write(
 	register lock_t	* l)
 {
-        register int	   i;
-	boolean_t          lock_miss = FALSE;
-#if	MACH_LDEBUG
-	int		   decrementer;
-#endif	/* MACH_LDEBUG */
-
-	simple_lock(&l->interlock);
-
-#if	MACH_LDEBUG
-	decrementer = DECREMENTER_TIMEOUT;
-#endif	/* MACH_LDEBUG */
-
-	/*
-	 *	Try to acquire the want_write bit.
-	 */
-	while (l->want_write) {
-		if (!lock_miss) {
-			lock_miss = TRUE;
-		}
-
-		i = lock_wait_time[l->can_sleep ? 1 : 0];
-		if (i != 0) {
-			simple_unlock(&l->interlock);
-#if	MACH_LDEBUG
-			if (!--decrementer)
-				Debugger("timeout - want_write");
-#endif	/* MACH_LDEBUG */
-			while (--i != 0 && l->want_write)
-				continue;
-			simple_lock(&l->interlock);
-		}
-
-		if (l->can_sleep && l->want_write) {
-			l->waiting = TRUE;
-			thread_sleep_simple_lock((event_t) l,
-					simple_lock_addr(l->interlock),
-					THREAD_UNINT);
-			/* interlock relocked */
-		}
-	}
-	l->want_write = TRUE;
-
-	/* Wait for readers (and upgrades) to finish */
-
-#if	MACH_LDEBUG
-	decrementer = DECREMENTER_TIMEOUT;
-#endif	/* MACH_LDEBUG */
-	while ((l->read_count != 0) || l->want_upgrade) {
-		if (!lock_miss) {
-			lock_miss = TRUE;
-		}
-
-		i = lock_wait_time[l->can_sleep ? 1 : 0];
-		if (i != 0) {
-			simple_unlock(&l->interlock);
-#if	MACH_LDEBUG
-			if (!--decrementer)
-				Debugger("timeout - wait for readers");
-#endif	/* MACH_LDEBUG */
-			while (--i != 0 && (l->read_count != 0 ||
-					    l->want_upgrade))
-				continue;
-			simple_lock(&l->interlock);
-		}
-
-		if (l->can_sleep && (l->read_count != 0 || l->want_upgrade)) {
-			l->waiting = TRUE;
-			thread_sleep_simple_lock((event_t) l,
-				simple_lock_addr(l->interlock),
-				THREAD_UNINT);
-			/* interlock relocked */
-		}
-	}
-
-	simple_unlock(&l->interlock);
+	lck_rw_lock_exclusive(l);
 }
 
 void
 lock_done(
 	register lock_t	* l)
 {
-	boolean_t	  do_wakeup = FALSE;
-
-
-	simple_lock(&l->interlock);
-
-	if (l->read_count != 0) {
-		l->read_count--;
-	}
-	else	
-		if (l->want_upgrade) {
-			l->want_upgrade = FALSE;
-		}
-	else {
-		l->want_write = FALSE;
-	}
-
-	/*
-	 *	There is no reason to wakeup a waiting thread
-	 *	if the read-count is non-zero.  Consider:
-	 *		we must be dropping a read lock
-	 *		threads are waiting only if one wants a write lock
-	 *		if there are still readers, they can't proceed
-	 */
-
-	if (l->waiting && (l->read_count == 0)) {
-		l->waiting = FALSE;
-		do_wakeup = TRUE;
-	}
-
-	simple_unlock(&l->interlock);
-
-	if (do_wakeup)
-		thread_wakeup((event_t) l);
+	(void) lck_rw_done(l);
 }
 
 void
 lock_read(
 	register lock_t	* l)
 {
-	register int	    i;
-#if	MACH_LDEBUG
-	int		   decrementer;
-#endif	/* MACH_LDEBUG */
-
-	simple_lock(&l->interlock);
-
-#if	MACH_LDEBUG
-	decrementer = DECREMENTER_TIMEOUT;
-#endif	/* MACH_LDEBUG */
-	while (l->want_write || l->want_upgrade) {
-		i = lock_wait_time[l->can_sleep ? 1 : 0];
-
-		if (i != 0) {
-			simple_unlock(&l->interlock);
-#if	MACH_LDEBUG
-			if (!--decrementer)
-				Debugger("timeout - wait no writers");
-#endif	/* MACH_LDEBUG */
-			while (--i != 0 && (l->want_write || l->want_upgrade))
-				continue;
-			simple_lock(&l->interlock);
-		}
-
-		if (l->can_sleep && (l->want_write || l->want_upgrade)) {
-			l->waiting = TRUE;
-			thread_sleep_simple_lock((event_t) l,
-					simple_lock_addr(l->interlock),
-					THREAD_UNINT);
-			/* interlock relocked */
-		}
-	}
-
-	l->read_count++;
-
-	simple_unlock(&l->interlock);
+	lck_rw_lock_shared(l);
 }
 
 
@@ -869,157 +726,16 @@ boolean_t
 lock_read_to_write(
 	register lock_t	* l)
 {
-	register int	    i;
-	boolean_t	    do_wakeup = FALSE;
-#if	MACH_LDEBUG
-	int		   decrementer;
-#endif	/* MACH_LDEBUG */
-
-	simple_lock(&l->interlock);
-
-	l->read_count--;	
-
-	if (l->want_upgrade) {
-		/*
-		 *	Someone else has requested upgrade.
-		 *	Since we've released a read lock, wake
-		 *	him up.
-		 */
-		if (l->waiting && (l->read_count == 0)) {
-			l->waiting = FALSE;
-			do_wakeup = TRUE;
-		}
-
-		simple_unlock(&l->interlock);
-
-		if (do_wakeup)
-			thread_wakeup((event_t) l);
-		return (TRUE);
-	}
-
-	l->want_upgrade = TRUE;
-
-#if	MACH_LDEBUG
-	decrementer = DECREMENTER_TIMEOUT;
-#endif	/* MACH_LDEBUG */
-	while (l->read_count != 0) {
-		i = lock_wait_time[l->can_sleep ? 1 : 0];
-
-		if (i != 0) {
-			simple_unlock(&l->interlock);
-#if	MACH_LDEBUG
-			if (!--decrementer)
-				Debugger("timeout - read_count");
-#endif	/* MACH_LDEBUG */
-			while (--i != 0 && l->read_count != 0)
-				continue;
-			simple_lock(&l->interlock);
-		}
-
-		if (l->can_sleep && l->read_count != 0) {
-			l->waiting = TRUE;
-			thread_sleep_simple_lock((event_t) l,
-					simple_lock_addr(l->interlock),
-					THREAD_UNINT);
-			/* interlock relocked */
-		}
-	}
-
-	simple_unlock(&l->interlock);
-
-	return (FALSE);
+	return lck_rw_lock_shared_to_exclusive(l);
 }
 
 void
 lock_write_to_read(
 	register lock_t	* l)
 {
-	boolean_t	   do_wakeup = FALSE;
-
-	simple_lock(&l->interlock);
-
-	l->read_count++;
-	if (l->want_upgrade)
-		l->want_upgrade = FALSE;
-	else
-	 	l->want_write = FALSE;
-
-	if (l->waiting) {
-		l->waiting = FALSE;
-		do_wakeup = TRUE;
-	}
-
-	simple_unlock(&l->interlock);
-
-	if (do_wakeup)
-		thread_wakeup((event_t) l);
-}
-
-
-#if	0	/* Unused */
-/*
- *	Routine:	lock_try_write
- *	Function:
- *		Tries to get a write lock.
- *
- *		Returns FALSE if the lock is not held on return.
- */
-
-boolean_t
-lock_try_write(
-	register lock_t	* l)
-{
-	pc_t		   pc;
-
-	simple_lock(&l->interlock);
-
-	if (l->want_write || l->want_upgrade || l->read_count) {
-		/*
-		 *	Can't get lock.
-		 */
-		simple_unlock(&l->interlock);
-		return(FALSE);
-	}
-
-	/*
-	 *	Have lock.
-	 */
-
-	l->want_write = TRUE;
-
-	simple_unlock(&l->interlock);
-
-	return(TRUE);
+	lck_rw_lock_exclusive_to_shared(l);
 }
 
-/*
- *	Routine:	lock_try_read
- *	Function:
- *		Tries to get a read lock.
- *
- *		Returns FALSE if the lock is not held on return.
- */
-
-boolean_t
-lock_try_read(
-	register lock_t	* l)
-{
-	pc_t		   pc;
-
-	simple_lock(&l->interlock);
-
-	if (l->want_write || l->want_upgrade) {
-		simple_unlock(&l->interlock);
-		return(FALSE);
-	}
-
-	l->read_count++;
-
-	simple_unlock(&l->interlock);
-
-	return(TRUE);
-}
-#endif		/* Unused */
 
 
 /*
@@ -1055,7 +771,10 @@ void
 lck_rw_init(
 	lck_rw_t	*lck,
 	lck_grp_t	*grp,
-	__unused lck_attr_t	*attr) {
+	lck_attr_t	*attr)
+{
+	lck_attr_t	*lck_attr = (attr != LCK_ATTR_NULL) ?
+					attr : &LockDefaultLckAttr;
 
 	hw_lock_init(&lck->interlock);
 	lck->want_write = FALSE;
@@ -1063,6 +782,8 @@ lck_rw_init(
 	lck->read_count = 0;
 	lck->can_sleep = TRUE;
 	lck->lck_rw_tag = 0;
+	lck->read_priority = (lck_attr->lck_attr_val &
+				 LCK_ATTR_RW_SHARED_PRIORITY) != 0;
 
 	lck_grp_reference(grp);
 	lck_grp_lckcnt_incr(grp, LCK_TYPE_RW);
@@ -1115,6 +836,21 @@ lck_interlock_unlock(lck_rw_t *lck, boolean_t istate)
 	ml_set_interrupts_enabled(istate);
 }
 
+
+/*
+ * This inline is used when busy-waiting for an rw lock.
+ * If interrupts were disabled when the lock primitive was called,
+ * we poll the IPI handler for pending tlb flushes.
+ * XXX This is a hack to avoid deadlocking on the pmap_system_lock.
+ */
+static inline void
+lck_rw_lock_pause(boolean_t interrupts_enabled)
+{
+	if (!interrupts_enabled)
+		handle_pending_TLB_flushes();
+	cpu_pause();
+}
+
 /*
  *      Routine:        lck_rw_lock_exclusive
  */
@@ -1154,7 +890,7 @@ lck_rw_lock_exclusive(
 				Debugger("timeout - want_write");
 #endif	/* MACH_LDEBUG */
 			while (--i != 0 && lck->want_write)
-				continue;
+				lck_rw_lock_pause(istate);
 			istate = lck_interlock_lock(lck);
 		}
 
@@ -1194,7 +930,7 @@ lck_rw_lock_exclusive(
 #endif	/* MACH_LDEBUG */
 			while (--i != 0 && (lck->read_count != 0 ||
 					    lck->want_upgrade))
-				continue;
+				lck_rw_lock_pause(istate);
 			istate = lck_interlock_lock(lck);
 		}
 
@@ -1349,7 +1085,10 @@ lck_rw_lock_shared(
 #if	MACH_LDEBUG
 	decrementer = DECREMENTER_TIMEOUT;
 #endif	/* MACH_LDEBUG */
-	while (lck->want_write || lck->want_upgrade) {
+	while ((lck->want_write && (lck->read_priority ?
+					lck->read_count == 0 : TRUE)) ||
+	       lck->want_upgrade) {
+
 		i = lock_wait_time[lck->can_sleep ? 1 : 0];
 
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_CODE) | DBG_FUNC_START,
@@ -1361,12 +1100,18 @@ lck_rw_lock_shared(
 			if (!--decrementer)
 				Debugger("timeout - wait no writers");
 #endif	/* MACH_LDEBUG */
-			while (--i != 0 && (lck->want_write || lck->want_upgrade))
-				continue;
+			while (--i != 0 &&
+				((lck->want_write && (lck->read_priority ?
+					lck->read_count == 0 : TRUE)) ||
+				 lck->want_upgrade))
+				lck_rw_lock_pause(istate);
 			istate = lck_interlock_lock(lck);
 		}
 
-		if (lck->can_sleep && (lck->want_write || lck->want_upgrade)) {
+		if (lck->can_sleep &&
+			((lck->want_write && (lck->read_priority ?
+						lck->read_count == 0 : TRUE)) ||
+			  lck->want_upgrade)) {
 			lck->waiting = TRUE;
 			res = assert_wait((event_t) lck, THREAD_UNINT);
 			if (res == THREAD_WAITING) {
@@ -1455,7 +1200,7 @@ lck_rw_lock_shared_to_exclusive(
 				Debugger("timeout - read_count");
 #endif	/* MACH_LDEBUG */
 			while (--i != 0 && lck->read_count != 0)
-				continue;
+				lck_rw_lock_pause(istate);
 			istate = lck_interlock_lock(lck);
 		}
 
@@ -1705,6 +1450,73 @@ lck_mtx_assert(
 {
 }
 
+/*
+ * Routine: 	lck_mtx_lock_spin
+ *
+ * Invoked trying to acquire a mutex when there is contention but
+ * the holder is running on another processor. We spin for up to a maximum
+ * time waiting for the lock to be released.
+ *
+ * Called with the interlock unlocked.
+ */
+void
+lck_mtx_lock_spin(
+	lck_mtx_t		*lck)
+{
+	thread_t		holder;
+	lck_mtx_t		*mutex;
+	uint64_t		deadline;
+
+	if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
+		mutex = lck;
+	else
+		mutex = &lck->lck_mtx_ptr->lck_mtx;
+
+        KERNEL_DEBUG(
+		MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_SPIN) | DBG_FUNC_START,
+		(int)lck, (int)mutex->lck_mtx_locked, 0, 0, 0);
+
+	deadline = mach_absolute_time() + MutexSpin;
+	/*
+	 * Spin while:
+	 *   - mutex is locked, and
+	 *   - owner is running on another processor, and
+	 *   - owner is not is the idle delay, and
+	 *   - we haven't spun for long enough.
+	 */
+	while ((holder = (thread_t) mutex->lck_mtx_locked) != NULL &&
+	       (holder->machine.specFlags & OnProc) != 0 &&
+	       (holder->options & TH_OPT_DELAYIDLE) == 0 &&
+	       mach_absolute_time() < deadline)
+		cpu_pause();
+}
+
+/*
+ * Called from assembly code when a mutex interlock is held.
+ * We spin here re-checking the interlock but panic if we timeout.
+ * Note: here with interrupts disabled.
+ */
+void
+lck_mtx_interlock_spin(
+	lck_mtx_t		*lck)
+{
+	lck_mtx_t		*mutex;
+	uint64_t		deadline;
+
+	if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
+		mutex = lck;
+	else
+		mutex = &lck->lck_mtx_ptr->lck_mtx;
+
+	deadline = mach_absolute_time() + LockTimeOut;
+	while (mutex->lck_mtx_ilk != 0) {
+		cpu_pause();
+		if (mach_absolute_time() > deadline)
+			panic("interlock timeout for mutex %p", lck);
+	}
+
+}
+
 #if	MACH_KDB
 
 void	db_show_one_lock(lock_t  *);
@@ -1720,7 +1532,7 @@ db_show_one_lock(
 	db_printf("%swaiting, %scan_sleep\n", 
 		  lock->waiting ? "" : "!", lock->can_sleep ? "" : "!");
 	db_printf("Interlock:\n");
-	db_show_one_simple_lock((db_expr_t)simple_lock_addr(lock->interlock),
+	db_show_one_simple_lock((db_expr_t) ((vm_offset_t)simple_lock_addr(lock->interlock)),
 			TRUE, (db_expr_t)0, (char *)0);
 }
 
@@ -1802,17 +1614,17 @@ _mutex_assert (
  * fashion.
  */
 
-char *simple_lock_labels =	"ENTRY    ILK THREAD   DURATION CALLER";
-char *mutex_labels =		"ENTRY    LOCKED WAITERS   THREAD CALLER";
+const char *simple_lock_labels =	"ENTRY    ILK THREAD   DURATION CALLER";
+const char *mutex_labels =		"ENTRY    LOCKED WAITERS   THREAD CALLER";
 
 void
 db_show_one_simple_lock (
 	db_expr_t	addr,
 	boolean_t	have_addr,
-	db_expr_t	count,
-	char		* modif)
+	__unused db_expr_t	count,
+	__unused char		* modif)
 {
-	simple_lock_t	saddr = (simple_lock_t)addr;
+	simple_lock_t	saddr = (simple_lock_t) ((vm_offset_t) addr);
 
 	if (saddr == (simple_lock_t)0 || !have_addr) {
 		db_error ("No simple_lock\n");
@@ -1844,10 +1656,10 @@ void
 db_show_one_mutex (
 	db_expr_t	addr,
 	boolean_t	have_addr,
-	db_expr_t	count,
-	char		* modif)
+	__unused db_expr_t	count,
+	__unused char		* modif)
 {
-	mutex_t		* maddr = (mutex_t *)addr;
+	mutex_t		* maddr = (mutex_t *)((vm_offset_t) addr);
 
 	if (maddr == (mutex_t *)0 || !have_addr)
 		db_error ("No mutex\n");
diff --git a/osfmk/i386/locore.s b/osfmk/i386/locore.s
index a17902e62..166c843f5 100644
--- a/osfmk/i386/locore.s
+++ b/osfmk/i386/locore.s
@@ -71,10 +71,10 @@
 #include <assym.s>
 #include <mach/exception_types.h>
 
-#include <i386/mp.h>
-
-#define	PREEMPT_DEBUG_LOG 0
+#define _ARCH_I386_ASM_HELP_H_          /* Prevent inclusion of user header */
+#include <mach/i386/syscall_sw.h>
 
+#include <i386/mp.h>
 
 /*
  * PTmap is recursive pagemap at top of virtual address space.
@@ -108,6 +108,49 @@
 
 #define	CX(addr,reg)	addr(,reg,4)
 
+/*
+ * The following macros make calls into C code.
+ * They dynamically align the stack to 16 bytes.
+ * Arguments are moved (not pushed) onto the correctly aligned stack.
+ * NOTE: EDI is destroyed in the process, and hence cannot
+ * be directly used as a parameter. Users of this macro must
+ * independently preserve EDI (a non-volatile) if the routine is
+ * intended to be called from C, for instance.
+ */
+
+#define CCALL(fn)			\
+	movl	%esp, %edi		;\
+	andl	$0xFFFFFFF0, %esp	;\
+	call	EXT(fn)			;\
+	movl	%edi, %esp
+
+#define CCALL1(fn, arg1)		\
+	movl	%esp, %edi		;\
+	subl	$4, %esp		;\
+	andl	$0xFFFFFFF0, %esp	;\
+	movl	arg1, 0(%esp)		;\
+	call	EXT(fn)			;\
+	movl	%edi, %esp
+
+#define CCALL2(fn, arg1, arg2)		\
+	movl	%esp, %edi		;\
+	subl	$8, %esp		;\
+	andl	$0xFFFFFFF0, %esp	;\
+	movl	arg2, 4(%esp)		;\
+	movl	arg1, 0(%esp)		;\
+	call	EXT(fn)			;\
+	movl	%edi, %esp
+
+#define CCALL3(fn, arg1, arg2, arg3)	\
+	movl	%esp, %edi		;\
+	subl	$12, %esp		;\
+	andl	$0xFFFFFFF0, %esp	;\
+	movl	arg3, 8(%esp)		;\
+	movl	arg2, 4(%esp)		;\
+	movl	arg1, 0(%esp)		;\
+	call	EXT(fn)			;\
+	movl	%edi, %esp
+
 	.text
 locore_start:
 
@@ -117,7 +160,6 @@ locore_start:
 
 #ifdef	__MACHO__
 #define	RECOVERY_SECTION	.section	__VECTORS, __recover 
-#define	RETRY_SECTION	.section	__VECTORS, __retries
 #else
 #define	RECOVERY_SECTION	.text
 #define	RECOVERY_SECTION	.text
@@ -143,34 +185,10 @@ LEXT(recover_table_end)			;\
 	.text
 
 /*
- * Retry table for certain successful faults.
- */
-#define	RETRY_TABLE_START	\
-	.align	3;		\
-	.globl	EXT(retry_table) ;\
-LEXT(retry_table)		;\
-	.text
-
-#define	RETRY(addr)		\
-	.align 3		;\
-	.long	9f		;\
-	.long	addr		;\
-	.text			;\
-9:
-
-#define	RETRY_TABLE_END			\
-	.align 3;			\
-	.globl	EXT(retry_table_end)	;\
-LEXT(retry_table_end)			;\
-	.text
-
-/*
- * Allocate recovery and retry tables.
+ * Allocate recovery and table.
  */
 	RECOVERY_SECTION
 	RECOVER_TABLE_START
-	RETRY_SECTION
-	RETRY_TABLE_START
 
 /*
  * Timing routines.
@@ -206,22 +224,30 @@ Entry(timer_grab)
 
 /*
  * Low 32-bits of nanotime returned in %eax.
- * Computed from tsc using conversion scale/shift from per-cpu data.
- * Uses %ecx and %edx.
- */
-#define NANOTIME32							      \
-	pushl	%esi				/* save %esi              */ ;\
-	movl	%gs:CPU_THIS,%esi		/* per-cpu data ptr       */ ;\
-	addl	$(CPU_RTC_NANOTIME),%esi	/* esi -> per-cpu nanotime*/ ;\
-	rdtsc					/* edx:eax = tsc          */ ;\
-	subl	RTN_TSC(%esi),%eax		/* eax = (tsc - base_tsc) */ ;\
-	mull	RTN_SCALE(%esi)			/* eax *= scale           */ ;\
-	movl	RTN_SHIFT(%esi),%ecx		/* ecx = shift            */ ;\
-	shrdl	%cl,%edx,%eax			/* edx:eax >> shift       */ ;\
-	andb	$32,%cl				/* shift == 32?           */ ;\
-	cmovnel	%edx,%eax			/* %eax = %edx if so      */ ;\
-	addl	RTN_NANOS(%esi),%eax		/* add base ns            */ ;\
-	popl	%esi
+ * Computed from tsc based on the scale factor
+ * and an implicit 32 bit shift.
+ *
+ * Uses %esi, %edi, %ebx, %ecx and %edx.
+ */
+#define RNT_INFO		_rtc_nanotime_info
+#define NANOTIME32											  \
+0:	movl	RNT_INFO+RNT_TSC_BASE,%esi						 ;\
+	movl	RNT_INFO+RNT_TSC_BASE+4,%edi					 ;\
+	rdtsc													 ;\
+	subl	%esi,%eax					/* tsc - tsc_base */ ;\
+	sbbl	%edi,%edx										 ;\
+	movl	RNT_INFO+RNT_SCALE,%ecx							 ;\
+	movl	%edx,%ebx					/* delta * scale */  ;\
+	mull	%ecx											 ;\
+	movl	%ebx,%eax										 ;\
+	movl	%edx,%ebx										 ;\
+	mull	%ecx											 ;\
+	addl	%ebx,%eax										 ;\
+	addl	RNT_INFO+RNT_NS_BASE,%eax	/* add ns_base */	 ;\
+	cmpl	RNT_INFO+RNT_TSC_BASE,%esi						 ;\
+	jne		0b												 ;\
+	cmpl	RNT_INFO+RNT_TSC_BASE+4,%edi					 ;\
+	jne		0b
 
 /*
  * Add 32-bit ns delta in register dreg to timer pointed to by register treg.
@@ -237,8 +263,6 @@ Entry(timer_grab)
  * Add time delta to old timer and start new.
  */
 #define TIMER_EVENT(old,new)                                                  \
-	pushl	%eax				/* must be invariant	  */ ;\
-	cli					/* block interrupts       */ ;\
 	NANOTIME32				/* eax low bits nanosecs  */ ;\
 	movl	%gs:CPU_PROCESSOR,%ecx		/* get current processor  */ ;\
 	movl 	CURRENT_TIMER(%ecx),%ecx	/* get current timer      */ ;\
@@ -248,25 +272,24 @@ Entry(timer_grab)
 	addl	$(new##_TIMER-old##_TIMER),%ecx	/* point to new timer     */ ;\
 	movl	%edx,TIMER_TSTAMP(%ecx)		/* set timestamp          */ ;\
 	movl	%gs:CPU_PROCESSOR,%edx		/* get current processor  */ ;\
-	movl	%ecx,CURRENT_TIMER(%edx)	/* set current timer      */ ;\
-	sti					/* interrupts on          */ ;\
-	popl	%eax				/* must be invariant	  */
+	movl	%ecx,CURRENT_TIMER(%edx)	/* set current timer      */
+
 
 /*
  * Update time on user trap entry.
- * Uses %ecx,%edx.
+ * Uses %eax,%ecx,%edx,%esi.
  */
 #define	TIME_TRAP_UENTRY	TIMER_EVENT(USER,SYSTEM)
 
 /*
  * update time on user trap exit.
- * Uses %ecx,%edx.
+ * Uses %eax,%ecx,%edx,%esi.
  */
 #define	TIME_TRAP_UEXIT		TIMER_EVENT(SYSTEM,USER)
 
 /*
  * update time on interrupt entry.
- * Uses %eax,%ecx,%edx.
+ * Uses %eax,%ecx,%edx,%esi.
  */
 #define	TIME_INT_ENTRY \
 	NANOTIME32				/* eax low bits nanosecs  */ ;\
@@ -281,7 +304,7 @@ Entry(timer_grab)
 
 /*
  * update time on interrupt exit.
- * Uses %eax, %ecx, %edx.
+ * Uses %eax, %ecx, %edx, %esi.
  */
 #define	TIME_INT_EXIT \
 	NANOTIME32				/* eax low bits nanosecs  */ ;\
@@ -296,75 +319,6 @@ Entry(timer_grab)
 
 #endif /* STAT_TIME */
 
-/*
- * Encapsulate the transfer of exception stack frames between a PCB
- * and a thread stack.  Since the whole point of these is to emulate
- * a call or exception that changes privilege level, both macros
- * assume that there is no user esp or ss stored in the source
- * frame (because there was no change of privilege to generate them).
- */
-
-/*
- * Transfer a stack frame from a thread's user stack to its PCB.
- * We assume the thread and stack addresses have been loaded into
- * registers (our arguments).
- *
- * The macro overwrites edi, esi, ecx and whatever registers hold the
- * thread and stack addresses (which can't be one of the above three).
- * The thread address is overwritten with the address of its saved state
- * (where the frame winds up).
- *
- * Must be called on kernel stack.
- */
-#define FRAME_STACK_TO_PCB(thread, stkp)				;\
-	movl	ACT_PCB(thread),thread	/* get act`s PCB */		;\
-	leal	PCB_ISS(thread),%edi	/* point to PCB`s saved state */;\
-	movl	%edi,thread		/* save for later */		;\
-	movl	stkp,%esi		/* point to start of frame */	;\
-	movl	$ R_UESP,%ecx						;\
-	sarl	$2,%ecx			/* word count for transfer */	;\
-	cld				/* we`re incrementing */	;\
-	rep								;\
-	movsl				/* transfer the frame */	;\
-	addl	$ R_UESP,stkp		/* derive true "user" esp */	;\
-	movl	stkp,R_UESP(thread)	/* store in PCB */		;\
-	movl	$0,%ecx							;\
-	mov	%ss,%cx			/* get current ss */		;\
-	movl	%ecx,R_SS(thread)	/* store in PCB */
-
-/*
- * Transfer a stack frame from a thread's PCB to the stack pointed
- * to by the PCB.  We assume the thread address has been loaded into
- * a register (our argument).
- *
- * The macro overwrites edi, esi, ecx and whatever register holds the
- * thread address (which can't be one of the above three).  The
- * thread address is overwritten with the address of its saved state
- * (where the frame winds up).
- *
- * Must be called on kernel stack.
- */
-#define FRAME_PCB_TO_STACK(thread)					;\
-	movl	ACT_PCB(thread),%esi	/* get act`s PCB */		;\
-	leal	PCB_ISS(%esi),%esi	/* point to PCB`s saved state */;\
-	movl	R_UESP(%esi),%edi	/* point to end of dest frame */;\
-	movl	ACT_MAP(thread),%ecx	/* get act's map */		;\
-	movl	MAP_PMAP(%ecx),%ecx	/* get map's pmap */		;\
-	cmpl	EXT(kernel_pmap), %ecx	/* If kernel loaded task */	;\
-	jz	1f			/* use kernel data segment */	;\
-	movl	$ USER_DS,%ecx		/* else use user data segment */;\
-	mov	%cx,%es							;\
-1:									;\
-	movl	$ R_UESP,%ecx						;\
-	subl	%ecx,%edi		/* derive start of frame */	;\
-	movl	%edi,thread		/* save for later */		;\
-	sarl	$2,%ecx			/* word count for transfer */	;\
-	cld				/* we`re incrementing */	;\
-	rep								;\
-	movsl				/* transfer the frame */	;\
-	mov	%ss,%cx			/* restore kernel segments */	;\
-	mov	%cx,%es							
-
 #undef PDEBUG
 
 #ifdef PDEBUG
@@ -397,7 +351,7 @@ label/**/exit:
 #define CAH(label)
 
 #endif	/* PDEBUG */
-
+	
 #if	MACH_KDB
 /*
  * Last-ditch debug code to handle faults that might result
@@ -458,13 +412,13 @@ Entry(db_task_gen_prot)
  */
 Entry(db_task_start)
 	movl	%esp,%edx
-	subl	$ISS_SIZE,%edx
+	subl	$(ISS32_SIZE),%edx
 	movl	%edx,%esp		/* allocate i386_saved_state on stack */
 	movl	%eax,R_ERR(%esp)
 	movl	%ebx,R_TRAPNO(%esp)
 	pushl	%edx
 	CPU_NUMBER(%edx)
-	movl	CX(EXT(mp_dbtss),%edx),%edx
+	movl	CX(EXT(master_dbtss),%edx),%edx
 	movl	TSS_LINK(%edx),%eax
 	pushl	%eax			/* pass along selector of previous TSS */
 	call	EXT(db_tss_to_frame)
@@ -478,572 +432,149 @@ Entry(db_task_start)
 #endif	/* MACH_KDB */
 
 /*
- * Trap/interrupt entry points.
- *
- * All traps must create the following save area on the PCB "stack":
- *
- *	gs
- *	fs
- *	es
- *	ds
- *	edi
- *	esi
- *	ebp
- *	cr2 if page fault - otherwise unused
- *	ebx
- *	edx
- *	ecx
- *	eax
- *	trap number
- *	error code
- *	eip
- *	cs
- *	eflags
- *	user esp - if from user
- *	user ss  - if from user
- *	es       - if from V86 thread
- *	ds       - if from V86 thread
- *	fs       - if from V86 thread
- *	gs       - if from V86 thread
- *
+ *	Called as a function, makes the current thread
+ *	return from the kernel as if from an exception.
  */
 
-/*
- * General protection or segment-not-present fault.
- * Check for a GP/NP fault in the kernel_return
- * sequence; if there, report it as a GP/NP fault on the user's instruction.
- *
- * esp->     0:	trap code (NP or GP)
- *	     4:	segment number in error
- *	     8	eip
- *	    12	cs
- *	    16	eflags
- *	    20	old registers (trap is from kernel)
- */
-Entry(t_gen_prot)
-	pushl	$(T_GENERAL_PROTECTION)	/* indicate fault type */
-	jmp	trap_check_kernel_exit	/* check for kernel exit sequence */
-
-Entry(t_segnp)
-	pushl	$(T_SEGMENT_NOT_PRESENT)
-					/* indicate fault type */
-
-trap_check_kernel_exit:
-	testl	$(EFL_VM),16(%esp)	/* is trap from V86 mode? */
-	jnz	EXT(alltraps)		/* isn`t kernel trap if so */
-	testl	$3,12(%esp)		/* is trap from kernel mode? */
-	jne	EXT(alltraps)		/* if so: */
-					/* check for the kernel exit sequence */
-	cmpl	$ EXT(kret_iret),8(%esp)	/* on IRET? */
-	je	fault_iret
-	cmpl	$ EXT(kret_popl_ds),8(%esp) /* popping DS? */
-	je	fault_popl_ds
-	cmpl	$ EXT(kret_popl_es),8(%esp) /* popping ES? */
-	je	fault_popl_es
-	cmpl	$ EXT(kret_popl_fs),8(%esp) /* popping FS? */
-	je	fault_popl_fs
-	cmpl	$ EXT(kret_popl_gs),8(%esp) /* popping GS? */
-	je	fault_popl_gs
-take_fault:				/* if none of the above: */
-	jmp	EXT(alltraps)		/* treat as normal trap. */
-
-/*
- * GP/NP fault on IRET: CS or SS is in error.
- * All registers contain the user's values.
- *
- * on SP is
- *  0	trap number
- *  4	errcode
- *  8	eip
- * 12	cs		--> trapno
- * 16	efl		--> errcode
- * 20	user eip
- * 24	user cs
- * 28	user eflags
- * 32	user esp
- * 36	user ss
- */
-fault_iret:
-	movl	%eax,8(%esp)		/* save eax (we don`t need saved eip) */
-	popl	%eax			/* get trap number */
-	movl	%eax,12-4(%esp)		/* put in user trap number */
-	popl	%eax			/* get error code */
-	movl	%eax,16-8(%esp)		/* put in user errcode */
-	popl	%eax			/* restore eax */
-	CAH(fltir)
-	jmp	EXT(alltraps)		/* take fault */
-
-/*
- * Fault restoring a segment register.  The user's registers are still
- * saved on the stack.  The offending segment register has not been
- * popped.
- */
-fault_popl_ds:
-	popl	%eax			/* get trap number */
-	popl	%edx			/* get error code */
-	addl	$12,%esp		/* pop stack to user regs */
-	jmp	push_es			/* (DS on top of stack) */
-fault_popl_es:
-	popl	%eax			/* get trap number */
-	popl	%edx			/* get error code */
-	addl	$12,%esp		/* pop stack to user regs */
-	jmp	push_fs			/* (ES on top of stack) */
-fault_popl_fs:
-	popl	%eax			/* get trap number */
-	popl	%edx			/* get error code */
-	addl	$12,%esp		/* pop stack to user regs */
-	jmp	push_gs			/* (FS on top of stack) */
-fault_popl_gs:
-	popl	%eax			/* get trap number */
-	popl	%edx			/* get error code */
-	addl	$12,%esp		/* pop stack to user regs */
-	jmp	push_segregs		/* (GS on top of stack) */
-
-push_es:
-	pushl	%es			/* restore es, */
-push_fs:
-	pushl	%fs			/* restore fs, */
-push_gs:
-	pushl	%gs			/* restore gs. */
-push_segregs:
-	movl	%eax,R_TRAPNO(%esp)	/* set trap number */
-	movl	%edx,R_ERR(%esp)	/* set error code */
-	CAH(fltpp)
-	jmp	trap_set_segs		/* take trap */
-
-/*
- * Debug trap.  Check for single-stepping across system call into
- * kernel.  If this is the case, taking the debug trap has turned
- * off single-stepping - save the flags register with the trace
- * bit set.
- */
-Entry(t_debug)
-	testl	$(EFL_VM),8(%esp)	/* is trap from V86 mode? */
-	jnz	0f			/* isn`t kernel trap if so */
-	testl	$3,4(%esp)		/* is trap from kernel mode? */
-	jnz	0f			/* if so: */
-	cmpl	$syscall_entry,(%esp)	/* system call entry? */
-	jne	1f			/* if so: */
-					/* flags are sitting where syscall */
-					/* wants them */
-	addl	$8,%esp			/* remove eip/cs */
-	jmp	syscall_entry_2		/* continue system call entry */
-
-1:	cmpl	$trap_unix_addr,(%esp)
-	jne	0f
-	addl	$8,%esp
-	jmp	trap_unix_2
-
-0:	pushl	$0			/* otherwise: */
-	pushl	$(T_DEBUG)		/* handle as normal */
-	jmp	EXT(alltraps)		/* debug fault */
+	.globl	EXT(thread_exception_return)
+	.globl	EXT(thread_bootstrap_return)
+LEXT(thread_exception_return)
+LEXT(thread_bootstrap_return)
+	cli
+	movl	%gs:CPU_KERNEL_STACK,%ecx
+	movl	(%ecx),%esp			/* switch back to PCB stack */
+	jmp	EXT(return_from_trap)
 
-/*
- * Page fault traps save cr2.
- */
-Entry(t_page_fault)
-	pushl	$(T_PAGE_FAULT)		/* mark a page fault trap */
-	pusha				/* save the general registers */
-	movl	%cr2,%eax		/* get the faulting address */
-	movl	%eax,12(%esp)		/* save in esp save slot */
-	jmp	trap_push_segs		/* continue fault */
+Entry(call_continuation)
+	movl	S_ARG0,%eax			/* get continuation */
+	movl	S_ARG1,%edx			/* continuation param */
+	movl	S_ARG2,%ecx			/* wait result */
+	movl	%gs:CPU_KERNEL_STACK,%esp	/* pop the stack */
+	xorl	%ebp,%ebp			/* zero frame pointer */
+	subl	$8,%esp				/* align the stack */
+	pushl	%ecx
+	pushl	%edx
+	call	*%eax				/* call continuation */
+	addl	$16,%esp
+	movl	%gs:CPU_ACTIVE_THREAD,%eax
+	pushl	%eax
+	call	EXT(thread_terminate)
+	
 
-/*
- * All 'exceptions' enter here with:
- *	esp->   trap number
- *		error code
- *		old eip
- *		old cs
- *		old eflags
- *		old esp		if trapped from user
- *		old ss		if trapped from user
+	
+/*******************************************************************************************************
+ *	
+ * All 64 bit task 'exceptions' enter lo_alltraps:
+ *	esp	-> x86_saved_state_t
+ * 
+ * The rest of the state is set up as:	
+ *	cr3	 -> kernel directory
+ *	esp	 -> low based stack
+ *	gs	 -> CPU_DATA_GS
+ *	cs	 -> KERNEL_CS
+ *	ss/ds/es -> KERNEL_DS
  *
- * NB: below use of CPU_NUMBER assumes that macro will use correct
- * segment register for any kernel data accesses.
- */
-Entry(alltraps)
-	pusha				/* save the general registers */
-trap_push_segs:
-	pushl	%ds			/* save the segment registers */
-	pushl	%es
-	pushl	%fs
-	pushl	%gs
-
-trap_set_segs:
-	movl	%ss,%eax
-	movl	%eax,%ds
-	movl	%eax,%es		/* switch to kernel data seg */
-	cld				/* clear direction flag */
-	testl	$(EFL_VM),R_EFLAGS(%esp) /* in V86 mode? */
-	jnz	trap_from_user		/* user mode trap if so */
-	testb	$3,R_CS(%esp)		/* user mode trap? */
-	jnz	trap_from_user
-	cmpl	$0,%gs:CPU_ACTIVE_KLOADED
-	je	trap_from_kernel	/* if clear, truly in kernel */
-#ifdef FIXME
-	cmpl	ETEXT_ADDR,R_EIP(%esp)	/* pc within kernel? */
-	jb	trap_from_kernel
-#endif
-trap_from_kloaded:
-	/*
-	 * We didn't enter here "through" PCB (i.e., using ring 0 stack),
-	 * so transfer the stack frame into the PCB explicitly, then
-	 * start running on resulting "PCB stack".  We have to set
-	 * up a simulated "uesp" manually, since there's none in the
-	 * frame.
-	 */
-	mov	$ CPU_DATA_GS,%dx
-	mov	%dx,%gs
-	CAH(atstart)
-	movl	%gs:CPU_ACTIVE_KLOADED,%ebx
-	movl	%gs:CPU_KERNEL_STACK,%eax
-	xchgl	%esp,%eax
-	FRAME_STACK_TO_PCB(%ebx,%eax)
-	CAH(atend)
-	jmp	EXT(take_trap)
-
-trap_from_user:
-	mov	$ CPU_DATA_GS,%ax
-	mov	%ax,%gs
-
+ *	interrupts disabled
+ *	direction flag cleared
+ */
+Entry(lo_alltraps)
+	movl	R_CS(%esp),%eax		/* assume 32-bit state */
+	cmpl	$(SS_64),SS_FLAVOR(%esp)/* 64-bit? */	
+	jne	1f
+	movl	R64_CS(%esp),%eax	/* 64-bit user mode */
+1:
+	testb	$3,%eax
+	jz	trap_from_kernel
+						/* user mode trap */
 	TIME_TRAP_UENTRY
 
 	movl	%gs:CPU_KERNEL_STACK,%ebx
-	xchgl	%ebx,%esp		/* switch to kernel stack */
-					/* user regs pointer already set */
-LEXT(take_trap)
-	pushl	%ebx			/* record register save area */
-	pushl	%ebx			/* pass register save area to trap */
-	call	EXT(user_trap)		/* call user trap routine */
-	movl	4(%esp),%esp		/* switch back to PCB stack */
+	xchgl	%ebx,%esp			/* switch to kernel stack */
+	sti
+
+	CCALL1(user_trap, %ebx)		/* call user trap routine */
+	cli				/* hold off intrs - critical section */
+	popl	%esp			/* switch back to PCB stack */
 
 /*
  * Return from trap or system call, checking for ASTs.
- * On PCB stack.
- */
-
+ * On lowbase PCB stack with intrs disabled
+ */	
 LEXT(return_from_trap)
-	movl	%gs:CPU_PENDING_AST,%edx
-	cmpl	$0,%edx
-	je	EXT(return_to_user)	/* if we need an AST: */
-
-	movl	%gs:CPU_KERNEL_STACK,%esp
-					/* switch to kernel stack */
-	pushl	$0			/* push preemption flag */
-	call	EXT(i386_astintr)	/* take the AST */
-	addl	$4,%esp			/* pop preemption flag */
+	movl	%gs:CPU_PENDING_AST,%eax
+	testl	%eax,%eax
+	je	EXT(return_to_user)	/* branch if no AST */
+
+	movl	%gs:CPU_KERNEL_STACK,%ebx
+	xchgl	%ebx,%esp		/* switch to kernel stack */
+	sti				/* interrupts always enabled on return to user mode */
+
+	pushl	%ebx			/* save PCB stack */
+	CCALL1(i386_astintr, $0)	/* take the AST */
+	cli
 	popl	%esp			/* switch back to PCB stack (w/exc link) */
 	jmp	EXT(return_from_trap)	/* and check again (rare) */
-					/* ASTs after this point will */
-					/* have to wait */
 
-/*
- * Arrange the checks needed for kernel-loaded (or kernel-loading)
- * threads so that branch is taken in kernel-loaded case.
- */
 LEXT(return_to_user)
 	TIME_TRAP_UEXIT
-	cmpl	$0,%gs:CPU_ACTIVE_KLOADED
-	jnz	EXT(return_xfer_stack)
-	movl	%gs:CPU_ACTIVE_THREAD, %ebx			/* get active thread */
-
-#if	MACH_RT
-#if	MACH_ASSERT
-	cmpl	$0,%gs:CPU_PREEMPTION_LEVEL
-	je	EXT(return_from_kernel)
-	int	$3
-#endif	/* MACH_ASSERT */
-#endif	/* MACH_RT */
-
-/*
- * Return from kernel mode to interrupted thread.
- */
-
-LEXT(return_from_kernel)
-LEXT(kret_popl_gs)
-	popl	%gs			/* restore segment registers */
-LEXT(kret_popl_fs)
-	popl	%fs
-LEXT(kret_popl_es)
-	popl	%es
-LEXT(kret_popl_ds)
-	popl	%ds
-	popa				/* restore general registers */
-	addl	$8,%esp			/* discard trap number and error code */
-
-LEXT(kret_iret)
-	iret				/* return from interrupt */
 
+LEXT(ret_to_user)
+	cmpl	$0, %gs:CPU_IS64BIT
+	je	EXT(lo_ret_to_user)
+	jmp	EXT(lo64_ret_to_user)
 
-LEXT(return_xfer_stack)
-	/*
-	 * If we're on PCB stack in a kernel-loaded task, we have
-	 * to transfer saved state back to thread stack and swap
-	 * stack pointers here, because the hardware's not going
-	 * to do so for us.
-	 */
-	CAH(rxsstart)
-	movl	%gs:CPU_KERNEL_STACK,%esp
-	movl	%gs:CPU_ACTIVE_KLOADED,%eax
-	FRAME_PCB_TO_STACK(%eax)
-	movl	%eax,%esp
-	CAH(rxsend)
-	jmp	EXT(return_from_kernel)
-
-/*
- * Hate to put this here, but setting up a separate swap_func for
- * kernel-loaded threads no longer works, since thread executes
- * "for a while" (i.e., until it reaches glue code) when first
- * created, even if it's nominally suspended.  Hence we can't
- * transfer the PCB when the thread first resumes, because we
- * haven't initialized it yet.
- */
-/*
- * Have to force transfer to new stack "manually".  Use a string
- * move to transfer all of our saved state to the stack pointed
- * to by iss.uesp, then install a pointer to it as our current
- * stack pointer.
- */
-LEXT(return_kernel_loading)
-	movl	%gs:CPU_KERNEL_STACK,%esp
-	movl	%gs:CPU_ACTIVE_THREAD, %ebx			/* get active thread */
-	movl	%ebx,%edx			/* save for later */
-	FRAME_PCB_TO_STACK(%ebx)
-	movl	%ebx,%esp			/* start running on new stack */
-	movl	$0,%gs:CPU_ACTIVE_KLOADED       /* set cached indicator */
-	jmp	EXT(return_from_kernel)
 
+	
 /*
- * Trap from kernel mode.  No need to switch stacks or load segment registers.
+ * Trap from kernel mode.  No need to switch stacks.
+ * Interrupts must be off here - we will set them to state at time of trap
+ * as soon as it's safe for us to do so and not recurse doing preemption
  */
 trap_from_kernel:
-#if	MACH_KDB || MACH_KGDB
-	mov	$ CPU_DATA_GS,%ax
-	mov	%ax,%gs
-	movl	%esp,%ebx		/* save current stack */
-
-	cmpl	EXT(int_stack_high),%esp /* on an interrupt stack? */
-	jb	6f			/* OK if so */
-
-#if     MACH_KGDB 
-        cmpl    $0,EXT(kgdb_active)     /* Unexpected trap in kgdb */
-        je      0f                      /* no */
-
-        pushl   %esp                    /* Already on kgdb stack */
-        cli
-        call    EXT(kgdb_trap)
-        addl    $4,%esp
-        jmp     EXT(return_from_kernel)
-0:                                      /* should kgdb handle this exception? */
-        cmpl $(T_NO_FPU),R_TRAPNO(%esp) /* FPU disabled? */
-        je      2f                      /* yes */
-        cmpl $(T_PAGE_FAULT),R_TRAPNO(%esp)	/* page fault? */
-        je      2f                      /* yes */
-1:
-        cli                             /* disable interrupts */
-        CPU_NUMBER(%edx)                /* get CPU number */
-        movl    CX(EXT(kgdb_stacks),%edx),%ebx
-        xchgl   %ebx,%esp               /* switch to kgdb stack */
-        pushl   %ebx                    /* pass old sp as an arg */
-        call    EXT(kgdb_from_kernel)
-        popl    %esp                    /* switch back to kernel stack */
-        jmp     EXT(return_from_kernel)
-2:
-#endif  /* MACH_KGDB */
-
-#if	MACH_KDB
-	cmpl	$0,EXT(db_active)       /* could trap be from ddb? */
-	je	3f			/* no */
-	CPU_NUMBER(%edx)		/* see if this CPU is in ddb */
-	cmpl	$0,CX(EXT(kdb_active),%edx)
-	je	3f			/* no */
-	pushl	%esp
-	call	EXT(db_trap_from_asm)
-	addl	$0x4,%esp
-	jmp	EXT(return_from_kernel)
-
-3:
-	/*
-	 * Dilemma:  don't want to switch to kernel_stack if trap
-	 * "belongs" to ddb; don't want to switch to db_stack if
-	 * trap "belongs" to kernel.  So have to duplicate here the
-	 * set of trap types that kernel_trap() handles.  Note that
-	 * "unexpected" page faults will not be handled by kernel_trap().
-	 * In this panic-worthy case, we fall into the debugger with
-	 * kernel_stack containing the call chain that led to the
-	 * bogus fault.
-	 */
-	movl	R_TRAPNO(%esp),%edx
-	cmpl	$(T_PAGE_FAULT),%edx
-	je	4f
-	cmpl	$(T_NO_FPU),%edx
-	je	4f
-	cmpl	$(T_FPU_FAULT),%edx
-	je	4f
-	cmpl	$(T_FLOATING_POINT_ERROR),%edx
-	je	4f
-	cmpl	$(T_PREEMPT),%edx
-	jne	7f
-4:
-#endif	/* MACH_KDB */
+	movl	%esp, %eax		/* saved state addr */
+	CCALL1(kernel_trap, %eax)	/* to kernel trap routine */
+	cli
 
-	cmpl	%gs:CPU_KERNEL_STACK,%esp
-					/* if not already on kernel stack, */
-	ja	5f			/*   check some more */
-	cmpl	%gs:CPU_ACTIVE_STACK,%esp
-	ja	6f			/* on kernel stack: no switch */
-5:
-	movl	%gs:CPU_KERNEL_STACK,%esp
-6:
-	pushl	%ebx			/* save old stack */
-	pushl	%ebx			/* pass as parameter */
-	call	EXT(kernel_trap)	/* to kernel trap routine */
-	addl	$4,%esp			/* pop parameter */
-	testl	%eax,%eax
-	jne	8f
-	/*
-	 * If kernel_trap returns false, trap wasn't handled.
-	 */
-7:
-#if	MACH_KDB
-	CPU_NUMBER(%edx)
-	movl	CX(EXT(db_stacks),%edx),%esp
-	pushl	%ebx			/* pass old stack as parameter */
-	call	EXT(db_trap_from_asm)
-#endif	/* MACH_KDB */
-#if	MACH_KGDB
-        cli                             /* disable interrupts */
-        CPU_NUMBER(%edx)                /* get CPU number */
-        movl    CX(EXT(kgdb_stacks),%edx),%esp
-	pushl	%ebx			/* pass old stack as parameter */
-	call	EXT(kgdb_from_kernel)
-#endif	/* MACH_KGDB */
-	addl	$4,%esp			/* pop parameter */
-	testl	%eax,%eax
-	jne	8f
-	/*
-	 * Likewise, if kdb_trap/kgdb_from_kernel returns false, trap
-	 * wasn't handled.
-	 */
-	pushl	%ebx			/* pass old stack as parameter */
-	call	EXT(panic_trap)
-	addl	$4,%esp			/* pop parameter */
-8:
-	movl	%ebx,%esp		/* get old stack (from callee-saves reg) */
-#else	/* MACH_KDB || MACH_KGDB */
-	pushl	%esp			/* pass parameter */
-	call	EXT(kernel_trap)	/* to kernel trap routine */
-	addl	$4,%esp			/* pop parameter */
-#endif	/* MACH_KDB || MACH_KGDB */
-
-#if	MACH_RT
 	movl	%gs:CPU_PENDING_AST,%eax		/* get pending asts */
 	testl	$ AST_URGENT,%eax	/* any urgent preemption? */
-	je	EXT(return_from_kernel)	/* no, nothing to do */
-	cmpl	$ T_PREEMPT,48(%esp)	/* preempt request? */
-	jne	EXT(return_from_kernel)	/* no, nothing to do */
+	je	ret_to_kernel			/* no, nothing to do */
+	cmpl	$ T_PREEMPT,R_TRAPNO(%esp)
+	je	ret_to_kernel			  /* T_PREEMPT handled in kernel_trap() */
+	testl	$ EFL_IF,R_EFLAGS(%esp)			/* interrupts disabled? */
+	je	ret_to_kernel
+	cmpl	$0,%gs:CPU_PREEMPTION_LEVEL		/* preemption disabled? */
+	jne	ret_to_kernel
 	movl	%gs:CPU_KERNEL_STACK,%eax
 	movl	%esp,%ecx
 	xorl	%eax,%ecx
 	andl	$(-KERNEL_STACK_SIZE),%ecx
 	testl	%ecx,%ecx		/* are we on the kernel stack? */
-	jne	EXT(return_from_kernel)	/* no, skip it */
-
-#if	PREEMPT_DEBUG_LOG
-        pushl   28(%esp)		/* stack pointer */
-        pushl   24+4(%esp)		/* frame pointer */
-        pushl   56+8(%esp)		/* stack pointer */
-        pushl   $0f
-        call    EXT(log_thread_action)
-        addl    $16, %esp
-        .data
-0:      String  "trap preempt eip"
-        .text
-#endif	/* PREEMPT_DEBUG_LOG */
-
-	pushl	$1			/* push preemption flag */
-	call	EXT(i386_astintr)	/* take the AST */
-	addl	$4,%esp			/* pop preemption flag */
-#endif	/* MACH_RT */
-
-	jmp	EXT(return_from_kernel)
+	jne	ret_to_kernel		/* no, skip it */
 
-/*
- *	Called as a function, makes the current thread
- *	return from the kernel as if from an exception.
- */
+	CCALL1(i386_astintr, $1)	/* take the AST */
 
-	.globl	EXT(thread_exception_return)
-	.globl	EXT(thread_bootstrap_return)
-LEXT(thread_exception_return)
-LEXT(thread_bootstrap_return)
-	movl	%esp,%ecx			/* get kernel stack */
-	or	$(KERNEL_STACK_SIZE-1),%ecx
-	movl	-3-IKS_SIZE(%ecx),%esp		/* switch back to PCB stack */
-	jmp	EXT(return_from_trap)
+ret_to_kernel:
+	cmpl	$0, %gs:CPU_IS64BIT
+	je	EXT(lo_ret_to_kernel)
+	jmp	EXT(lo64_ret_to_kernel)
 
-Entry(call_continuation)
-	movl	S_ARG0,%eax			/* get continuation */
-	movl	S_ARG1,%edx			/* continuation param */
-	movl	S_ARG2,%ecx			/* wait result */
-	movl	%esp,%ebp			/* get kernel stack */
-	or	$(KERNEL_STACK_SIZE-1),%ebp
-	addl	$(-3-IKS_SIZE),%ebp
-	movl	%ebp,%esp			/* pop the stack */
-	xorl	%ebp,%ebp			/* zero frame pointer */
-	pushl	%ecx
-	pushl	%edx
-	call	*%eax				/* call continuation */
-	addl	$8,%esp
-	movl	%gs:CPU_ACTIVE_THREAD,%eax
-	pushl	%eax
-	call	EXT(thread_terminate)
 
-#if 0
-#define LOG_INTERRUPT(info,msg)			\
-        pushal				;	\
-        pushl	msg			;	\
-        pushl	info			;	\
-        call	EXT(log_thread_action)	;	\
-        add	$8,%esp			;	\
-        popal
-#define CHECK_INTERRUPT_TIME(n)                 \
-	pushal				;	\
-	pushl	$n			;	\
-	call	EXT(check_thread_time)	;	\
-	add	$4,%esp			;	\
-	popal
-#else
-#define LOG_INTERRUPT(info,msg)
-#define CHECK_INTERRUPT_TIME(n)
-#endif
-	 
-.data
-imsg_start:
-	String	"interrupt start"
-imsg_end:
-	String	"interrupt end"
-
-.text
-/*
- * All interrupts enter here.
- * old %eax on stack; interrupt number in %eax.
- */
-Entry(all_intrs)
-	pushl	%ecx			/* save registers */
-	pushl	%edx
-	cld				/* clear direction flag */
-
-	pushl	%ds			/* save segment registers */
-	pushl	%es
-	pushl	%fs
-	pushl	%gs
-	mov	%ss,%dx			/* switch to kernel segments */
-	mov	%dx,%ds
-	mov	%dx,%es
-	mov	$ CPU_DATA_GS,%dx
-	mov	%dx,%gs
 
+/*******************************************************************************************************
+ *
+ * All interrupts on all tasks enter here with:
+ *	esp->	 -> x86_saved_state_t
+ *
+ *	cr3	 -> kernel directory
+ *	esp	 -> low based stack
+ *	gs	 -> CPU_DATA_GS
+ *	cs	 -> KERNEL_CS
+ *	ss/ds/es -> KERNEL_DS
+ *
+ *	interrupts disabled
+ *	direction flag cleared
+ */
+Entry(lo_allintrs)
 	/*
 	 * test whether already on interrupt stack
 	 */
@@ -1053,710 +584,319 @@ Entry(all_intrs)
 	leal	-INTSTACK_SIZE(%ecx),%edx
 	cmpl	%esp,%edx
 	jb	int_from_intstack
-1:
-	movl	%esp,%edx		/* & i386_interrupt_state */
+1:	
 	xchgl	%ecx,%esp		/* switch to interrupt stack */
 
+	movl	%cr0,%eax		/* get cr0 */
+	orl	$(CR0_TS),%eax		/* or in TS bit */
+	movl	%eax,%cr0		/* set cr0 */
+
+	subl	$8, %esp		/* for 16-byte stack alignment */
 	pushl	%ecx			/* save pointer to old stack */
-	pushl	%edx			/* pass &i386_interrupt_state to pe_incoming_interrupt */
-	pushl	%eax			/* push trap number */
+	movl	%ecx,%gs:CPU_INT_STATE	/* save intr state */
 	
 	TIME_INT_ENTRY			/* do timing */
 
-#if	MACH_RT
 	incl	%gs:CPU_PREEMPTION_LEVEL
-#endif	/* MACH_RT */
 	incl	%gs:CPU_INTERRUPT_LEVEL
 
-	call	EXT(PE_incoming_interrupt)		/* call generic interrupt routine */
-	addl	$8,%esp			/* Pop trap number and eip */
+	movl	%gs:CPU_INT_STATE, %eax
+	CCALL1(PE_incoming_interrupt, %eax) /* call generic interrupt routine */
+
+	cli				/* just in case we returned with intrs enabled */
+	xorl	%eax,%eax
+	movl	%eax,%gs:CPU_INT_STATE	/* clear intr state pointer */
 
 	.globl	EXT(return_to_iret)
 LEXT(return_to_iret)			/* (label for kdb_kintr and hardclock) */
 
 	decl	%gs:CPU_INTERRUPT_LEVEL
-
-#if	MACH_RT
 	decl	%gs:CPU_PREEMPTION_LEVEL
-#endif	/* MACH_RT */
 
 	TIME_INT_EXIT			/* do timing */
 
+	movl	%gs:CPU_ACTIVE_THREAD,%eax
+	movl	ACT_PCB(%eax),%eax	/* get act`s PCB */
+	movl	PCB_FPS(%eax),%eax	/* get pcb's ims.ifps */
+	cmpl	$0,%eax			/* Is there a context */
+	je	1f			/* Branch if not */
+	movl	FP_VALID(%eax),%eax	/* Load fp_valid */
+	cmpl	$0,%eax			/* Check if valid */
+	jne	1f			/* Branch if valid */
+	clts				/* Clear TS */
+	jmp	2f
+1:
+	movl	%cr0,%eax		/* get cr0 */
+	orl	$(CR0_TS),%eax		/* or in TS bit */
+	movl	%eax,%cr0		/* set cr0 */
+2:
 	popl	%esp			/* switch back to old stack */
 
-	movl	%gs:CPU_PENDING_AST,%eax
-	testl	%eax,%eax		/* any pending asts? */
-	je	1f			/* no, nothing to do */
-	testl	$(EFL_VM),I_EFL(%esp)	/* if in V86 */
-	jnz	ast_from_interrupt	/* take it */
-	testb	$3,I_CS(%esp)		/* user mode, */
-	jnz	ast_from_interrupt	/* take it */
-#ifdef FIXME
-	cmpl	ETEXT_ADDR,I_EIP(%esp) /* if within kernel-loaded task, */
-	jnb	ast_from_interrupt	/* take it */
-#endif
+	/* Load interrupted code segment into %eax */
+	movl	R_CS(%esp),%eax		/* assume 32-bit state */
+	cmpl	$(SS_64),SS_FLAVOR(%esp)/* 64-bit? */	
+	jne	3f
+	movl	R64_CS(%esp),%eax	/* 64-bit user mode */
+3:
+	testb	$3,%eax			/* user mode, */
+	jnz	ast_from_interrupt_user	/* go handle potential ASTs */
+	/*
+	 * we only want to handle preemption requests if
+	 * the interrupt fell in the kernel context
+	 * and preemption isn't disabled
+	 */
+	movl	%gs:CPU_PENDING_AST,%eax	
+	testl	$ AST_URGENT,%eax		/* any urgent requests? */
+	je	ret_to_kernel			/* no, nothing to do */
+
+	cmpl	$0,%gs:CPU_PREEMPTION_LEVEL	/* preemption disabled? */
+	jne	ret_to_kernel			/* yes, skip it */
 
-#if	MACH_RT
-	cmpl	$0,%gs:CPU_PREEMPTION_LEVEL		/* preemption masked? */
-	jne	1f			/* yes, skip it */
-	testl	$ AST_URGENT,%eax	/* any urgent requests? */
-	je	1f			/* no, skip it */
-	cmpl	$ EXT(locore_end),I_EIP(%esp)	/* are we in locore code? */
-	jb	1f			/* yes, skip it */
 	movl	%gs:CPU_KERNEL_STACK,%eax
 	movl	%esp,%ecx
 	xorl	%eax,%ecx
 	andl	$(-KERNEL_STACK_SIZE),%ecx
-	testl	%ecx,%ecx		/* are we on the kernel stack? */
-	jne	1f			/* no, skip it */
+	testl	%ecx,%ecx			/* are we on the kernel stack? */
+	jne	ret_to_kernel			/* no, skip it */
 
-/*
- * Take an AST from kernel space.  We don't need (and don't want)
- * to do as much as the case where the interrupt came from user
- * space.
- */
-#if	PREEMPT_DEBUG_LOG
-	pushl 	$0
-	pushl 	$0
-	pushl	I_EIP+8(%esp)
-	pushl	$0f
-	call	EXT(log_thread_action)
-	addl	$16, %esp
-	.data
-0:	String  "intr preempt eip"
-	.text
-#endif	/* PREEMPT_DEBUG_LOG */
+	/*
+	 * Take an AST from kernel space.  We don't need (and don't want)
+	 * to do as much as the case where the interrupt came from user
+	 * space.
+	 */
+	CCALL1(i386_astintr, $1)
 
-	sti
-	pushl	$1			/* push preemption flag */
-	call	EXT(i386_astintr)	/* take the AST */
-	addl	$4,%esp			/* pop preemption flag */
-#endif	/* MACH_RT */
+	jmp	ret_to_kernel
 
-1:
-	pop	%gs
-	pop	%fs
-	pop	%es			/* restore segment regs */
-	pop	%ds
-	pop	%edx
-	pop	%ecx
-	pop	%eax
-	iret				/* return to caller */
 
+/*
+ * nested int - simple path, can't preempt etc on way out
+ */
 int_from_intstack:
-#if	MACH_RT
 	incl	%gs:CPU_PREEMPTION_LEVEL
-#endif	/* MACH_RT */
-
 	incl	%gs:CPU_INTERRUPT_LEVEL
 
-	movl	%esp, %edx		/* i386_interrupt_state */
-	pushl	%edx			/* pass &i386_interrupt_state to PE_incoming_interrupt /*
-	
-	pushl	%eax			/* Push trap number */
-
-	call	EXT(PE_incoming_interrupt)
-	addl	$20,%esp			/* pop i386_interrupt_state, gs,fs,es,ds */
-
-LEXT(return_to_iret_i)			/* ( label for kdb_kintr) */
-
-	addl	$4,%esp			/* pop trap number */
+	movl	%esp, %edx		/* i386_saved_state */
+	CCALL1(PE_incoming_interrupt, %edx)
 
 	decl	%gs:CPU_INTERRUPT_LEVEL
-
-#if	MACH_RT
 	decl	%gs:CPU_PREEMPTION_LEVEL
-#endif	/* MACH_RT */
 
-	pop	%edx			/* must have been on kernel segs */
-	pop	%ecx
-	pop	%eax			/* no ASTs */
-	iret
+	jmp	ret_to_kernel
 
 /*
- *	Take an AST from an interrupt.
- *	On PCB stack.
- * sp->	es	-> edx
- *	ds	-> ecx
- *	edx	-> eax
- *	ecx	-> trapno
- *	eax	-> code
- *	eip
- *	cs
- *	efl
- *	esp
- *	ss
- */
-ast_from_interrupt:
-	pop	%gs
-	pop	%fs
-	pop	%es			/* restore all registers ... */
-	pop	%ds
-	popl	%edx
-	popl	%ecx
-	popl	%eax
-	sti				/* Reenable interrupts */
-	pushl	$0			/* zero code */
-	pushl	$0			/* zero trap number */
-	pusha				/* save general registers */
-	push	%ds			/* save segment registers */
-	push	%es
-	push	%fs
-	push	%gs
-	mov	%ss,%dx			/* switch to kernel segments */
-	mov	%dx,%ds
-	mov	%dx,%es
-	mov	$ CPU_DATA_GS,%dx
-	mov	%dx,%gs
-
-	/*
-	 * See if we interrupted a kernel-loaded thread executing
-	 * in its own task.
-	 */
-	CPU_NUMBER(%edx)
-	testl	$(EFL_VM),R_EFLAGS(%esp) /* in V86 mode? */
-	jnz	0f			/* user mode trap if so */
-	testb	$3,R_CS(%esp)		
-	jnz	0f			/* user mode, back to normal */
-#ifdef FIXME
-	cmpl	ETEXT_ADDR,R_EIP(%esp)
-	jb	0f			/* not kernel-loaded, back to normal */
-#endif
+ *	Take an AST from an interrupted user
+ */
+ast_from_interrupt_user:
+	movl	%gs:CPU_PENDING_AST,%eax
+	testl	%eax,%eax		/* pending ASTs? */
+	je	EXT(ret_to_user)	/* no, nothing to do */
 
-	/*
-	 * Transfer the current stack frame by hand into the PCB.
-	 */
-	CAH(afistart)
-	movl	%gs:CPU_ACTIVE_KLOADED,%eax
-	movl	%gs:CPU_KERNEL_STACK,%ebx
-	xchgl	%ebx,%esp
-	FRAME_STACK_TO_PCB(%eax,%ebx)
-	CAH(afiend)
-	TIME_TRAP_UENTRY
-	jmp	3f
-0:
 	TIME_TRAP_UENTRY
 
-	movl	%gs:CPU_KERNEL_STACK,%eax
-					/* switch to kernel stack */
-	xchgl	%eax,%esp
-3:
-	pushl	%eax
-	pushl	$0			/* push preemption flag */
-	call	EXT(i386_astintr)	/* take the AST */
-	addl	$4,%esp			/* pop preemption flag */
-	popl	%esp			/* back to PCB stack */
 	jmp	EXT(return_from_trap)	/* return */
 
-#if	MACH_KDB || MACH_KGDB 
-/*
- * kdb_kintr:	enter kdb from keyboard interrupt.
- * Chase down the stack frames until we find one whose return
- * address is the interrupt handler.   At that point, we have:
- *
- * frame->	saved %ebp
- *		return address in interrupt handler
- *		ivect
- *		saved SPL
- *		return address == return_to_iret_i
- *		saved %edx
- *		saved %ecx
- *		saved %eax
- *		saved %eip
- *		saved %cs
- *		saved %efl
+
+/*******************************************************************************************************
  *
- * OR:
- * frame->	saved %ebp
- *		return address in interrupt handler
- *		ivect
- *		saved SPL
- *		return address == return_to_iret
- *		pointer to save area on old stack
- *	      [ saved %ebx, if accurate timing ]
+ * 32bit Tasks
+ * System call entries via INTR_GATE or sysenter:
  *
- * old stack:	saved %es
- *		saved %ds
- *		saved %edx
- *		saved %ecx
- *		saved %eax
- *		saved %eip
- *		saved %cs
- *		saved %efl
+ *	esp	 -> i386_saved_state_t
+ *	cr3	 -> kernel directory
+ *	esp	 -> low based stack
+ *	gs	 -> CPU_DATA_GS
+ *	cs	 -> KERNEL_CS
+ *	ss/ds/es -> KERNEL_DS
  *
- * Call kdb, passing it that register save area.
- */
-
-#if MACH_KGDB
-Entry(kgdb_kintr)
-#endif /* MACH_KGDB */
-#if MACH_KDB
-Entry(kdb_kintr)
-#endif /* MACH_KDB */
-	movl	%ebp,%eax		/* save caller`s frame pointer */
-	movl	$ EXT(return_to_iret),%ecx /* interrupt return address 1 */
-	movl	$ EXT(return_to_iret_i),%edx /* interrupt return address 2 */
-
-0:	cmpl	16(%eax),%ecx		/* does this frame return to */
-					/* interrupt handler (1)? */
-	je	1f
-	cmpl	$kdb_from_iret,16(%eax)
-	je	1f
-	cmpl	16(%eax),%edx		/* interrupt handler (2)? */
-	je	2f			/* if not: */
-	cmpl	$kdb_from_iret_i,16(%eax)
-	je	2f
-	movl	(%eax),%eax		/* try next frame */
-	jmp	0b
-
-1:	movl	$kdb_from_iret,16(%eax)	/* returns to kernel/user stack */
-	ret
-
-2:	movl	$kdb_from_iret_i,16(%eax)
-					/* returns to interrupt stack */
-	ret
-
-/*
- * On return from keyboard interrupt, we will execute
- * kdb_from_iret_i
- *	if returning to an interrupt on the interrupt stack
- * kdb_from_iret
- *	if returning to an interrupt on the user or kernel stack
- */
-kdb_from_iret:
-					/* save regs in known locations */
-	pushl	%ebx			/* caller`s %ebx is in reg */
-	pushl	%ebp
-	pushl	%esi
-	pushl	%edi
-	push	%fs
-	push	%gs
-#if MACH_KGDB
-	cli
-        pushl   %esp                    /* pass regs */
-        call    EXT(kgdb_kentry)        /* to kgdb */
-        addl    $4,%esp                 /* pop parameters */
-#endif /* MACH_KGDB */
-#if MACH_KDB
-	pushl	%esp			/* pass regs */
-        call    EXT(kdb_kentry)		/* to kdb */
-	addl	$4,%esp			/* pop parameters */
-#endif /* MACH_KDB */
-	pop	%gs			/* restore registers */
-	pop	%fs
-	popl	%edi
-	popl	%esi
-	popl	%ebp
-	popl	%ebx
-	jmp	EXT(return_to_iret)	/* normal interrupt return */
-
-kdb_from_iret_i:			/* on interrupt stack */
-	pop	%edx			/* restore saved registers */
-	pop	%ecx
-	pop	%eax
-	pushl	$0			/* zero error code */
-	pushl	$0			/* zero trap number */
-	pusha				/* save general registers */
-	push	%ds			/* save segment registers */
-	push	%es
-	push	%fs
-	push	%gs
-#if MACH_KGDB
-	cli				/* disable interrupts */
-        CPU_NUMBER(%edx)                /* get CPU number */
-        movl    CX(EXT(kgdb_stacks),%edx),%ebx
-        xchgl   %ebx,%esp               /* switch to kgdb stack */
-        pushl   %ebx                    /* pass old sp as an arg */
-        call    EXT(kgdb_from_kernel)
-        popl    %esp                    /* switch back to interrupt stack */
-#endif /* MACH_KGDB */
-#if MACH_KDB
-        pushl   %esp                    /* pass regs, */
-        pushl   $0                      /* code, */
-        pushl   $-1                     /* type to kdb */
-        call    EXT(kdb_trap)
-        addl    $12,%esp
-#endif /* MACH_KDB */
-	pop	%gs			/* restore segment registers */
-	pop	%fs
-	pop	%es
-	pop	%ds
-	popa				/* restore general registers */
-	addl	$8,%esp
-	iret
-
-#endif	/* MACH_KDB || MACH_KGDB */
-
-
-/*
- * Mach RPC enters through a call gate, like a system call.
- */
-
-Entry(mach_rpc)
-	pushf				/* save flags as soon as possible */
-	pushl	%eax			/* save system call number */
-	pushl	$0			/* clear trap number slot */
-
-	pusha				/* save the general registers */
-	pushl	%ds			/* and the segment registers */
-	pushl	%es
-	pushl	%fs
-	pushl	%gs
-
-	mov	%ss,%dx			/* switch to kernel data segment */
-	mov	%dx,%ds
-	mov	%dx,%es
-	mov	$ CPU_DATA_GS,%dx
-	mov	%dx,%gs
-
-/*
- * Shuffle eflags,eip,cs into proper places
+ *	interrupts disabled
+ *	direction flag cleared
  */
 
-	movl	R_EIP(%esp),%ebx	/* eflags are in EIP slot */
-	movl	R_CS(%esp),%ecx		/* eip is in CS slot */
-	movl	R_EFLAGS(%esp),%edx	/* cs is in EFLAGS slot */
-	movl	%ecx,R_EIP(%esp)	/* fix eip */
-	movl	%edx,R_CS(%esp)		/* fix cs */
-	movl	%ebx,R_EFLAGS(%esp)	/* fix eflags */
-
+Entry(lo_sysenter)
+	/*
+	 * We can be here either for a mach syscall or a unix syscall,
+	 * as indicated by the sign of the code:
+	 */
+	movl	R_EAX(%esp),%eax
+	testl	%eax,%eax
+	js	EXT(lo_mach_scall)		/* < 0 => mach */
+						/* > 0 => unix */
+	
+Entry(lo_unix_scall)
         TIME_TRAP_UENTRY
 
-        negl    %eax                    /* get system call number */
-        shll    $4,%eax                 /* manual indexing */
-
-/*
- * Check here for mach_rpc from kernel-loaded task --
- *  - Note that kernel-loaded task returns via real return.
- * We didn't enter here "through" PCB (i.e., using ring 0 stack),
- * so transfer the stack frame into the PCB explicitly, then
- * start running on resulting "PCB stack".  We have to set
- * up a simulated "uesp" manually, since there's none in the
- * frame.
- */
-        cmpl    $0,%gs:CPU_ACTIVE_KLOADED
-        jz      2f
-        CAH(mrstart)
-        movl    %gs:CPU_ACTIVE_KLOADED,%ebx
-        movl    %gs:CPU_KERNEL_STACK,%edx
-        xchgl   %edx,%esp
-
-        FRAME_STACK_TO_PCB(%ebx,%edx)
-        CAH(mrend)
-
-        jmp     3f
-
-2:
 	movl	%gs:CPU_KERNEL_STACK,%ebx
-					/* get current kernel stack */
-	xchgl	%ebx,%esp		/* switch stacks - %ebx points to */
-					/* user registers. */
-
-3:
-
-/*
- * Register use on entry:
- *   eax contains syscall number
- *   ebx contains user regs pointer
- */
-#undef 	RPC_TRAP_REGISTERS
-#ifdef	RPC_TRAP_REGISTERS
-	pushl	R_ESI(%ebx)
-	pushl	R_EDI(%ebx)
-	pushl	R_ECX(%ebx)
-	pushl	R_EDX(%ebx)
-#else
-	movl	EXT(mach_trap_table)(%eax),%ecx
-					/* get number of arguments */
-	jecxz	2f			/* skip argument copy if none */
-	movl	R_UESP(%ebx),%esi	/* get user stack pointer */
-	lea	4(%esi,%ecx,4),%esi	/* skip user return address, */
-					/* and point past last argument */
-	movl	%gs:CPU_ACTIVE_KLOADED,%edx
-					/* point to current thread */
-	orl	%edx,%edx		/* if ! kernel-loaded, check addr */
-	jz	4f			/* else */
-	mov	%ds,%dx			/* kernel data segment access */
-	jmp	5f
-4:
- 	cmpl	$(VM_MAX_ADDRESS),%esi	/* in user space? */
- 	ja	mach_call_addr		/* address error if not */
-	movl	$ USER_DS,%edx		/* user data segment access */
-5:
-	mov	%dx,%fs
-	movl	%esp,%edx		/* save kernel ESP for error recovery */
-1:
-	subl	$4,%esi
-	RECOVERY_SECTION
-	RECOVER(mach_call_addr_push)
-	pushl	%fs:(%esi)		/* push argument on stack */
-	loop	1b			/* loop for all arguments */
-#endif
+	xchgl	%ebx,%esp		/* switch to kernel stack */
 
-/*
- * Register use on entry:
- *   eax contains syscall number << 4
- * mach_call_munger is declared regparm(1), so the first arg is %eax
- */
-2:
+	sti
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
+	movl	ACT_TASK(%ecx),%ecx		/* point to current task  */
+	addl	$1,TASK_SYSCALLS_UNIX(%ecx)	/* increment call count   */
 
-	call	EXT(mach_call_munger)
+	CCALL1(unix_syscall, %ebx)
+	/*
+	 * always returns through thread_exception_return
+	 */
 	
-	movl	%esp,%ecx		/* get kernel stack */
-	or	$(KERNEL_STACK_SIZE-1),%ecx
-	movl	-3-IKS_SIZE(%ecx),%esp	/* switch back to PCB stack */
-	movl	%eax,R_EAX(%esp)	/* save return value */
-	jmp	EXT(return_from_trap)	/* return to user */
-
 
-/*
- * Special system call entry for "int 0x80", which has the "eflags"
- * register saved at the right place already.
- * Fall back to the common syscall path after saving the registers.
- *
- * esp ->	old eip
- *		old cs
- * 		old eflags
- *		old esp		if trapped from user
- * 		old ss		if trapped from user
- *
- * XXX:	for the moment, we don't check for int 0x80 from kernel mode.
- */
-Entry(syscall_int80)
-	pushl	%eax			/* save system call number */
-	pushl	$0			/* clear trap number slot */
+Entry(lo_mach_scall)
+	TIME_TRAP_UENTRY
 
-	pusha				/* save the general registers */
-	pushl	%ds			/* and the segment registers */
-	pushl	%es
-	pushl	%fs
-	pushl	%gs
+	movl	%gs:CPU_KERNEL_STACK,%ebx
+	xchgl	%ebx,%esp		/* switch to kernel stack */
 
-	mov	%ss,%dx			/* switch to kernel data segment */
-	mov	%dx,%ds
-	mov	%dx,%es
-	mov	$ CPU_DATA_GS,%dx
-	mov	%dx,%gs
+	sti
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
+	movl	ACT_TASK(%ecx),%ecx		/* point to current task  */
+	addl	$1,TASK_SYSCALLS_MACH(%ecx)	/* increment call count   */
 
-	jmp	syscall_entry_3
+	CCALL1(mach_call_munger, %ebx)
+	/*
+	 * always returns through thread_exception_return
+	 */
 
-/*
- * System call enters through a call gate.  Flags are not saved -
- * we must shuffle stack to look like trap save area.
- *
- * esp->	old eip
- *		old cs
- *		old esp
- *		old ss
- *
- * eax contains system call number.
- *
- * NB: below use of CPU_NUMBER assumes that macro will use correct
- * correct segment register for any kernel data accesses.
- */
-Entry(syscall)
-syscall_entry:
-	pushf				/* save flags as soon as possible */
-syscall_entry_2:
-	pushl	%eax			/* save system call number */
-	pushl	$0			/* clear trap number slot */
-
-	pusha				/* save the general registers */
-	pushl	%ds			/* and the segment registers */
-	pushl	%es
-	pushl	%fs
-	pushl	%gs
-
-	mov	%ss,%dx			/* switch to kernel data segment */
-	mov	%dx,%ds
-	mov	%dx,%es
-	mov	$ CPU_DATA_GS,%dx
-	mov	%dx,%gs
+	
+Entry(lo_mdep_scall)
+        TIME_TRAP_UENTRY
 
-/*
- * Shuffle eflags,eip,cs into proper places
- */
+	movl	%gs:CPU_KERNEL_STACK,%ebx
+	xchgl	%ebx,%esp		/* switch to kernel stack */
 
-	movl	R_EIP(%esp),%ebx	/* eflags are in EIP slot */
-	movl	R_CS(%esp),%ecx		/* eip is in CS slot */
-	movl	R_EFLAGS(%esp),%edx	/* cs is in EFLAGS slot */
-	movl	%ecx,R_EIP(%esp)	/* fix eip */
-	movl	%edx,R_CS(%esp)		/* fix cs */
-	movl	%ebx,R_EFLAGS(%esp)	/* fix eflags */
+	sti
 
-syscall_entry_3:
-/*
- * Check here for syscall from kernel-loaded task --
- * We didn't enter here "through" PCB (i.e., using ring 0 stack),
- * so transfer the stack frame into the PCB explicitly, then
- * start running on resulting "PCB stack".  We have to set
- * up a simulated "uesp" manually, since there's none in the
- * frame.
- */
-	cmpl	$0,%gs:CPU_ACTIVE_KLOADED
-	jz	0f
-	CAH(scstart)
-	movl	%gs:CPU_ACTIVE_KLOADED,%ebx
-	movl	%gs:CPU_KERNEL_STACK,%edx
-	xchgl	%edx,%esp
-	FRAME_STACK_TO_PCB(%ebx,%edx)
-	CAH(scend)
-	TIME_TRAP_UENTRY
-	jmp	1f
+	CCALL1(machdep_syscall, %ebx)
+	/*
+	 * always returns through thread_exception_return
+	 */
+	
 
-0:
+Entry(lo_diag_scall)
 	TIME_TRAP_UENTRY
 
-	movl	%gs:CPU_KERNEL_STACK,%ebx
-					/* get current kernel stack */
-	xchgl	%ebx,%esp		/* switch stacks - %ebx points to */
-					/* user registers. */
-					/* user regs pointer already set */
-
-/*
- * Native system call.
- * Register use on entry:
- *   eax contains syscall number
- *   ebx points to user regs
- */
-1:
-	negl	%eax			/* get system call number */
-	jl	mach_call_range		/* out of range if it was positive */
-
-	cmpl	EXT(mach_trap_count),%eax /* check system call table bounds */
-	jg	mach_call_range		/* error if out of range */
-	shll	$4,%eax			/* manual indexing */
+	movl	%gs:CPU_KERNEL_STACK,%ebx	// Get the address of the kernel stack
+	xchgl	%ebx,%esp		// Switch to it, saving the previous
 
-	movl	EXT(mach_trap_table)+4(%eax),%edx
-					/* get procedure */
-	cmpl	$ EXT(kern_invalid),%edx	/* if not "kern_invalid" */
-	jne	do_native_call		/* go on with Mach syscall */
-	shrl	$4,%eax			/* restore syscall number */
-	jmp	mach_call_range		/* try it as a "server" syscall */
+	CCALL1(diagCall, %ebx)		// Call diagnostics
+	cli				// Disable interruptions just in case they were enabled
+	popl	%esp			// Get back the original stack
+	
+	cmpl	$0,%eax			// What kind of return is this?
+	jne	EXT(return_to_user)	// Normal return, do not check asts...
+				
+	CCALL3(i386_exception, $EXC_SYSCALL, $0x6000, $1)
+		// pass what would be the diag syscall
+		// error return - cause an exception
+	/* no return */
+	
 
-/*
- * Register use on entry:
- *   eax contains syscall number
- *   ebx contains user regs pointer
- */
-do_native_call:
-	movl	EXT(mach_trap_table)(%eax),%ecx
-					/* get number of arguments */
-	jecxz	mach_call_call		/* skip argument copy if none */
-	movl	R_UESP(%ebx),%esi	/* get user stack pointer */
-	lea	4(%esi,%ecx,4),%esi	/* skip user return address, */
-					/* and point past last argument */
-	movl	%gs:CPU_ACTIVE_KLOADED,%edx
-					/* point to current thread */
-	orl	%edx,%edx		/* if kernel-loaded, skip addr check */
-	jz	0f			/* else */
-	mov	%ds,%dx			/* kernel data segment access  */
-	jmp	1f
-0:
-	cmpl	$(VM_MAX_ADDRESS),%esi	/* in user space? */
-	ja	mach_call_addr		/* address error if not */
-	movl	$ USER_DS,%edx		/* user data segment access */
-1:
-	mov	%dx,%fs
-	movl	%esp,%edx		/* save kernel ESP for error recovery */
-2:
-	subl	$4,%esi
-	RECOVERY_SECTION
-	RECOVER(mach_call_addr_push)
-	pushl	%fs:(%esi)		/* push argument on stack */
-	loop	2b			/* loop for all arguments */
 
-/*
- * Register use on entry:
- *   eax contains syscall number
- *   ebx contains user regs pointer
+/*******************************************************************************************************
+ *
+ * 64bit Tasks
+ * System call entries via syscall only:
+ *
+ *	esp	 -> x86_saved_state64_t
+ *	cr3	 -> kernel directory
+ *	esp	 -> low based stack
+ *	gs	 -> CPU_DATA_GS
+ *	cs	 -> KERNEL_CS
+ *	ss/ds/es -> KERNEL_DS
+ *
+ *	interrupts disabled
+ *	direction flag cleared
  */
-mach_call_call:
 
-	CAH(call_call)
+Entry(lo_syscall)
+	/*
+	 * We can be here either for a mach, unix machdep or diag syscall,
+	 * as indicated by the syscall class:
+	 */
+	movl	R64_RAX(%esp), %eax		/* syscall number/class */
+	movl	%eax, %ebx
+	andl	$(SYSCALL_CLASS_MASK), %ebx	/* syscall class */
+	cmpl	$(SYSCALL_CLASS_MACH<<SYSCALL_CLASS_SHIFT), %ebx
+	je	EXT(lo64_mach_scall)
+	cmpl	$(SYSCALL_CLASS_UNIX<<SYSCALL_CLASS_SHIFT), %ebx
+	je	EXT(lo64_unix_scall)
+	cmpl	$(SYSCALL_CLASS_MDEP<<SYSCALL_CLASS_SHIFT), %ebx
+	je	EXT(lo64_mdep_scall)
+	cmpl	$(SYSCALL_CLASS_DIAG<<SYSCALL_CLASS_SHIFT), %ebx
+	je	EXT(lo64_diag_scall)
+
+	/* Syscall class unknown */
+	CCALL3(i386_exception, $(EXC_SYSCALL), %eax, $1)
+	/* no return */
 
-#if	ETAP_EVENT_MONITOR
-	cmpl	$0x200, %eax			/* is this mach_msg? */
-	jz	make_syscall			/* if yes, don't record event */
+Entry(lo64_unix_scall)
+        TIME_TRAP_UENTRY
 
-        pushal					/* Otherwise: save registers */
-        pushl	%eax				/*   push syscall number on stack*/
-        call	EXT(etap_machcall_probe1)	/*   call event begin probe */
-        add	$4,%esp				/*   restore stack */
-        popal					/*   restore registers */
+	movl	%gs:CPU_KERNEL_STACK,%ebx
+	xchgl	%ebx,%esp		/* switch to kernel stack */
 
-	call	*EXT(mach_trap_table)+4(%eax)	/* call procedure */
-        pushal
-        call	EXT(etap_machcall_probe2)	/* call event end probe */
-        popal
-	jmp	skip_syscall			/* syscall already made */
-#endif	/* ETAP_EVENT_MONITOR */
+	sti
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
+	movl	ACT_TASK(%ecx),%ecx		/* point to current task  */
+	addl	$1,TASK_SYSCALLS_UNIX(%ecx)	/* increment call count   */
 
-make_syscall:
+	CCALL1(unix_syscall64, %ebx)
+	/*
+	 * always returns through thread_exception_return
+	 */
+	
 
-/*
- * mach_call_munger is declared regparm(1) so the first arg is %eax
- */
-	call	EXT(mach_call_munger)
+Entry(lo64_mach_scall)
+	TIME_TRAP_UENTRY
 
-skip_syscall:
+	movl	%gs:CPU_KERNEL_STACK,%ebx
+	xchgl	%ebx,%esp		/* switch to kernel stack */
 
-	movl	%esp,%ecx		/* get kernel stack */
-	or	$(KERNEL_STACK_SIZE-1),%ecx
-	movl	-3-IKS_SIZE(%ecx),%esp	/* switch back to PCB stack */
-	movl	%eax,R_EAX(%esp)	/* save return value */
-	jmp	EXT(return_from_trap)	/* return to user */
+	sti
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
+	movl	ACT_TASK(%ecx),%ecx		/* point to current task  */
+	addl	$1,TASK_SYSCALLS_MACH(%ecx)	/* increment call count   */
 
-/*
- * Address out of range.  Change to page fault.
- * %esi holds failing address.
- * Register use on entry:
- *   ebx contains user regs pointer
- */
-mach_call_addr_push:
-	movl	%edx,%esp		/* clean parameters from stack */
-mach_call_addr:
-	movl	%esi,R_CR2(%ebx)	/* set fault address */
-	movl	$(T_PAGE_FAULT),R_TRAPNO(%ebx)
-					/* set page-fault trap */
-	movl	$(T_PF_USER),R_ERR(%ebx)
-					/* set error code - read user space */
-	CAH(call_addr)
-	jmp	EXT(take_trap)		/* treat as a trap */
+	CCALL1(mach_call_munger64, %ebx)
+	/*
+	 * always returns through thread_exception_return
+	 */
 
-/*
- * System call out of range.  Treat as invalid-instruction trap.
- * (? general protection?)
- * Register use on entry:
- *   eax contains syscall number
- */
-mach_call_range:
-	push 	%eax
-	movl	%esp,%edx
-	push	$1			/* code_cnt = 1 */
-	push	%edx			/* exception_type_t (see i/f docky) */
-	push	$ EXC_SYSCALL
-	CAH(call_range)
-	call	EXT(exception_triage)
-	/* no return */
+	
+Entry(lo64_mdep_scall)
+        TIME_TRAP_UENTRY
 
-	.globl	EXT(syscall_failed)
-LEXT(syscall_failed)
-	movl	%esp,%ecx		/* get kernel stack */
-	or	$(KERNEL_STACK_SIZE-1),%ecx
-	movl	-3-IKS_SIZE(%ecx),%esp	/* switch back to PCB stack */
 	movl	%gs:CPU_KERNEL_STACK,%ebx
-					/* get current kernel stack */
-	xchgl	%ebx,%esp		/* switch stacks - %ebx points to */
-					/* user registers. */
-					/* user regs pointer already set */
+	xchgl	%ebx,%esp		/* switch to kernel stack */
+
+	sti
+
+	CCALL1(machdep_syscall64, %ebx)
+	/*
+	 * always returns through thread_exception_return
+	 */
+	
+
+Entry(lo64_diag_scall)
+	TIME_TRAP_UENTRY
 
-	movl	$(T_INVALID_OPCODE),R_TRAPNO(%ebx)
-					/* set invalid-operation trap */
-	movl	$0,R_ERR(%ebx)		/* clear error code */
-	CAH(failed)
-	jmp	EXT(take_trap)		/* treat as a trap */
+	movl	%gs:CPU_KERNEL_STACK,%ebx	// Get the address of the kernel stack
+	xchgl	%ebx,%esp		// Switch to it, saving the previous
+	
+	pushl	%ebx			// Push the previous stack
+	CCALL1(diagCall64, %ebx)	// Call diagnostics
+	cli				// Disable interruptions just in case they were enabled
+	popl	%esp			// Get back the original stack
+	
+	cmpl	$0,%eax			// What kind of return is this?
+	jne	EXT(return_to_user)	// Normal return, do not check asts...
+				
+	CCALL3(i386_exception, $EXC_SYSCALL, $0x6000, $1)
+	/* no return */
+	
 
+			
+/******************************************************************************************************
+			
 /**/
 /*
  * Utility routines.
@@ -1764,32 +904,34 @@ LEXT(syscall_failed)
 
 
 /*
- * Copy from user address space.
- * arg0:	user address
+ * Copy from user/kernel address space.
+ * arg0:	window offset or kernel address
  * arg1:	kernel address
  * arg2:	byte count
  */
-Entry(copyinmsg)
-ENTRY(copyin)
+ENTRY(copyinphys_user)
+	movl	$(USER_WINDOW_SEL),%ecx	/* user data segment access through kernel window */
+	mov	%cx,%ds
+
+ENTRY(copyinphys_kern)
+	movl	$(PHYS_WINDOW_SEL),%ecx	/* physical access through kernel window */
+	mov	%cx,%es
+	jmp	copyin_common
+
+ENTRY(copyin_user)
+	movl	$(USER_WINDOW_SEL),%ecx	/* user data segment access through kernel window */
+	mov	%cx,%ds
+
+ENTRY(copyin_kern)
+
+copyin_common:
 	pushl	%esi
 	pushl	%edi			/* save registers */
 
-	movl	8+S_ARG0,%esi		/* get user start address */
-	movl	8+S_ARG1,%edi		/* get kernel destination address */
+	movl	8+S_ARG0,%esi		/* get source - window offset or kernel address */
+	movl	8+S_ARG1,%edi		/* get destination - kernel address */
 	movl	8+S_ARG2,%edx		/* get count */
 
-	lea	0(%esi,%edx),%eax	/* get user end address + 1 */
-
-	movl	%gs:CPU_ACTIVE_THREAD,%ecx			/* get active thread */
-	movl	ACT_MAP(%ecx),%ecx		/* get act->map */
-	movl	MAP_PMAP(%ecx),%ecx		/* get map->pmap */
-	cmpl	EXT(kernel_pmap), %ecx
-	jz	1f
-	movl	$ USER_DS,%ecx		/* user data segment access */
-	mov	%cx,%ds
-1:
-	cmpl	%esi,%eax
-	jb	copyin_fail		/* fail if wrap-around */
 	cld				/* count up */
 	movl	%edx,%ecx		/* move by longwords first */
 	shrl	$2,%ecx
@@ -1804,49 +946,47 @@ ENTRY(copyin)
 	rep
 	movsb
 	xorl	%eax,%eax		/* return 0 for success */
-copy_ret:
-	mov	%ss,%di			/* restore kernel data segment */
-	mov	%di,%ds
+copyin_ret:
+	mov	%ss,%cx			/* restore kernel data and extended segments */
+	mov	%cx,%ds
+	mov	%cx,%es
 
 	popl	%edi			/* restore registers */
 	popl	%esi
 	ret				/* and return */
 
 copyin_fail:
-	movl	$ EFAULT,%eax			/* return error for failure */
-	jmp	copy_ret		/* pop frame and return */
+	movl	$(EFAULT),%eax		/* return error for failure */
+	jmp	copyin_ret		/* pop frame and return */
+
 
+	
 /*
- * Copy string from user address space.
- * arg0:	user address
+ * Copy string from user/kern address space.
+ * arg0:	window offset or kernel address
  * arg1:	kernel address
  * arg2:	max byte count
  * arg3:	actual byte count (OUT)
  */
-Entry(copyinstr)
+Entry(copyinstr_kern)
+	mov	%ds,%cx
+	jmp	copyinstr_common	
+
+Entry(copyinstr_user)
+	movl	$(USER_WINDOW_SEL),%ecx	/* user data segment access through kernel window */
+
+copyinstr_common:
+	mov	%cx,%fs
+
 	pushl	%esi
 	pushl	%edi			/* save registers */
 
-	movl	8+S_ARG0,%esi		/* get user start address */
-	movl	8+S_ARG1,%edi		/* get kernel destination address */
+	movl	8+S_ARG0,%esi		/* get source - window offset or kernel address */
+	movl	8+S_ARG1,%edi		/* get destination - kernel address */
 	movl	8+S_ARG2,%edx		/* get count */
 
-	lea	0(%esi,%edx),%eax	/* get user end address + 1 */
-
-	movl	%gs:CPU_ACTIVE_THREAD,%ecx			/* get active thread */
-	movl	ACT_MAP(%ecx),%ecx		/* get act->map */
-	movl	MAP_PMAP(%ecx),%ecx		/* get map->pmap */
-	cmpl	EXT(kernel_pmap), %ecx
-	jne	0f
-	mov	%ds,%cx			/* kernel data segment access  */
-	jmp	1f
-0:
-	movl	$ USER_DS,%ecx		/* user data segment access */
-1:
-	mov	%cx,%fs
-	xorl	%eax,%eax
-	cmpl	$0,%edx
-	je	4f
+	xorl	%eax,%eax		/* set to 0 here so that the high 24 bits */
+					/* are 0 for the cmpl against 0 */
 2:
 	RECOVERY_SECTION
 	RECOVER(copystr_fail)		/* copy bytes... */
@@ -1857,13 +997,11 @@ Entry(copyinstr)
 	movb	%al,(%edi)		/* copy the byte */
 	incl	%edi
 3:
-	decl	%edx
-	je	5f			/* Zero count.. error out */
-	cmpl	$0,%eax
-	jne	2b			/* .. a NUL found? */
-	jmp	4f			/* return zero (%eax) */
-5:
-	movl	$ ENAMETOOLONG,%eax	/* String is too long.. */
+	testl	%eax,%eax		/* did we just stuff the 0-byte? */
+	jz	4f			/* yes, return 0 status already in %eax */
+	decl	%edx			/* decrement #bytes left in buffer */
+	jnz	2b			/* buffer not full so copy in another byte */
+	movl	$(ENAMETOOLONG),%eax	/* buffer full but no 0-byte: ENAMETOOLONG */
 4:
 	movl	8+S_ARG3,%edi		/* get OUT len ptr */
 	cmpl	$0,%edi
@@ -1876,271 +1014,66 @@ copystr_ret:
 	ret				/* and return */
 
 copystr_fail:
-	movl	$ EFAULT,%eax		/* return error for failure */
-	jmp	copy_ret		/* pop frame and return */
+	movl	$(EFAULT),%eax		/* return error for failure */
+	jmp	copystr_ret		/* pop frame and return */
+
 
 /*
- * Copy to user address space.
+ * Copy to user/kern address space.
  * arg0:	kernel address
- * arg1:	user address
+ * arg1:	window offset or kernel address
  * arg2:	byte count
  */
-Entry(copyoutmsg)
-ENTRY(copyout)
-	pushl	%esi
-	pushl	%edi			/* save registers */
-	pushl	%ebx
-
-	movl	12+S_ARG0,%esi		/* get kernel start address */
-	movl	12+S_ARG1,%edi		/* get user start address */
-	movl	12+S_ARG2,%edx		/* get count */
+ENTRY(copyoutphys_user)
+	movl	$(USER_WINDOW_SEL),%ecx	/* user data segment access through kernel window */
+	mov	%cx,%es
 
-	leal	0(%edi,%edx),%eax	/* get user end address + 1 */
+ENTRY(copyoutphys_kern)
+	movl	$(PHYS_WINDOW_SEL),%ecx	/* physical access through kernel window */
+	mov	%cx,%ds
+	jmp	copyout_common
 
-	movl	%gs:CPU_ACTIVE_THREAD,%ecx			/* get active thread */
-	movl	ACT_MAP(%ecx),%ecx		/* get act->map */
-	movl	MAP_PMAP(%ecx),%ecx		/* get map->pmap */
-	cmpl	EXT(kernel_pmap), %ecx
-	jne	0f
-	mov	%ds,%cx			/* else kernel data segment access  */
-	jmp	1f
-0:
-	movl	$ USER_DS,%ecx
-1:
+ENTRY(copyout_user)
+	movl	$(USER_WINDOW_SEL),%ecx	/* user data segment access through kernel window */
 	mov	%cx,%es
 
-/*
- * Check whether user address space is writable
- * before writing to it - hardware is broken.
- *
- * Skip check if "user" address is really in
- * kernel space (i.e., if it's in a kernel-loaded
- * task).
- *
- * Register usage:
- *	esi/edi	source/dest pointers for rep/mov
- *	ecx	counter for rep/mov
- *	edx	counts down from 3rd arg
- *	eax	count of bytes for each (partial) page copy
- *	ebx	shadows edi, used to adjust edx
- */
-	movl	%edi,%ebx		/* copy edi for syncing up */
-copyout_retry:
-	/* if restarting after a partial copy, put edx back in sync, */
-	addl	%ebx,%edx		/* edx -= (edi - ebx); */
-	subl	%edi,%edx
-	movl	%edi,%ebx		/* ebx = edi; */
+ENTRY(copyout_kern)
+
+copyout_common:
+	pushl	%esi
+	pushl	%edi			/* save registers */
+
+	movl	8+S_ARG0,%esi		/* get source - kernel address */
+	movl	8+S_ARG1,%edi		/* get destination - window offset or kernel address */
+	movl	8+S_ARG2,%edx		/* get count */
 
-/*
- * Copy only what fits on the current destination page.
- * Check for write-fault again on the next page.
- */
-	leal	NBPG(%edi),%eax		/* point to */
-	andl	$(-NBPG),%eax		/* start of next page */
-	subl	%edi,%eax		/* get number of bytes to that point */
-	cmpl	%edx,%eax		/* bigger than count? */
-	jle	1f			/* if so, */
-	movl	%edx,%eax		/* use count */
-1:
 	cld				/* count up */
-	movl	%eax,%ecx		/* move by longwords first */
+	movl	%edx,%ecx		/* move by longwords first */
 	shrl	$2,%ecx
 	RECOVERY_SECTION
 	RECOVER(copyout_fail)
-	RETRY_SECTION
-	RETRY(copyout_retry)
 	rep
 	movsl
-	movl	%eax,%ecx		/* now move remaining bytes */
+	movl	%edx,%ecx		/* now move remaining bytes */
 	andl	$3,%ecx
 	RECOVERY_SECTION
 	RECOVER(copyout_fail)
-	RETRY_SECTION
-	RETRY(copyout_retry)
 	rep
 	movsb				/* move */
-	movl	%edi,%ebx		/* copy edi for syncing up */
-	subl	%eax,%edx		/* and decrement count */
-	jg	copyout_retry		/* restart on next page if not done */
 	xorl	%eax,%eax		/* return 0 for success */
 copyout_ret:
-	mov	%ss,%di			/* restore kernel segment */
-	mov	%di,%es
+	mov	%ss,%cx			/* restore kernel segment */
+	mov	%cx,%es
+	mov	%cx,%ds
 
-	popl	%ebx
 	popl	%edi			/* restore registers */
 	popl	%esi
 	ret				/* and return */
 
 copyout_fail:
-	movl	$ EFAULT,%eax		/* return error for failure */
+	movl	$(EFAULT),%eax		/* return error for failure */
 	jmp	copyout_ret		/* pop frame and return */
 
-/*
- * FPU routines.
- */
-
-/*
- * Initialize FPU.
- */
-ENTRY(_fninit)
-	fninit
-	ret
-
-/*
- * Read control word
- */
-ENTRY(_fstcw)
-	pushl	%eax		/* get stack space */
-	fstcw	(%esp)
-	popl	%eax
-	ret
-
-/*
- * Set control word
- */
-ENTRY(_fldcw)
-	fldcw	4(%esp)
-	ret
-
-/*
- * Read status word
- */
-ENTRY(_fnstsw)
-	xor	%eax,%eax		/* clear high 16 bits of eax */
-	fnstsw	%ax			/* read FP status */
-	ret
-
-/*
- * Clear FPU exceptions
- */
-ENTRY(_fnclex)
-	fnclex
-	ret
-
-/*
- * Clear task-switched flag.
- */
-ENTRY(_clts)
-	clts
-	ret
-
-/*
- * Save complete FPU state.  Save error for later.
- */
-ENTRY(_fpsave)
-	movl	4(%esp),%eax		/* get save area pointer */
-	fnsave	(%eax)			/* save complete state, including */
-					/* errors */
-	ret
-
-/*
- * Restore FPU state.
- */
-ENTRY(_fprestore)
-	movl	4(%esp),%eax		/* get save area pointer */
-	frstor	(%eax)			/* restore complete state */
-	ret
-
-/*
- * Set cr3
- */
-ENTRY(set_cr3)
-	CPU_NUMBER(%eax)
-	orl	4(%esp), %eax
-	/*
-	 * Don't set PDBR to a new value (hence invalidating the
-	 * "paging cache") if the new value matches the current one.
-	 */
-	movl	%cr3,%edx		/* get current cr3 value */
-	cmpl	%eax,%edx
-	je	0f			/* if two are equal, don't set */
-	movl	%eax,%cr3		/* load it (and flush cache) */
-0:
-	ret
-
-/*
- * Read cr3
- */
-ENTRY(get_cr3)
-	movl	%cr3,%eax
-	andl	$(~0x7), %eax		/* remove cpu number */
-	ret
-
-/*
- * Flush TLB
- */
-ENTRY(flush_tlb)
-	movl	%cr3,%eax		/* flush tlb by reloading CR3 */
-	movl	%eax,%cr3		/* with itself */
-	ret
-
-/*
- * Read cr2
- */
-ENTRY(get_cr2)
-	movl	%cr2,%eax
-	ret
-
-/*
- * Read cr4
- */
-ENTRY(get_cr4)
-	.byte	0x0f,0x20,0xe0		/* movl	%cr4, %eax */
-	ret
-
-/*
- * Write cr4
- */
-ENTRY(set_cr4)
-	movl	4(%esp), %eax
-	.byte	0x0f,0x22,0xe0		/* movl	%eax, %cr4 */
-	ret
-
-/*
- * Read ldtr
- */
-Entry(get_ldt)
-	xorl	%eax,%eax
-	sldt	%ax
-	ret
-
-/*
- * Set ldtr
- */
-Entry(set_ldt)
-	lldt	4(%esp)
-	ret
-
-/*
- * Read task register.
- */
-ENTRY(get_tr)
-	xorl	%eax,%eax
-	str	%ax
-	ret
-
-/*
- * Set task register.  Also clears busy bit of task descriptor.
- */
-ENTRY(set_tr)
-	movl	S_ARG0,%eax		/* get task segment number */
-	subl	$8,%esp			/* push space for SGDT */
-	sgdt	2(%esp)			/* store GDT limit and base (linear) */
-	movl	4(%esp),%edx		/* address GDT */
-	movb	$(K_TSS),5(%edx,%eax)	/* fix access byte in task descriptor */
-	ltr	%ax			/* load task register */
-	addl	$8,%esp			/* clear stack */
-	ret				/* and return */
-
-/*
- * Set task-switched flag.
- */
-ENTRY(_setts)
-	movl	%cr0,%eax		/* get cr0 */
-	orl	$(CR0_TS),%eax		/* or in TS bit */
-	movl	%eax,%cr0		/* set cr0 */
-	ret
 
 /*
  * io register must not be used on slaves (no AT bus)
@@ -2399,79 +1332,6 @@ ENTRY(insl)
 	POP_FRAME
 	ret
 
-
-/*
- * int inst_fetch(int eip, int cs);
- *
- * Fetch instruction byte.  Return -1 if invalid address.
- */
-	.globl	EXT(inst_fetch)
-LEXT(inst_fetch)
-	movl	S_ARG1, %eax		/* get segment */
-	movw	%ax,%fs			/* into FS */
-	movl	S_ARG0, %eax		/* get offset */
-	RETRY_SECTION
-	RETRY(EXT(inst_fetch))		/* re-load FS on retry */
-	RECOVERY_SECTION
-	RECOVER(EXT(inst_fetch_fault))
-	movzbl	%fs:(%eax),%eax		/* load instruction byte */
-	ret
-
-LEXT(inst_fetch_fault)
-	movl	$-1,%eax		/* return -1 if error */
-	ret
-
-
-#if MACH_KDP
-/*
- * kdp_copy_kmem(char *src, char *dst, int count)
- *
- * Similar to copyin except that both addresses are kernel addresses.
- */
-
-ENTRY(kdp_copy_kmem)
-	pushl	%esi
-	pushl	%edi			/* save registers */
-
-	movl	8+S_ARG0,%esi		/* get kernel start address */
-	movl	8+S_ARG1,%edi		/* get kernel destination address */
-
-	movl	8+S_ARG2,%edx		/* get count */
-
-	lea	0(%esi,%edx),%eax	/* get kernel end address + 1 */
-
-	cmpl	%esi,%eax
-	jb	kdp_vm_read_fail	/* fail if wrap-around */
-	cld				/* count up */
-	movl	%edx,%ecx		/* move by longwords first */
-	shrl	$2,%ecx
-	RECOVERY_SECTION
-	RECOVER(kdp_vm_read_fail)
-	rep
-	movsl				/* move longwords */
-	movl	%edx,%ecx		/* now move remaining bytes */
-	andl	$3,%ecx
-	RECOVERY_SECTION
-	RECOVER(kdp_vm_read_fail)
-	rep
-	movsb
-kdp_vm_read_done:
-	movl	8+S_ARG2,%edx		/* get count */
-	subl	%ecx,%edx		/* Return number of bytes transfered */
-	movl	%edx,%eax
-
-	popl	%edi			/* restore registers */
-	popl	%esi
-	ret				/* and return */
-
-kdp_vm_read_fail:
-	xorl	%eax,%eax	/* didn't copy a thing. */
-
-	popl	%edi
-	popl	%esi
-	ret
-#endif
-
 /*
  * int rdmsr_carefully(uint32_t msr, uint32_t *lo, uint32_t *hi)
  */
@@ -2492,12 +1352,10 @@ rdmsr_fail:
 	ret
 
 /*
- * Done with recovery and retry tables.
+ * Done with recovery table.
  */
 	RECOVERY_SECTION
 	RECOVER_TABLE_END
-	RETRY_SECTION
-	RETRY_TABLE_END
 
 
 
@@ -2572,6 +1430,7 @@ dr_msk:
 ENTRY(dr_addr)
 	.long	0,0,0,0
 	.long	0,0,0,0
+
 	.text
 
 ENTRY(get_cr0)
@@ -2665,34 +1524,6 @@ ENTRY(get_pc)
 	movl	4(%ebp),%eax
 	ret
 
-#if	ETAP
-
-ENTRY(etap_get_pc)
-	movl	4(%ebp), %eax		/* fetch pc of caller */
-	ret
-
-ENTRY(tvals_to_etap)
-        movl	S_ARG0, %eax
-	movl	$1000000000, %ecx
-	mull	%ecx
-	addl	S_ARG1, %eax
-        adc     $0, %edx
-       	ret
-
-/* etap_time_t
- * etap_time_sub(etap_time_t stop, etap_time_t start)
- *	
- *	64bit subtract, returns stop - start
- */		 	
-ENTRY(etap_time_sub)
-	movl	S_ARG0, %eax		/* stop.low */
-	movl	S_ARG1, %edx		/* stop.hi */
-	subl	S_ARG2, %eax		/* stop.lo - start.lo */
-	sbbl	S_ARG3, %edx		/* stop.hi - start.hi */
-	ret
-		
-#endif	/* ETAP */
-	
 ENTRY(minsecurity)
 	pushl	%ebp
 	movl	%esp,%ebp
@@ -2776,174 +1607,33 @@ ENTRY(mul_scale)
 	POP_FRAME
 	ret
 
-#ifdef	MACH_BSD
-/*
- * BSD System call entry point.. 
- */
-
-Entry(trap_unix_syscall)
-trap_unix_addr:	
-	pushf				/* save flags as soon as possible */
-trap_unix_2:	
-	pushl	%eax			/* save system call number */
-	pushl	$0			/* clear trap number slot */
-
-	pusha				/* save the general registers */
-	pushl	%ds			/* and the segment registers */
-	pushl	%es
-	pushl	%fs
-	pushl	%gs
-
-	mov	%ss,%dx			/* switch to kernel data segment */
-	mov	%dx,%ds
-	mov	%dx,%es
-	mov	$ CPU_DATA_GS,%dx
-	mov	%dx,%gs
-
-/*
- * Shuffle eflags,eip,cs into proper places
- */
-
-	movl	R_EIP(%esp),%ebx	/* eflags are in EIP slot */
-	movl	R_CS(%esp),%ecx		/* eip is in CS slot */
-	movl	R_EFLAGS(%esp),%edx	/* cs is in EFLAGS slot */
-	movl	%ecx,R_EIP(%esp)	/* fix eip */
-	movl	%edx,R_CS(%esp)		/* fix cs */
-	movl	%ebx,R_EFLAGS(%esp)	/* fix eflags */
-
-        TIME_TRAP_UENTRY
-
-        negl    %eax                    /* get system call number */
-        shll    $4,%eax                 /* manual indexing */
-
-	movl	%gs:CPU_KERNEL_STACK,%ebx
-					/* get current kernel stack */
-	xchgl	%ebx,%esp		/* switch stacks - %ebx points to */
-					/* user registers. */
-
-/*
- * Register use on entry:
- *   eax contains syscall number
- *   ebx contains user regs pointer
- */
-	CAH(call_call)
-	pushl	%ebx			/* Push the regs set onto stack */
-	call	EXT(unix_syscall)
-	popl	%ebx
-	movl	%esp,%ecx		/* get kernel stack */
-	or	$(KERNEL_STACK_SIZE-1),%ecx
-	movl	-3-IKS_SIZE(%ecx),%esp	/* switch back to PCB stack */
-	movl	%eax,R_EAX(%esp)	/* save return value */
-	jmp	EXT(return_from_trap)	/* return to user */
 
+	
 /*
- * Entry point for machdep system calls..
+ * Double-fault exception handler task. The last gasp...
  */
+Entry(df_task_start)
+	CCALL1(panic_double_fault, $(T_DOUBLE_FAULT))
+	hlt
 
-Entry(trap_machdep_syscall)
-	pushf				/* save flags as soon as possible */
-	pushl	%eax			/* save system call number */
-	pushl	$0			/* clear trap number slot */
-
-	pusha				/* save the general registers */
-	pushl	%ds			/* and the segment registers */
-	pushl	%es
-	pushl	%fs
-	pushl	%gs
-
-	mov	%ss,%dx			/* switch to kernel data segment */
-	mov	%dx,%ds
-	mov	%dx,%es
-	mov	$ CPU_DATA_GS,%dx
-	mov	%dx,%gs
 
 /*
- * Shuffle eflags,eip,cs into proper places
+ * machine-check handler task. The last gasp...
  */
-
-	movl	R_EIP(%esp),%ebx	/* eflags are in EIP slot */
-	movl	R_CS(%esp),%ecx		/* eip is in CS slot */
-	movl	R_EFLAGS(%esp),%edx	/* cs is in EFLAGS slot */
-	movl	%ecx,R_EIP(%esp)	/* fix eip */
-	movl	%edx,R_CS(%esp)		/* fix cs */
-	movl	%ebx,R_EFLAGS(%esp)	/* fix eflags */
-
-        TIME_TRAP_UENTRY
-
-        negl    %eax                    /* get system call number */
-        shll    $4,%eax                 /* manual indexing */
-
-	movl	%gs:CPU_KERNEL_STACK,%ebx
-					/* get current kernel stack */
-	xchgl	%ebx,%esp		/* switch stacks - %ebx points to */
-					/* user registers. */
-
-/*
- * Register use on entry:
- *   eax contains syscall number
- *   ebx contains user regs pointer
- */
-	CAH(call_call)
-	pushl	%ebx
-	call	EXT(machdep_syscall)
-	popl	%ebx
-	movl	%esp,%ecx		/* get kernel stack */
-	or	$(KERNEL_STACK_SIZE-1),%ecx
-	movl	-3-IKS_SIZE(%ecx),%esp	/* switch back to PCB stack */
-	movl	%eax,R_EAX(%esp)	/* save return value */
-	jmp	EXT(return_from_trap)	/* return to user */
-
-Entry(trap_mach25_syscall)
-	pushf				/* save flags as soon as possible */
-	pushl	%eax			/* save system call number */
-	pushl	$0			/* clear trap number slot */
-
-	pusha				/* save the general registers */
-	pushl	%ds			/* and the segment registers */
-	pushl	%es
-	pushl	%fs
-	pushl	%gs
-
-	mov	%ss,%dx			/* switch to kernel data segment */
-	mov	%dx,%ds
-	mov	%dx,%es
-	mov	$ CPU_DATA_GS,%dx
-	mov	%dx,%gs
+Entry(mc_task_start)
+	CCALL1(panic_machine_check, $(T_MACHINE_CHECK))
+	hlt
 
 /*
- * Shuffle eflags,eip,cs into proper places
+ * Compatibility mode's last gasp...
  */
+Entry(lo_df64)
+	movl	%esp, %eax
+	CCALL1(panic_double_fault64, %eax)
+	hlt
 
-	movl	R_EIP(%esp),%ebx	/* eflags are in EIP slot */
-	movl	R_CS(%esp),%ecx		/* eip is in CS slot */
-	movl	R_EFLAGS(%esp),%edx	/* cs is in EFLAGS slot */
-	movl	%ecx,R_EIP(%esp)	/* fix eip */
-	movl	%edx,R_CS(%esp)		/* fix cs */
-	movl	%ebx,R_EFLAGS(%esp)	/* fix eflags */
+Entry(lo_mc64)
+	movl	%esp, %eax
+	CCALL1(panic_machine_check64, %eax)
+	hlt
 
-        TIME_TRAP_UENTRY
-
-        negl    %eax                    /* get system call number */
-        shll    $4,%eax                 /* manual indexing */
-
-	movl	%gs:CPU_KERNEL_STACK,%ebx
-					/* get current kernel stack */
-	xchgl	%ebx,%esp		/* switch stacks - %ebx points to */
-					/* user registers. */
-
-/*
- * Register use on entry:
- *   eax contains syscall number
- *   ebx contains user regs pointer
- */
-	CAH(call_call)
-	pushl	%ebx
-	call	EXT(mach25_syscall)
-	popl	%ebx
-	movl	%esp,%ecx		/* get kernel stack */
-	or	$(KERNEL_STACK_SIZE-1),%ecx
-	movl	-3-IKS_SIZE(%ecx),%esp	/* switch back to PCB stack */
-	movl	%eax,R_EAX(%esp)	/* save return value */
-	jmp	EXT(return_from_trap)	/* return to user */
-
-#endif
diff --git a/osfmk/i386/loose_ends.c b/osfmk/i386/loose_ends.c
index c65c7280e..5fe64cb27 100644
--- a/osfmk/i386/loose_ends.c
+++ b/osfmk/i386/loose_ends.c
@@ -63,6 +63,7 @@
 #include <mach/i386/vm_param.h>
 #include <kern/kern_types.h>
 #include <kern/misc_protos.h>
+#include <sys/errno.h>
 #include <i386/param.h>
 #include <i386/misc_protos.h>
 #include <i386/cpu_data.h>
@@ -73,20 +74,26 @@
 #include <vm/vm_kern.h>
 #include <vm/vm_fault.h>
 
+#include <libkern/OSAtomic.h>
+#include <sys/kdebug.h>
+
+#if 0
+
+#undef KERNEL_DEBUG
+#define KERNEL_DEBUG KERNEL_DEBUG_CONSTANT
+#define KDEBUG 1
+
+#endif
+
 /* XXX - should be gone from here */
 extern void		invalidate_icache64(addr64_t addr, unsigned cnt, int phys);
 extern void		flush_dcache64(addr64_t addr, unsigned count, int phys);
 extern boolean_t	phys_page_exists(ppnum_t);
-extern pt_entry_t	*pmap_mapgetpte(vm_map_t, vm_offset_t);
 extern void		bcopy_no_overwrite(const char *from, char *to,vm_size_t bytes);
 extern void		pmap_set_reference(ppnum_t pn);
 extern void		mapping_set_mod(ppnum_t pa); 
 extern void		mapping_set_ref(ppnum_t pn);
-extern void		switch_to_serial_console(void);
-extern kern_return_t	copyp2p(vm_offset_t	source,
-				vm_offset_t	dest,
-				unsigned int	size,
-				unsigned int	flush_action);
+
 extern void		fillPage(ppnum_t pa, unsigned int fill);
 extern void		ovbcopy(const char	*from,
 				char		*to,
@@ -103,71 +110,28 @@ bzero_phys(
 	   addr64_t src64,
 	   vm_size_t bytes)
 {
-  vm_offset_t src = low32(src64);
-  pt_entry_t save2;
-        mp_disable_preemption();
-	if (*(pt_entry_t *) CM2)
-		panic("bzero_phys: CMAP busy");
+        mapwindow_t *map;
+	pt_entry_t save;
 
-	*(pt_entry_t *) CM2 = INTEL_PTE_VALID | INTEL_PTE_RW | (src & PG_FRAME) | 
-	  INTEL_PTE_REF | INTEL_PTE_MOD;
-	save2=*(pt_entry_t *)CM2;
-	invlpg((u_int)CA2);
-
-        bzero((void *)((unsigned int)CA2 | (src & INTEL_OFFMASK)), bytes);
-	if (save2 != *(pt_entry_t *)CM2)  panic("bzero_phys CMAP changed");
-	*(pt_entry_t *) CM2 = 0;
-	mp_enable_preemption();
-}
+        mp_disable_preemption();
+	map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | ((pmap_paddr_t)src64 & PG_FRAME) | INTEL_PTE_REF | INTEL_PTE_MOD));
+	if (map == 0) {
+	        panic("bzero_phys: CMAP busy");
+	}
+	save = *map->prv_CMAP;
 
-/*
- * copy 'size' bytes from physical to physical address
- * the caller must validate the physical ranges 
- *
- * if flush_action == 0, no cache flush necessary
- * if flush_action == 1, flush the source
- * if flush_action == 2, flush the dest
- * if flush_action == 3, flush both source and dest
- */
+	invlpg((uintptr_t)map->prv_CADDR);
 
-kern_return_t
-copyp2p(vm_offset_t	source,
-	vm_offset_t	dest,
-	unsigned int	size,
-	unsigned int	flush_action)
-{
-
-        switch(flush_action) {
-	case 1:
-	        flush_dcache(source, size, 1);
-		break;
-	case 2:
-	        flush_dcache(dest, size, 1);
-		break;
-	case 3:
-	        flush_dcache(source, size, 1);
-	        flush_dcache(dest, size, 1);
-		break;
+	bzero((void *)((uintptr_t)map->prv_CADDR | ((uint32_t)src64 & INTEL_OFFMASK)), bytes);
 
-	}
-        bcopy_phys((addr64_t)source, (addr64_t)dest, (vm_size_t)size);	/* Do a physical copy */
-
-        switch(flush_action) {
-	case 1:
-	        flush_dcache(source, size, 1);
-		break;
-	case 2:
-	        flush_dcache(dest, size, 1);
-		break;
-	case 3:
-	        flush_dcache(source, size, 1);
-	        flush_dcache(dest, size, 1);
-		break;
+	if (save != *map->prv_CMAP)
+	        panic("bzero_phys: CMAP changed");
+	*map->prv_CMAP = 0;
 
-	}
-	return KERN_SUCCESS;
+	mp_enable_preemption();
 }
 
+
 /*
  * bcopy_phys - like bcopy but copies from/to physical addresses.
  */
@@ -178,30 +142,38 @@ bcopy_phys(
 	   addr64_t dst64,
 	   vm_size_t bytes)
 {
-  vm_offset_t src = low32(src64);
-  vm_offset_t dst = low32(dst64);
-  pt_entry_t save1,save2;
-  /* ensure we stay within a page */
-  if ( (((src & (NBPG-1)) + bytes) > NBPG) ||
-       (((dst & (NBPG-1)) + bytes) > NBPG) ) panic("bcopy_phys");
-  mp_disable_preemption();
-	if (*(pt_entry_t *) CM1 || *(pt_entry_t *) CM2)
-		panic("bcopy_phys: CMAP busy");
-
-	*(pt_entry_t *) CM1 = INTEL_PTE_VALID | (src & PG_FRAME) | INTEL_PTE_REF;
-	*(pt_entry_t *) CM2 = INTEL_PTE_VALID | INTEL_PTE_RW | (dst & PG_FRAME) | 
-	  INTEL_PTE_REF | INTEL_PTE_MOD;
-	save1 = *(pt_entry_t *)CM1;save2 = *(pt_entry_t *)CM2;
-	invlpg((u_int)CA1);
-	invlpg((u_int)CA2);
-
-        bcopy((void *) ((uintptr_t)CA1 | (src & INTEL_OFFMASK)),
-	      (void *) ((uintptr_t)CA2 | (dst & INTEL_OFFMASK)), bytes);
-	if ( (save1 != *(pt_entry_t *)CM1) || (save2 != *(pt_entry_t *)CM2)) panic("bcopy_phys CMAP changed");
-	*(pt_entry_t *) CM1 = 0;
-	*(pt_entry_t *) CM2 = 0;
-	mp_enable_preemption();
+        mapwindow_t *src_map, *dst_map;
+	pt_entry_t save1, save2;
+
+	/* ensure we stay within a page */
+	if ( ((((uint32_t)src64 & (NBPG-1)) + bytes) > NBPG) || ((((uint32_t)dst64 & (NBPG-1)) + bytes) > NBPG) ) {
+	        panic("bcopy_phys alignment");
+	}
+	mp_disable_preemption();
+
+	src_map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | ((pmap_paddr_t)src64 & PG_FRAME) | INTEL_PTE_REF));
+	dst_map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | ((pmap_paddr_t)dst64 & PG_FRAME) |
+						  INTEL_PTE_REF | INTEL_PTE_MOD));
+
+	if (src_map == 0 || dst_map == 0) {
+	        panic("bcopy_phys: CMAP busy");
+	}
+	save1 = *src_map->prv_CMAP;
+	save2 = *dst_map->prv_CMAP;
+
+	invlpg((uintptr_t)src_map->prv_CADDR);
+	invlpg((uintptr_t)dst_map->prv_CADDR);
+
+	bcopy((void *) ((uintptr_t)src_map->prv_CADDR | ((uint32_t)src64 & INTEL_OFFMASK)),
+	      (void *) ((uintptr_t)dst_map->prv_CADDR | ((uint32_t)dst64 & INTEL_OFFMASK)), bytes);
+
+	if ( (save1 != *src_map->prv_CMAP) || (save2 != *dst_map->prv_CMAP))
+	        panic("bcopy_phys CMAP changed");
 
+	*src_map->prv_CMAP = 0;
+	*dst_map->prv_CMAP = 0;
+
+	mp_enable_preemption();
 }
 
 /* 
@@ -236,217 +208,242 @@ ovbcopy(
 
 
 static unsigned int
-ml_phys_read_data( vm_offset_t paddr, int size )
+ml_phys_read_data(pmap_paddr_t paddr, int size )
 {
-    unsigned int result;
-    pt_entry_t save;
-    mp_disable_preemption();
-	if (*(pt_entry_t *) CM3)
-		panic("ml_phys_read_data: CMAP busy");
+        mapwindow_t *map;
+	unsigned int result;
+	pt_entry_t save;
 
-	*(pt_entry_t *) CM3 = INTEL_PTE_VALID | (paddr & PG_FRAME) | INTEL_PTE_REF;
-        save = *(pt_entry_t *)CM3;
-	invlpg((u_int)CA3);
+	mp_disable_preemption();
+	map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | (paddr & PG_FRAME) | INTEL_PTE_REF));
+	if (map == 0) {
+		panic("ml_phys_read_data: CMAP busy");
+	}
 
+	save = *map->prv_CMAP;
+	invlpg((uintptr_t)map->prv_CADDR);
 
         switch (size) {
             unsigned char s1;
             unsigned short s2;
         case 1:
-            s1 = *(unsigned char *)((unsigned int)CA3 | (paddr & INTEL_OFFMASK));
+            s1 = *(unsigned char *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK));
             result = s1;
             break;
         case 2:
-            s2 = *(unsigned short *)((unsigned int)CA3 | (paddr & INTEL_OFFMASK));
+            s2 = *(unsigned short *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK));
             result = s2;
             break;
         case 4:
         default:
-            result = *(unsigned int *)((unsigned int)CA3 | (paddr & INTEL_OFFMASK));
+            result = *(unsigned int *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK));
             break;
         }
 
-	if (save != *(pt_entry_t *)CM3) panic("ml_phys_read_data CMAP changed");
-	*(pt_entry_t *) CM3 = 0;
+	if (save != *map->prv_CMAP)
+	        panic("ml_phys_read_data CMAP changed");
+        *map->prv_CMAP = 0;
 	mp_enable_preemption();
+
         return result;
 }
 
 static unsigned long long
-ml_phys_read_long_long( vm_offset_t paddr )
+ml_phys_read_long_long(pmap_paddr_t paddr )
 {
-    unsigned long long result;
-    pt_entry_t save;
-    mp_disable_preemption();
-	if (*(pt_entry_t *) CM3)
-		panic("ml_phys_read_data: CMAP busy");
+        mapwindow_t *map;
+	unsigned long long result;
+	pt_entry_t save;
 
-	*(pt_entry_t *) CM3 = INTEL_PTE_VALID | (paddr & PG_FRAME) | INTEL_PTE_REF;
-        save = *(pt_entry_t *)CM3;
-	invlpg((u_int)CA3);
+	mp_disable_preemption();
+	map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | (paddr & PG_FRAME) | INTEL_PTE_REF));
 
-        result = *(unsigned long long *)((unsigned int)CA3 | (paddr & INTEL_OFFMASK));
+	if (map == 0) {
+		panic("ml_phys_read_long_long: CMAP busy");
+	}
+
+	save = *map->prv_CMAP;
+	invlpg((uintptr_t)map->prv_CADDR);
 
-	if (save != *(pt_entry_t *)CM3) panic("ml_phys_read_data CMAP changed");
-	*(pt_entry_t *) CM3 = 0;
+	result = *(unsigned long long *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK));
+
+	if (save != *map->prv_CMAP)
+	        panic("ml_phys_read_long_long CMAP changed");
+	*map->prv_CMAP = 0;
 	mp_enable_preemption();
-        return result;
+
+	return result;
 }
 
-unsigned int ml_phys_read( vm_offset_t paddr)
+
+
+unsigned int ml_phys_read(vm_offset_t paddr)
 {
-    return ml_phys_read_data(paddr, 4);
+        return ml_phys_read_data((pmap_paddr_t)paddr, 4);
 }
 
 unsigned int ml_phys_read_word(vm_offset_t paddr) {
-    return ml_phys_read_data(paddr, 4);
+
+        return ml_phys_read_data((pmap_paddr_t)paddr, 4);
 }
 
 unsigned int ml_phys_read_64(addr64_t paddr64)
 {
-    return ml_phys_read_data(low32(paddr64), 4);
+        return ml_phys_read_data((pmap_paddr_t)paddr64, 4);
 }
 
 unsigned int ml_phys_read_word_64(addr64_t paddr64)
 {
-    return ml_phys_read_data(low32(paddr64), 4);
+        return ml_phys_read_data((pmap_paddr_t)paddr64, 4);
 }
 
 unsigned int ml_phys_read_half(vm_offset_t paddr)
 {
-    return ml_phys_read_data(paddr, 2);
+        return ml_phys_read_data((pmap_paddr_t)paddr, 2);
 }
 
 unsigned int ml_phys_read_half_64(addr64_t paddr64)
 {
-    return ml_phys_read_data(low32(paddr64), 2);
+        return ml_phys_read_data((pmap_paddr_t)paddr64, 2);
 }
 
 unsigned int ml_phys_read_byte(vm_offset_t paddr)
 {
-    return ml_phys_read_data(paddr, 1);
+        return ml_phys_read_data((pmap_paddr_t)paddr, 1);
 }
 
 unsigned int ml_phys_read_byte_64(addr64_t paddr64)
 {
-    return ml_phys_read_data(low32(paddr64), 1);
+        return ml_phys_read_data((pmap_paddr_t)paddr64, 1);
 }
 
 unsigned long long ml_phys_read_double(vm_offset_t paddr)
 {
-    return ml_phys_read_long_long(paddr);
+        return ml_phys_read_long_long((pmap_paddr_t)paddr);
 }
 
-unsigned long long ml_phys_read_double_64(addr64_t paddr)
+unsigned long long ml_phys_read_double_64(addr64_t paddr64)
 {
-    return ml_phys_read_long_long(low32(paddr));
+        return ml_phys_read_long_long((pmap_paddr_t)paddr64);
 }
 
 
+
 /*
  *  Write data to a physical address. Memory should not be cache inhibited.
  */
 
 static void
-ml_phys_write_data( vm_offset_t paddr, unsigned long data, int size )
+ml_phys_write_data(pmap_paddr_t paddr, unsigned long data, int size)
 {
-    pt_entry_t save;
-    mp_disable_preemption();
-	if (*(pt_entry_t *) CM3)
+        mapwindow_t *map;
+	pt_entry_t save;
+
+	mp_disable_preemption();
+	map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | (paddr & PG_FRAME) | 
+					  INTEL_PTE_REF | INTEL_PTE_MOD));
+
+	if (map == 0) {
 		panic("ml_phys_write_data: CMAP busy");
+	}
 
-	*(pt_entry_t *) CM3 = INTEL_PTE_VALID | INTEL_PTE_RW | (paddr & PG_FRAME) | 
-	  INTEL_PTE_REF | INTEL_PTE_MOD;
-        save = *(pt_entry_t *)CM3;
-	invlpg((u_int)CA3);
+	save = *map->prv_CMAP;
+	invlpg((uintptr_t)map->prv_CADDR);
 
         switch (size) {
         case 1:
-            *(unsigned char *)((unsigned int)CA3 | (paddr & INTEL_OFFMASK)) = (unsigned char)data;
+	    *(unsigned char *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK)) = (unsigned char)data;
             break;
         case 2:
-            *(unsigned short *)((unsigned int)CA3 | (paddr & INTEL_OFFMASK)) = (unsigned short)data;
+	    *(unsigned short *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK)) = (unsigned short)data;
             break;
         case 4:
         default:
-            *(unsigned int *)((unsigned int)CA3 | (paddr & INTEL_OFFMASK)) = data;
+	    *(unsigned int *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK)) = data;
             break;
         }
 
-	if (save != *(pt_entry_t *)CM3) panic("ml_phys_write_data CMAP changed");
-	*(pt_entry_t *) CM3 = 0;
+	if (save != *map->prv_CMAP)
+	        panic("ml_phys_write_data CMAP changed");
+	*map->prv_CMAP = 0;
+
 	mp_enable_preemption();
 }
 
 static void
-ml_phys_write_long_long( vm_offset_t paddr, unsigned long long data )
+ml_phys_write_long_long(pmap_paddr_t paddr, unsigned long long data)
 {
-    pt_entry_t save;
-    mp_disable_preemption();
-	if (*(pt_entry_t *) CM3)
+        mapwindow_t *map;
+	pt_entry_t save;
+
+	mp_disable_preemption();
+	map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | (paddr & PG_FRAME) | 
+					      INTEL_PTE_REF | INTEL_PTE_MOD));
+	if (map == 0) {
 		panic("ml_phys_write_data: CMAP busy");
+	}
 
-	*(pt_entry_t *) CM3 = INTEL_PTE_VALID | INTEL_PTE_RW | (paddr & PG_FRAME) | 
-	  INTEL_PTE_REF | INTEL_PTE_MOD;
-        save = *(pt_entry_t *)CM3;
-	invlpg((u_int)CA3);
+	save = *map->prv_CMAP;
+	invlpg((uintptr_t)map->prv_CADDR);
 
-        *(unsigned long long *)((unsigned int)CA3 | (paddr & INTEL_OFFMASK)) = data;
+	*(unsigned long long *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK)) = data;
 
-	if (save != *(pt_entry_t *)CM3) panic("ml_phys_write_data CMAP changed");
-	*(pt_entry_t *) CM3 = 0;
+	if (save != *map->prv_CMAP)
+	        panic("ml_phys_write_data CMAP changed");
+	*map->prv_CMAP = 0;
 	mp_enable_preemption();
 }
 
+
+
 void ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
 {
-    ml_phys_write_data(paddr, data, 1);
+        ml_phys_write_data((pmap_paddr_t)paddr, data, 1);
 }
 
-void ml_phys_write_byte_64(addr64_t paddr, unsigned int data)
+void ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)
 {
-    ml_phys_write_data(low32(paddr), data, 1);
+        ml_phys_write_data((pmap_paddr_t)paddr64, data, 1);
 }
 
 void ml_phys_write_half(vm_offset_t paddr, unsigned int data)
 {
-    ml_phys_write_data(paddr, data, 2);
+        ml_phys_write_data((pmap_paddr_t)paddr, data, 2);
 }
 
-void ml_phys_write_half_64(addr64_t paddr, unsigned int data)
+void ml_phys_write_half_64(addr64_t paddr64, unsigned int data)
 {
-    ml_phys_write_data(low32(paddr), data, 2);
+        ml_phys_write_data((pmap_paddr_t)paddr64, data, 2);
 }
 
 void ml_phys_write(vm_offset_t paddr, unsigned int data)
 {
-    ml_phys_write_data(paddr, data, 4);
+        ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
 }
 
-void ml_phys_write_64(addr64_t paddr, unsigned int data)
+void ml_phys_write_64(addr64_t paddr64, unsigned int data)
 {
-    ml_phys_write_data(low32(paddr), data, 4);
+        ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
 }
 
 void ml_phys_write_word(vm_offset_t paddr, unsigned int data)
 {
-    ml_phys_write_data(paddr, data, 4);
+        ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
 }
 
-void ml_phys_write_word_64(addr64_t paddr, unsigned int data)
+void ml_phys_write_word_64(addr64_t paddr64, unsigned int data)
 {
-    ml_phys_write_data(low32(paddr), data, 4);
+        ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
 }
 
-
 void ml_phys_write_double(vm_offset_t paddr, unsigned long long data)
 {
-    ml_phys_write_long_long(paddr, data);
+        ml_phys_write_long_long((pmap_paddr_t)paddr, data);
 }
 
-void ml_phys_write_double_64(addr64_t paddr, unsigned long long data)
+void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)
 {
-    ml_phys_write_long_long(low32(paddr), data);
+        ml_phys_write_long_long((pmap_paddr_t)paddr64, data);
 }
 
 
@@ -462,8 +459,9 @@ void ml_phys_write_double_64(addr64_t paddr, unsigned long long data)
 boolean_t
 ml_probe_read(vm_offset_t paddr, unsigned int *val)
 {
-    *val = ml_phys_read(paddr);
-    return TRUE;
+        *val = ml_phys_read((pmap_paddr_t)paddr);
+
+	return TRUE;
 }
 
 /*
@@ -473,10 +471,11 @@ ml_probe_read(vm_offset_t paddr, unsigned int *val)
  *  We assume that machine checks are enabled both in MSR and HIDs
  */
 boolean_t 
-ml_probe_read_64(addr64_t paddr, unsigned int *val)
+ml_probe_read_64(addr64_t paddr64, unsigned int *val)
 {
-    *val = ml_phys_read_64(paddr);
-    return TRUE;
+        *val = ml_phys_read_64((pmap_paddr_t)paddr64);
+
+	return TRUE;
 }
 
 
@@ -532,8 +531,6 @@ strlen(
 	return string - 1 - ret;
 }
 
-#include <libkern/OSAtomic.h>
-
 uint32_t
 hw_atomic_add(
 	uint32_t	*dest,
@@ -630,23 +627,26 @@ void machine_callstack(
 
 void fillPage(ppnum_t pa, unsigned int fill)
 {
-  pmap_paddr_t src;
-  int i;
-  int cnt = PAGE_SIZE/sizeof(unsigned int);
-  unsigned int *addr;
-  mp_disable_preemption();
-  if (*(pt_entry_t *) CM2)
+    mapwindow_t *map;
+    pmap_paddr_t src;
+    int i;
+    int cnt = PAGE_SIZE/sizeof(unsigned int);
+    unsigned int *addr;
+
+    mp_disable_preemption();
+    src = i386_ptob(pa);
+    map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | (src & PG_FRAME) | 
+					  INTEL_PTE_REF | INTEL_PTE_MOD));
+    if (map == 0) {
         panic("fillPage: CMAP busy");
-  src = (pmap_paddr_t)i386_ptob(pa);
-  *(pt_entry_t *) CM2 = INTEL_PTE_VALID | INTEL_PTE_RW | (src & PG_FRAME) | 
-      INTEL_PTE_REF | INTEL_PTE_MOD;
-      invlpg((u_int)CA2);
+    }
+    invlpg((uintptr_t)map->prv_CADDR);
 
-  for (i = 0, addr = (unsigned int *)CA2; i < cnt ; i++ )
-    *addr++ = fill;
+    for (i = 0, addr = (unsigned int *)map->prv_CADDR; i < cnt ; i++ )
+        *addr++ = fill;
 
-  *(pt_entry_t *) CM2 = 0;
-  mp_enable_preemption();
+    *map->prv_CMAP = 0;
+    mp_enable_preemption();
 }
 
 static inline void __sfence(void)
@@ -663,26 +663,30 @@ static inline void __wbinvd(void)
 }
 static inline void __clflush(void *ptr)
 {
-    __asm__ volatile(".byte 0x0F; .byte 0xae; .byte 0x38" : : "a" (ptr));
+	__asm__ volatile("clflush (%0)" : : "r" (ptr));
 }
 
 void dcache_incoherent_io_store64(addr64_t pa, unsigned int count)
 {
-    if (cpuid_features() & CPUID_FEATURE_CLFSH)
-    {
+        mapwindow_t *map;
         uint32_t  linesize = cpuid_info()->cache_linesize;
         addr64_t  addr;
         uint32_t  offset, chunk;
         boolean_t istate;
 
+	__mfence();
+
         istate = ml_set_interrupts_enabled(FALSE);
 
-        if (*(pt_entry_t *) CM2)
+        offset = pa & (linesize - 1);
+        addr   = pa - offset;
+
+        map = pmap_get_mapwindow((pt_entry_t)(i386_ptob(atop_64(addr)) | INTEL_PTE_VALID));
+        if (map == 0) {
                 panic("cache_flush_page_phys: CMAP busy");
+        }
 
-        offset = pa & (linesize - 1);
         count += offset;
-        addr   = pa - offset;
         offset = addr & ((addr64_t) (page_size - 1));
         chunk  = page_size - offset;
 
@@ -691,11 +695,11 @@ void dcache_incoherent_io_store64(addr64_t pa, unsigned int count)
             if (chunk > count)
                 chunk = count;
 
-            *(pt_entry_t *) CM2 = i386_ptob(atop_64(addr)) | INTEL_PTE_VALID;
-            invlpg((u_int)CA2);
+            *map->prv_CMAP = (pt_entry_t)(i386_ptob(atop_64(addr)) | INTEL_PTE_VALID);
+            invlpg((uintptr_t)map->prv_CADDR);
     
             for (; offset < chunk; offset += linesize)
-                __clflush((void *)(((u_int)CA2) + offset));
+                __clflush((void *)(((uintptr_t)map->prv_CADDR) + offset));
 
             count -= chunk;
             addr  += chunk;
@@ -704,13 +708,11 @@ void dcache_incoherent_io_store64(addr64_t pa, unsigned int count)
         }
         while (count);
 
-        *(pt_entry_t *) CM2 = 0;
+        *map->prv_CMAP = 0;
 
         (void) ml_set_interrupts_enabled(istate);
-    }
-    else
-        __wbinvd();
-    __sfence();
+
+	__mfence();
 }
 
 void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count)
@@ -732,174 +734,586 @@ invalidate_icache64(__unused addr64_t addr,
 {
 }
 
-kern_return_t copypv(addr64_t		src64,
-		     addr64_t		snk64,
-		     unsigned int	size,
-		     int		which)
+
+addr64_t         vm_last_addr;
+
+void
+mapping_set_mod(ppnum_t pn)
 {
- 
-	vm_map_t map;
-	kern_return_t ret;
-	vm_offset_t source, sink;
-	vm_offset_t vaddr;
-	vm_offset_t paddr;
-	spl_t s;
-	unsigned int lop, csize;
-	int needtran, bothphys;
-	vm_prot_t prot;
-	pt_entry_t *ptep;
-	
-	map = (which & cppvKmap) ? kernel_map : current_map_fast();
+  pmap_set_modify(pn);
+}
 
-	source = low32(src64);
-	sink = low32(snk64);
+void
+mapping_set_ref(ppnum_t pn)
+{
+  pmap_set_reference(pn);
+}
 
-	if((which & (cppvPsrc | cppvPsnk)) == 0 ) {		/* Make sure that only one is virtual */
-		panic("copypv: no more than 1 parameter may be virtual\n");	/* Not allowed */
+void
+cache_flush_page_phys(ppnum_t pa)
+{
+        mapwindow_t     *map;
+	boolean_t	istate;
+	int		i;
+	unsigned char	*cacheline_addr;
+	int		cacheline_size = cpuid_info()->cache_linesize;
+	int		cachelines_in_page = PAGE_SIZE/cacheline_size;
+
+	__mfence();
+
+	istate = ml_set_interrupts_enabled(FALSE);
+
+        map = pmap_get_mapwindow((pt_entry_t)(i386_ptob(pa) | INTEL_PTE_VALID));
+	if (map == 0) {
+		panic("cache_flush_page_phys: CMAP busy");
+        }
+
+	invlpg((uintptr_t)map->prv_CADDR);
+
+	for (i = 0, cacheline_addr = (unsigned char *)map->prv_CADDR;
+	     i < cachelines_in_page;
+	     i++, cacheline_addr += cacheline_size) {
+		__clflush((void *) cacheline_addr);
 	}
+
+        *map->prv_CMAP = 0;
+
+	(void) ml_set_interrupts_enabled(istate);
+
+	__mfence();
+}
+
+
+void exit_funnel_section(void)
+{
+        thread_t thread;
+
+	thread = current_thread();
+
+        if (thread->funnel_lock)
+	        (void) thread_funnel_set(thread->funnel_lock, FALSE);
+}
+
+
+
+/*
+ * the copy engine has the following characteristics
+ *   - copyio handles copies to/from user or kernel space
+ *   - copypv deals with physical or virtual addresses
+ *
+ * implementation details as follows
+ *   - a cache of up to NCOPY_WINDOWS is maintained per thread for
+ *     access of user virutal space
+ *   - the window size is determined by the amount of virtual space
+ *     that can be mapped by a single page table
+ *   - the mapping is done by copying the page table pointer from
+ *     the user's directory entry corresponding to the window's
+ *     address in user space to the directory entry corresponding
+ *     to the window slot in the kernel's address space
+ *   - the set of mappings is preserved across context switches,
+ *     so the copy can run with pre-emption enabled
+ *   - there is a gdt entry set up to anchor the kernel window on
+ *     each processor
+ *   - the copies are done using the selector corresponding to the
+ *     gdt entry
+ *   - the addresses corresponding to the user virtual address are
+ *     relative to the beginning of the window being used to map
+ *     that region... thus the thread can be pre-empted and switched
+ *     to a different processor while in the midst of a copy
+ *   - the window caches must be invalidated if the pmap changes out
+ *     from under the thread... this can happen during vfork/exec...
+ *     inval_copy_windows is the invalidation routine to be used
+ *   - the copyio engine has 4 different states associated with it
+ *     that allows for lazy tlb flushes and the ability to avoid
+ *     a flush all together if we've just come from user space
+ *     the 4 states are as follows...
+ *
+ *	WINDOWS_OPENED - set by copyio to indicate to the context
+ *	  switch code that it is necessary to do a tlbflush after
+ * 	  switching the windows since we're in the middle of a copy
+ *
+ *	WINDOWS_CLOSED - set by copyio to indicate that it's done
+ *	  using the windows, so that the context switch code need
+ *	  not do the tlbflush... instead it will set the state to...
+ *
+ *	WINDOWS_DIRTY - set by the context switch code to indicate
+ *	  to the copy engine that it is responsible for doing a 
+ *	  tlbflush before using the windows again... it's also
+ *	  set by the inval_copy_windows routine to indicate the
+ *	  same responsibility.
+ *
+ *	WINDOWS_CLEAN - set by the return to user path to indicate
+ * 	  that a tlbflush has happened and that there is no need
+ *	  for copyio to do another when it is entered next...
+ *
+ *   - a window for mapping single physical pages is provided for copypv
+ *   - this window is maintained across context switches and has the
+ *     same characteristics as the user space windows w/r to pre-emption
+ */
+
+extern int copyout_user(const char *, vm_offset_t, vm_size_t);
+extern int copyout_kern(const char *, vm_offset_t, vm_size_t);
+extern int copyin_user(const vm_offset_t, char *, vm_size_t);
+extern int copyin_kern(const vm_offset_t, char *, vm_size_t);
+extern int copyoutphys_user(const char *, vm_offset_t, vm_size_t);
+extern int copyoutphys_kern(const char *, vm_offset_t, vm_size_t);
+extern int copyinphys_user(const vm_offset_t, char *, vm_size_t);
+extern int copyinphys_kern(const vm_offset_t, char *, vm_size_t);
+extern int copyinstr_user(const vm_offset_t, char *, vm_size_t, vm_size_t *);
+extern int copyinstr_kern(const vm_offset_t, char *, vm_size_t, vm_size_t *);
+
+static int copyio(int, user_addr_t, char *, vm_size_t, vm_size_t *, int);
+static int copyio_phys(addr64_t, addr64_t, vm_size_t, int);
+
+
+#define COPYIN		0
+#define COPYOUT		1
+#define COPYINSTR	2
+#define COPYINPHYS	3
+#define COPYOUTPHYS	4
+
+
+
+void inval_copy_windows(thread_t thread)
+{
+        int	i;
 	
-	bothphys = 1;									/* Assume both are physical */
-	
-	if(!(which & cppvPsnk)) {						/* Is there a virtual page here? */
-		vaddr = sink;								/* Sink side is virtual */
-		bothphys = 0;								/* Show both aren't physical */
-		prot = VM_PROT_READ | VM_PROT_WRITE;		/* Sink always must be read/write */
-	} else /* if(!(which & cppvPsrc)) */ {				/* Source side is virtual */
-		vaddr = source;								/* Source side is virtual */
-		bothphys = 0;								/* Show both aren't physical */
-		prot = VM_PROT_READ; 						/* Virtual source is always read only */
+	for (i = 0; i < NCOPY_WINDOWS; i++) {
+                thread->machine.copy_window[i].user_base = -1;
 	}
+	thread->machine.nxt_window = 0;
+	thread->machine.copyio_state = WINDOWS_DIRTY;
 
-	needtran = 1;									/* Show we need to map the virtual the first time */
-	s = splhigh();									/* Don't bother me */
-
-	while(size) {
-
-		if(!bothphys && (needtran || !(vaddr & 4095LL))) {	/* If first time or we stepped onto a new page, we need to translate */
-		        needtran = 0;
-			while(1) {
-			  ptep = pmap_mapgetpte(map, vaddr);
-			  if((0 == ptep) || ((*ptep & INTEL_PTE_VALID) == 0)) {
-					splx(s);						/* Restore the interrupt level */
-					ret = vm_fault(map, vm_map_trunc_page(vaddr), prot, FALSE, THREAD_UNINT, NULL, 0);	/* Didn't find it, try to fault it in... */
-				
-					if(ret != KERN_SUCCESS)return KERN_FAILURE;	/* Didn't find any, return no good... */
-					
-					s = splhigh();					/* Don't bother me */
-					continue;						/* Go try for the map again... */
-	
-				}
-		
-				/* Note that we have to have the destination writable.  So, if we already have it, or we are mapping the source,
-					we can just leave.
-				*/		
-				if((which & cppvPsnk) || (*ptep & INTEL_PTE_WRITE)) break;		/* We got it mapped R/W or the source is not virtual, leave... */
-				splx(s);							/* Restore the interrupt level */
-				
-				ret = vm_fault(map, vm_map_trunc_page(vaddr), VM_PROT_READ | VM_PROT_WRITE, FALSE, THREAD_UNINT, NULL, 0);	/* check for a COW area */
-				if (ret != KERN_SUCCESS) return KERN_FAILURE;	/* We couldn't get it R/W, leave in disgrace... */
-				s = splhigh();						/* Don't bother me */
-			}
+	KERNEL_DEBUG(0xeff70058 | DBG_FUNC_NONE, (int)thread, (int)thread->map, 0, 0, 0);
+}
 
-		        paddr = pte_to_pa(*ptep) | (vaddr & 4095);
-	
-			if(which & cppvPsrc) sink = paddr;		/* If source is physical, then the sink is virtual */
-			else source = paddr;					/* Otherwise the source is */
+
+static int
+copyio(int copy_type, user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes, vm_size_t *lencopied, int use_kernel_map)
+{
+        thread_t	thread;
+	pmap_t		pmap;
+	pt_entry_t	*updp;
+	pt_entry_t	*kpdp;
+	user_addr_t 	user_base;
+	vm_offset_t 	user_offset;
+	vm_offset_t 	kern_vaddr;
+	vm_size_t	cnt;
+	vm_size_t	bytes_copied;
+	int		error = 0;
+	int		window_index;
+	int		copyio_state;
+        boolean_t	istate;
+#if KDEBUG
+	int		debug_type = 0xeff70010;
+	debug_type += (copy_type << 2);
+#endif
+
+	thread = current_thread();
+
+	KERNEL_DEBUG(debug_type | DBG_FUNC_START, (int)(user_addr >> 32), (int)user_addr, (int)nbytes, thread->machine.copyio_state, 0);
+
+	if (nbytes == 0) {
+	        KERNEL_DEBUG(debug_type | DBG_FUNC_END, (int)user_addr, (int)kernel_addr, (int)nbytes, 0, 0);
+	        return (0);
+	}
+        pmap = thread->map->pmap;
+
+        if (pmap == kernel_pmap || use_kernel_map) {
+
+	        kern_vaddr = (vm_offset_t)user_addr;
+	  
+	        switch (copy_type) {
+
+		case COPYIN:
+		        error = copyin_kern(kern_vaddr, kernel_addr, nbytes);
+			break;
+
+		case COPYOUT:
+		        error = copyout_kern(kernel_addr, kern_vaddr, nbytes);
+			break;
+
+		case COPYINSTR:
+		        error = copyinstr_kern(kern_vaddr, kernel_addr, nbytes, lencopied);
+			break;
+
+		case COPYINPHYS:
+		        error = copyinphys_kern(kern_vaddr, kernel_addr, nbytes);
+			break;
+
+		case COPYOUTPHYS:
+		        error = copyoutphys_kern(kernel_addr, kern_vaddr, nbytes);
+			break;
 		}
-			
-		lop = (unsigned int)(4096LL - (sink & 4095LL));		/* Assume sink smallest */
-		if(lop > (unsigned int)(4096LL - (source & 4095LL))) lop = (unsigned int)(4096LL - (source & 4095LL));	/* No, source is smaller */
-		
-		csize = size;								/* Assume we can copy it all */
-		if(lop < size) csize = lop;					/* Nope, we can't do it all */
-		
-		if(which & cppvFsrc) flush_dcache64((addr64_t)source, csize, 1);	/* If requested, flush source before move */
-		if(which & cppvFsnk) flush_dcache64((addr64_t)sink, csize, 1);	/* If requested, flush sink before move */
+		KERNEL_DEBUG(debug_type | DBG_FUNC_END, (int)kern_vaddr, (int)kernel_addr, (int)nbytes, error | 0x80000000, 0);
 
-		bcopy_phys((addr64_t)source, (addr64_t)sink, csize);			/* Do a physical copy, virtually */
-		
-		if(which & cppvFsrc) flush_dcache64((addr64_t)source, csize, 1);	/* If requested, flush source after move */
-		if(which & cppvFsnk) flush_dcache64((addr64_t)sink, csize, 1);	/* If requested, flush sink after move */
+		return (error);
+	}
+	user_base = user_addr & ~((user_addr_t)(NBPDE - 1));
+	user_offset = user_addr & (NBPDE - 1);
 
+	KERNEL_DEBUG(debug_type | DBG_FUNC_NONE, (int)(user_base >> 32), (int)user_base, (int)user_offset, 0, 0);
 
-/*
- *		Note that for certain ram disk flavors, we may be copying outside of known memory.
- *		Therefore, before we try to mark it modifed, we check if it exists.
- */
+	cnt = NBPDE - user_offset;
+
+	if (cnt > nbytes)
+	        cnt = nbytes;
+
+	istate = ml_set_interrupts_enabled(FALSE);
+
+	copyio_state = thread->machine.copyio_state;
+	thread->machine.copyio_state = WINDOWS_OPENED;
+
+	(void) ml_set_interrupts_enabled(istate);
+
+
+	for (;;) {
 
-		if( !(which & cppvNoModSnk)) {
-		  if (phys_page_exists((ppnum_t)sink >> 12))
-			mapping_set_mod((ppnum_t)(sink >> 12));		/* Make sure we know that it is modified */
+	        for (window_index = 0; window_index < NCOPY_WINDOWS; window_index++) {
+		        if (thread->machine.copy_window[window_index].user_base == user_base)
+			        break;
 		}
-		if( !(which & cppvNoRefSrc)) {
-		  if (phys_page_exists((ppnum_t)source >> 12))
-			mapping_set_ref((ppnum_t)(source >> 12));		/* Make sure we know that it is modified */
+	        if (window_index >= NCOPY_WINDOWS) {
+
+		        window_index = thread->machine.nxt_window;
+			thread->machine.nxt_window++;
+
+			if (thread->machine.nxt_window >= NCOPY_WINDOWS)
+			        thread->machine.nxt_window = 0;
+			thread->machine.copy_window[window_index].user_base = user_base;
+
+			/*
+			 * it's necessary to disable pre-emption
+			 * since I have to compute the kernel descriptor pointer
+			 * for the new window
+			 */
+			istate = ml_set_interrupts_enabled(FALSE);
+
+		        updp = pmap_pde(pmap, user_base);
+
+			kpdp = current_cpu_datap()->cpu_copywindow_pdp;
+			kpdp += window_index;
+
+			pmap_store_pte(kpdp, updp ? *updp : 0);
+
+			(void) ml_set_interrupts_enabled(istate);
+
+		        copyio_state = WINDOWS_DIRTY;
+
+			KERNEL_DEBUG(0xeff70040 | DBG_FUNC_NONE, window_index, (int)user_base, (int)updp, (int)kpdp, 0);
+
 		}
+#if JOE_DEBUG
+		else {
+			istate = ml_set_interrupts_enabled(FALSE);
 
+		        updp = pmap_pde(pmap, user_base);
 
-		size = size - csize;						/* Calculate what is left */
-		vaddr = vaddr + csize;						/* Move to next sink address */
-		source = source + csize;					/* Bump source to next physical address */
-		sink = sink + csize;						/* Bump sink to next physical address */
+			kpdp = current_cpu_datap()->cpu_copywindow_pdp;
+
+			kpdp += window_index;
+
+			if ((*kpdp & PG_FRAME) != (*updp & PG_FRAME)) {
+				panic("copyio: user pdp mismatch - kpdp = 0x%x,  updp = 0x%x\n", kpdp, updp);
+			}
+			(void) ml_set_interrupts_enabled(istate);
+		}
+#endif
+		if (copyio_state == WINDOWS_DIRTY) {
+		        flush_tlb();
+
+		        copyio_state = WINDOWS_CLEAN;
+
+			KERNEL_DEBUG(0xeff70054 | DBG_FUNC_NONE, window_index, 0, 0, 0, 0);
+		}
+		user_offset += (window_index * NBPDE);
+
+		KERNEL_DEBUG(0xeff70044 | DBG_FUNC_NONE, (int)user_offset, (int)kernel_addr, cnt, 0, 0);
+
+	        switch (copy_type) {
+
+		case COPYIN:
+		        error = copyin_user(user_offset, kernel_addr, cnt);
+			break;
+			
+		case COPYOUT:
+		        error = copyout_user(kernel_addr, user_offset, cnt);
+			break;
+
+		case COPYINPHYS:
+		        error = copyinphys_user(user_offset, kernel_addr, cnt);
+			break;
+			
+		case COPYOUTPHYS:
+		        error = copyoutphys_user(kernel_addr, user_offset, cnt);
+			break;
+
+		case COPYINSTR:
+		        error = copyinstr_user(user_offset, kernel_addr, cnt, &bytes_copied);
+
+			/*
+			 * lencopied should be updated on success
+			 * or ENAMETOOLONG...  but not EFAULT
+			 */
+			if (error != EFAULT)
+			        *lencopied += bytes_copied;
+
+			/*
+			 * if we still have room, then the ENAMETOOLONG
+			 * is just an artifact of the buffer straddling
+			 * a window boundary and we should continue
+			 */
+			if (error == ENAMETOOLONG && nbytes > cnt)
+			        error = 0;
+
+			if (error) {
+#if KDEBUG
+			        nbytes = *lencopied;
+#endif
+			        break;
+			}
+			if (*(kernel_addr + bytes_copied - 1) == 0) {
+			        /*
+				 * we found a NULL terminator... we're done
+				 */
+#if KDEBUG
+			        nbytes = *lencopied;
+#endif
+				goto done;
+			}
+			if (cnt == nbytes) {
+			        /*
+				 * no more room in the buffer and we haven't
+				 * yet come across a NULL terminator
+				 */
+#if KDEBUG
+			        nbytes = *lencopied;
+#endif
+			        error = ENAMETOOLONG;
+				break;
+			}
+			assert(cnt == bytes_copied);
+
+			break;
+		}
+		if (error)
+		        break;
+		if ((nbytes -= cnt) == 0)
+		        break;
+
+		kernel_addr += cnt;
+		user_base += NBPDE;
+		user_offset = 0;
+
+		if (nbytes > NBPDE)
+		        cnt = NBPDE;
+		else
+		        cnt = nbytes;
 	}
-	
-	splx(s);										/* Open up for interrupts */
+done:
+	thread->machine.copyio_state = WINDOWS_CLOSED;
 
-	return KERN_SUCCESS;
+	KERNEL_DEBUG(debug_type | DBG_FUNC_END, (int)user_addr, (int)kernel_addr, (int)nbytes, error, 0);
+
+	return (error);
 }
 
-void switch_to_serial_console(void)
+
+static int
+copyio_phys(addr64_t source, addr64_t sink, vm_size_t csize, int which)
 {
+        pmap_paddr_t paddr;
+	user_addr_t  vaddr;
+	char        *window_offset;
+	pt_entry_t  pentry;
+	int         ctype;
+	int	    retval;
+	boolean_t   istate;
+
+	if (which & cppvPsnk) {
+		paddr  = (pmap_paddr_t)sink;
+	        vaddr  = (user_addr_t)source;
+		ctype  = COPYINPHYS;
+		pentry = (pt_entry_t)(INTEL_PTE_VALID | (paddr & PG_FRAME) | INTEL_PTE_RW);
+	} else {
+	        paddr  = (pmap_paddr_t)source;
+		vaddr  = (user_addr_t)sink;
+		ctype  = COPYOUTPHYS;
+		pentry = (pt_entry_t)(INTEL_PTE_VALID | (paddr & PG_FRAME));
+	}
+	window_offset = (char *)((uint32_t)paddr & (PAGE_SIZE - 1));
+
+	if (current_thread()->machine.physwindow_busy) {
+	        pt_entry_t	old_pentry;
+
+	        KERNEL_DEBUG(0xeff70048 | DBG_FUNC_NONE, paddr, csize, 0, -1, 0);
+		/*
+		 * we had better be targeting wired memory at this point
+		 * we will not be able to handle a fault with interrupts
+		 * disabled... we disable them because we can't tolerate
+		 * being preempted during this nested use of the window
+		 */
+		istate = ml_set_interrupts_enabled(FALSE);
+
+		old_pentry = *(current_cpu_datap()->cpu_physwindow_ptep);
+		pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep), pentry);
+
+		invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);
+
+		retval = copyio(ctype, vaddr, window_offset, csize, NULL, which & cppvKmap);
+
+		pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep), old_pentry);
+
+		invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);
+
+		(void) ml_set_interrupts_enabled(istate);
+	} else {
+	        /*
+		 * mark the window as in use... if an interrupt hits while we're
+		 * busy, or we trigger another coyppv from the fault path into
+		 * the driver on a user address space page fault due to a copyin/out
+		 * then we need to save and restore the current window state instead
+		 * of caching the window preserving it across context switches
+		 */
+	        current_thread()->machine.physwindow_busy = 1;
+
+	        if (current_thread()->machine.physwindow_pte != pentry) {
+		        KERNEL_DEBUG(0xeff70048 | DBG_FUNC_NONE, paddr, csize, 0, 0, 0);
+
+			current_thread()->machine.physwindow_pte = pentry;
+			
+			/*
+			 * preemption at this point would be bad since we
+			 * could end up on the other processor after we grabbed the
+			 * pointer to the current cpu data area, but before we finished
+			 * using it to stuff the page table entry since we would
+			 * be modifying a window that no longer belonged to us
+			 * the invlpg can be done unprotected since it only flushes
+			 * this page address from the tlb... if it flushes the wrong
+			 * one, no harm is done, and the context switch that moved us
+			 * to the other processor will have already take care of 
+			 * flushing the tlb after it reloaded the page table from machine.physwindow_pte
+			 */
+			istate = ml_set_interrupts_enabled(FALSE);
+			*(current_cpu_datap()->cpu_physwindow_ptep) = pentry;
+			(void) ml_set_interrupts_enabled(istate);
+
+			invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);
+		}
+#if JOE_DEBUG
+		else {
+		        if (pentry !=
+			    (*(current_cpu_datap()->cpu_physwindow_ptep) & (INTEL_PTE_VALID | PG_FRAME | INTEL_PTE_RW)))
+			        panic("copyio_phys: pentry != *physwindow_ptep");
+		}
+#endif
+		retval = copyio(ctype, vaddr, window_offset, csize, NULL, which & cppvKmap);
+
+	        current_thread()->machine.physwindow_busy = 0;
+	}
+	return (retval);
 }
 
-addr64_t         vm_last_addr;
 
-void
-mapping_set_mod(ppnum_t pn)
+
+int
+copyinmsg(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes)
 {
-  pmap_set_modify(pn);
-}
+        return (copyio(COPYIN, user_addr, kernel_addr, nbytes, NULL, 0));
+}    
 
-void
-mapping_set_ref(ppnum_t pn)
+int
+copyin(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes)
 {
-  pmap_set_reference(pn);
+        return (copyio(COPYIN, user_addr, kernel_addr, nbytes, NULL, 0));
 }
 
-void
-cache_flush_page_phys(ppnum_t pa)
+int
+copyinstr(const user_addr_t user_addr,  char *kernel_addr, vm_size_t nbytes, vm_size_t *lencopied)
 {
-	boolean_t	istate;
-	int		i;
-	unsigned int	*cacheline_addr;
-	int		cacheline_size = cpuid_info()->cache_linesize;
-	int		cachelines_in_page = PAGE_SIZE/cacheline_size;
+	*lencopied = 0;
 
-	/*
-	 * If there's no clflush instruction, we're sadly forced to use wbinvd.
-	 */
-	if (!(cpuid_features() & CPUID_FEATURE_CLFSH)) {
-		asm volatile("wbinvd" : : : "memory");
-		return;
-	} 
+        return (copyio(COPYINSTR, user_addr, kernel_addr, nbytes, lencopied, 0));
+}
 
-	istate = ml_set_interrupts_enabled(FALSE);
+int
+copyoutmsg(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
+{
+        return (copyio(COPYOUT, user_addr, (char *)kernel_addr, nbytes, NULL, 0));
+}
 
-	if (*(pt_entry_t *) CM2)
-		panic("cache_flush_page_phys: CMAP busy");
+int
+copyout(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
+{
+        return (copyio(COPYOUT, user_addr, (char *)kernel_addr, nbytes, NULL, 0));
+}
 
-	*(pt_entry_t *) CM2 = i386_ptob(pa) | INTEL_PTE_VALID;
-	invlpg((u_int)CA2);
 
-	for (i = 0, cacheline_addr = (unsigned int *)CA2;
-	     i < cachelines_in_page;
-	     i++, cacheline_addr += cacheline_size) {
-		asm volatile("clflush %0" : : "m" (cacheline_addr));
-	}
+kern_return_t copypv(addr64_t src64, addr64_t snk64, unsigned int size, int which)
+{
+	unsigned int lop, csize;
+	int bothphys = 0;
+	
 
-	*(pt_entry_t *) CM2 = 0;
+	KERNEL_DEBUG(0xeff7004c | DBG_FUNC_START, (int)src64, (int)snk64, size, which, 0);
 
-	(void) ml_set_interrupts_enabled(istate);
+	if ((which & (cppvPsrc | cppvPsnk)) == 0 )				/* Make sure that only one is virtual */
+		panic("copypv: no more than 1 parameter may be virtual\n");	/* Not allowed */
 
-}
+	if ((which & (cppvPsrc | cppvPsnk)) == (cppvPsrc | cppvPsnk))
+	        bothphys = 1;							/* both are physical */
+
+	while (size) {
+	  
+	        if (bothphys) {
+		        lop = (unsigned int)(PAGE_SIZE - (snk64 & (PAGE_SIZE - 1)));		/* Assume sink smallest */
+
+			if (lop > (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1))))
+			        lop = (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1)));	/* No, source is smaller */
+		} else {
+		        /*
+			 * only need to compute the resid for the physical page
+			 * address... we don't care about where we start/finish in
+			 * the virtual since we just call the normal copyin/copyout
+			 */
+		        if (which & cppvPsrc)
+			        lop = (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1)));
+			else
+			        lop = (unsigned int)(PAGE_SIZE - (snk64 & (PAGE_SIZE - 1)));
+		}
+		csize = size;						/* Assume we can copy it all */
+		if (lop < size)
+		        csize = lop;					/* Nope, we can't do it all */
+#if 0		
+		/*
+		 * flush_dcache64 is currently a nop on the i386... 
+		 * it's used when copying to non-system memory such
+		 * as video capture cards... on PPC there was a need
+		 * to flush due to how we mapped this memory... not
+		 * sure if it's needed on i386.
+		 */
+		if (which & cppvFsrc)
+		        flush_dcache64(src64, csize, 1);		/* If requested, flush source before move */
+		if (which & cppvFsnk)
+		        flush_dcache64(snk64, csize, 1);		/* If requested, flush sink before move */
+#endif
+		if (bothphys)
+		        bcopy_phys(src64, snk64, csize);		/* Do a physical copy, virtually */
+		else {
+		        if (copyio_phys(src64, snk64, csize, which))
+			        return (KERN_FAILURE);
+		}
+#if 0
+		if (which & cppvFsrc)
+		        flush_dcache64(src64, csize, 1);	/* If requested, flush source after move */
+		if (which & cppvFsnk)
+		        flush_dcache64(snk64, csize, 1);	/* If requested, flush sink after move */
+#endif
+		size  -= csize;					/* Calculate what is left */
+		snk64 += csize;					/* Bump sink to next physical address */
+		src64 += csize;					/* Bump source to next physical address */
+	}
+	KERNEL_DEBUG(0xeff7004c | DBG_FUNC_END, (int)src64, (int)snk64, size, which, 0);
 
+	return KERN_SUCCESS;
+}
diff --git a/osfmk/i386/lowglobals.h b/osfmk/i386/lowglobals.h
new file mode 100644
index 000000000..97645667b
--- /dev/null
+++ b/osfmk/i386/lowglobals.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ *		Header files for the Low Memory Globals (lg) 
+ */
+#ifndef	_LOW_MEMORY_GLOBALS_H_
+#define	_LOW_MEMORY_GLOBALS_H_
+
+#include <mach/mach_types.h>
+#include <mach/vm_types.h>
+#include <mach/machine/vm_types.h>
+#include <mach/vm_prot.h>
+
+/*
+ * Don't change these structures unless you change the corresponding assembly code
+ * which is in lowmem_vectors.s
+ */
+ 
+/* 
+ *	This is where we put constants, pointers, and data areas that must be accessed
+ *	quickly through assembler.  They are designed to be accessed directly with 
+ *	absolute addresses, not via a base register.  This is a global area, and not
+ *	per processor.
+ */
+ 
+#pragma pack(4)		/* Make sure the structure stays as we defined it */
+typedef struct lowglo {
+
+	unsigned char	lgVerCode[8];		/* 0x2000 System verification code */
+	unsigned long long lgZero;		/* 0x2008 Double constant 0 */
+	uint32_t	lgRsv010[3];		/* 0x2010 Reserved */
+	uint32_t	lgVersion;		/* 0x201C Pointer to kernel version string */
+	uint32_t	lgRsv020[280];		/* 0X2020 Reserved */
+	uint32_t	lgKmodptr;		/* 0x2480 Pointer to kmod, debugging aid */
+	uint32_t	lgTransOff;		/* 0x2484 Pointer to kdp_trans_off, debugging aid */
+	uint32_t	lgRsv028;		/* 0x2488 Reserved */
+	uint32_t	lgDevSlot1;		/* 0x248C For developer use */
+	uint32_t	lgDevSlot2;		/* 0x2490 For developer use */
+	uint32_t	lgRsv494[731];		/* Reserved - push to 1 page */
+} lowglo;
+
+
+#endif /* _LOW_MEMORY_GLOBALS_H_ */
diff --git a/osfmk/i386/lowmem_vectors.s b/osfmk/i386/lowmem_vectors.s
new file mode 100644
index 000000000..88052cc38
--- /dev/null
+++ b/osfmk/i386/lowmem_vectors.s
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+
+#include <platforms.h>
+#include <mach_kdb.h>
+
+#include <i386/asm.h>
+#include <i386/proc_reg.h>
+#include <i386/postcode.h>
+#include <assym.s>
+
+
+/* on x86 the low mem vectors live here and get mapped to 0x2000 at
+ * system startup time
+ */
+
+	.text
+	.align	12
+	
+	.globl	EXT(lowGlo)
+EXT(lowGlo):
+
+	.ascii "Catfish "		/* 0x2000 System verification code */
+	.long   0			/* 0x2008 Double constant 0 */
+	.long   0
+	.long	0			/* 0x2010 Reserved */
+	.long	0			/* 0x2014 Reserved */
+	.long	0			/* 0x2018 Reserved */
+	.long	EXT(version)		/* 0x201C Pointer to kernel version string */
+	.fill	280, 4, 0		/* 0x2020 Reserved */
+	.long	EXT(kmod)		/* 0x2480 Pointer to kmod, debugging aid */
+	.long	EXT(kdp_trans_off)	/* 0x2484 Pointer to kdp_trans_off, debugging aid */
+	.long	0			/* 0x2488 Reserved */
+	.long	0			/* 0x248C Reserved for developer use */
+	.long	0			/* 0x2490 Reserved for developer use */
+	.fill	731, 4, 0
diff --git a/osfmk/i386/machdep_call.c b/osfmk/i386/machdep_call.c
index 841d3cde0..831bbe18c 100644
--- a/osfmk/i386/machdep_call.c
+++ b/osfmk/i386/machdep_call.c
@@ -41,11 +41,6 @@
 #include <i386/machdep_call.h>
 
 extern kern_return_t	kern_invalid(void);
-#ifdef	FIXME
-extern kern_return_t	PCcreate(), PCldt(), PCresume();
-extern kern_return_t	PCcopyBIOSData(), PCmapBIOSRom();
-extern kern_return_t	PCsizeBIOSExtData(), PCcopyBIOSExtData();
-#endif
 
 machdep_call_t		machdep_call_table[] = {
 	MACHDEP_CALL_ROUTINE(thread_get_cthread_self,0),
@@ -53,16 +48,19 @@ machdep_call_t		machdep_call_table[] = {
 	MACHDEP_CALL_ROUTINE(kern_invalid,0),
 	MACHDEP_CALL_ROUTINE(thread_fast_set_cthread_self,1),
 	MACHDEP_CALL_ROUTINE(thread_set_user_ldt,3),
-#ifdef	FIXME
-	MACHDEP_CALL_ROUTINE(PCcreate,3),
-	MACHDEP_CALL_ROUTINE(PCldt,3),
-	MACHDEP_CALL_ROUTINE(PCresume,0),
-	MACHDEP_CALL_ROUTINE(PCcopyBIOSData,1),
-	MACHDEP_CALL_ROUTINE(PCsizeBIOSExtData,0),
-	MACHDEP_CALL_ROUTINE(PCcopyBIOSExtData,1),
-	MACHDEP_CALL_ROUTINE(PCmapBIOSRom,3),
-#endif
+	MACHDEP_BSD_CALL_ROUTINE(i386_set_ldt,3),
+	MACHDEP_BSD_CALL_ROUTINE(i386_get_ldt,3),
+};
+machdep_call_t		machdep_call_table64[] = {
+	MACHDEP_CALL_ROUTINE(kern_invalid,0),
+	MACHDEP_CALL_ROUTINE(kern_invalid,0),
+	MACHDEP_CALL_ROUTINE(kern_invalid,0),
+	MACHDEP_CALL_ROUTINE64(thread_fast_set_cthread_self64,1),
+	MACHDEP_CALL_ROUTINE(kern_invalid,0),
+	MACHDEP_CALL_ROUTINE(kern_invalid,0),
+	MACHDEP_CALL_ROUTINE(kern_invalid,0),
 };
 
 int	machdep_call_count =
     (sizeof (machdep_call_table) / sizeof (machdep_call_t));
+
diff --git a/osfmk/i386/machdep_call.h b/osfmk/i386/machdep_call.h
index b78c7113f..62c2cd141 100644
--- a/osfmk/i386/machdep_call.h
+++ b/osfmk/i386/machdep_call.h
@@ -39,27 +39,42 @@
 typedef union {
 	kern_return_t		(*args_0)(void);
 	kern_return_t		(*args_1)(uint32_t);
+	kern_return_t		(*args64_1)(uint64_t);
 	kern_return_t		(*args_2)(uint32_t,uint32_t);
 	kern_return_t		(*args_3)(uint32_t,uint32_t,uint32_t);
-	kern_return_t		(*args_4)(uint32_t, uint32_t,uint32_t,uint32_t);
+	kern_return_t		(*args_4)(uint32_t,uint32_t,uint32_t,uint32_t);
 	kern_return_t		(*args_var)(uint32_t,...);
+	int			(*args_bsd_3)(int *, uint32_t,uint32_t,uint32_t);
 } machdep_call_routine_t;
 
-#define MACHDEP_CALL_ROUTINE(func,args)	\
-	{ { .args_ ## args = func }, args }
+#define MACHDEP_CALL_ROUTINE(func, args)	\
+	{ { .args_ ## args = func }, args, 0 }
+	
+#define MACHDEP_CALL_ROUTINE64(func, args)	\
+	{ { .args64_ ## args = func }, args, 0 }
+	
+#define MACHDEP_BSD_CALL_ROUTINE(func, args)	\
+	{ { .args_bsd_ ## args = func }, args, 1 }
 
 typedef struct {
     machdep_call_routine_t	routine;
     int				nargs;
+    int				bsd_style;
 } machdep_call_t;
 
 extern machdep_call_t		machdep_call_table[];
+extern machdep_call_t		machdep_call_table64[];
+
 extern int			machdep_call_count;
 
 extern kern_return_t		thread_get_cthread_self(void);
 extern kern_return_t		thread_set_cthread_self(uint32_t);
 extern kern_return_t		thread_fast_set_cthread_self(uint32_t);
+extern kern_return_t		thread_fast_set_cthread_self64(uint64_t);
 extern kern_return_t		thread_set_user_ldt(uint32_t,uint32_t,uint32_t);
 
-extern void			mach25_syscall(struct i386_saved_state *);
-extern void			machdep_syscall(struct i386_saved_state *);
+extern int			i386_set_ldt(int *,uint32_t,uint32_t,uint32_t);
+extern int			i386_get_ldt(int *,uint32_t,uint32_t,uint32_t);
+
+extern void			machdep_syscall(x86_saved_state_t *);
+extern void			machdep_syscall64(x86_saved_state_t *);
diff --git a/osfmk/i386/machine_check.c b/osfmk/i386/machine_check.c
new file mode 100644
index 000000000..08fbb1116
--- /dev/null
+++ b/osfmk/i386/machine_check.c
@@ -0,0 +1,268 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <i386/cpu_data.h>
+#include <i386/cpuid.h>
+#include <i386/machine_check.h>
+#include <i386/proc_reg.h>
+
+#define IF(bool,str)	((bool) ? (str) : "")
+
+static boolean_t	mca_initialized = FALSE;
+static boolean_t	mca_MCE_present = FALSE;
+static boolean_t	mca_MCA_present = FALSE;
+static unsigned int	mca_error_bank_count = 0;
+static boolean_t	mca_control_MSR_present = FALSE;
+static boolean_t	mca_threshold_status_present = FALSE;
+static boolean_t	mca_extended_MSRs_present = FALSE;
+static unsigned int	mca_extended_MSRs_count = 0;
+static ia32_mcg_cap_t	ia32_mcg_cap;
+
+static void
+mca_get_availability(void)
+{
+	uint64_t	features = cpuid_info()->cpuid_features;
+	uint32_t	family =  cpuid_info()->cpuid_family;
+
+	mca_MCE_present = (features & CPUID_FEATURE_MCE) != 0;
+	mca_MCA_present = (features & CPUID_FEATURE_MCA) != 0;
+	
+	/*
+	 * If MCA, the number of banks etc is reported by the IA32_MCG_CAP MSR.
+	 */
+	if (mca_MCA_present) {
+		ia32_mcg_cap.u64 = rdmsr64(IA32_MCG_CAP);
+		mca_error_bank_count = ia32_mcg_cap.bits.count;
+		mca_control_MSR_present = ia32_mcg_cap.bits.mcg_ctl_p;
+		mca_threshold_status_present = ia32_mcg_cap.bits.mcg_tes_p;
+		if (family == 0x0F) {
+			mca_extended_MSRs_present = ia32_mcg_cap.bits.mcg_ext_p;
+			mca_extended_MSRs_count = ia32_mcg_cap.bits.mcg_ext_cnt;
+		}
+	}
+}
+
+void
+mca_cpu_init(void)
+{
+	unsigned int	i;
+
+	/*
+	 * The first (boot) processor is responsible for discovering the
+	 * machine check architecture present on this machine.
+	 */
+	if (!mca_initialized) {
+		mca_get_availability();
+		mca_initialized = TRUE;
+	}
+
+	if (mca_MCA_present) {
+
+		/* Enable all MCA features */
+		if (mca_control_MSR_present)
+			wrmsr64(IA32_MCG_CTL, IA32_MCG_CTL_ENABLE);
+	
+		switch (cpuid_family()) {
+		case 0x06:
+			/* Enable all but mc0 */
+			for (i = 1; i < mca_error_bank_count; i++)
+				wrmsr64(IA32_MCi_CTL(i),0xFFFFFFFFFFFFFFFFULL); 
+			
+			/* Clear all errors */
+			for (i = 0; i < mca_error_bank_count; i++)
+				wrmsr64(IA32_MCi_STATUS(i), 0ULL);
+			break;
+		case 0x0F:
+			/* Enable all banks */
+			for (i = 0; i < mca_error_bank_count; i++)
+				wrmsr64(IA32_MCi_CTL(i),0xFFFFFFFFFFFFFFFFULL); 
+			
+			/* Clear all errors */
+			for (i = 0; i < mca_error_bank_count; i++)
+				wrmsr64(IA32_MCi_STATUS(i), 0ULL);
+			break;
+		}
+	}
+
+	/* Enable machine check exception handling if available */
+	if (mca_MCE_present) {
+		set_cr4(get_cr4()|CR4_MCE);
+	}
+}
+
+static void mca_dump_64bit_state(void)
+{
+	kdb_printf("Extended Machine Check State:\n");
+	kdb_printf("  IA32_MCG_RAX:    0x%016qx\n", rdmsr64(IA32_MCG_RAX));
+	kdb_printf("  IA32_MCG_RBX:    0x%016qx\n", rdmsr64(IA32_MCG_RBX));
+	kdb_printf("  IA32_MCG_RCX:    0x%016qx\n", rdmsr64(IA32_MCG_RCX));
+	kdb_printf("  IA32_MCG_RDX:    0x%016qx\n", rdmsr64(IA32_MCG_RDX));
+	kdb_printf("  IA32_MCG_RSI:    0x%016qx\n", rdmsr64(IA32_MCG_RSI));
+	kdb_printf("  IA32_MCG_RDI:    0x%016qx\n", rdmsr64(IA32_MCG_RDI));
+	kdb_printf("  IA32_MCG_RBP:    0x%016qx\n", rdmsr64(IA32_MCG_RBP));
+	kdb_printf("  IA32_MCG_RSP:    0x%016qx\n", rdmsr64(IA32_MCG_RSP));
+	kdb_printf("  IA32_MCG_RFLAGS: 0x%016qx\n", rdmsr64(IA32_MCG_RFLAGS));
+	kdb_printf("  IA32_MCG_RIP:    0x%016qx\n", rdmsr64(IA32_MCG_RIP));
+	kdb_printf("  IA32_MCG_MISC:   0x%016qx\n", rdmsr64(IA32_MCG_MISC));
+	kdb_printf("  IA32_MCG_R8:     0x%016qx\n", rdmsr64(IA32_MCG_R8));
+	kdb_printf("  IA32_MCG_R9:     0x%016qx\n", rdmsr64(IA32_MCG_R9));
+	kdb_printf("  IA32_MCG_R10:    0x%016qx\n", rdmsr64(IA32_MCG_R10));
+	kdb_printf("  IA32_MCG_R11:    0x%016qx\n", rdmsr64(IA32_MCG_R11));
+	kdb_printf("  IA32_MCG_R12:    0x%016qx\n", rdmsr64(IA32_MCG_R12));
+	kdb_printf("  IA32_MCG_R13:    0x%016qx\n", rdmsr64(IA32_MCG_R13));
+	kdb_printf("  IA32_MCG_R14:    0x%016qx\n", rdmsr64(IA32_MCG_R14));
+	kdb_printf("  IA32_MCG_R15:    0x%016qx\n", rdmsr64(IA32_MCG_R15));
+}
+
+static uint32_t rdmsr32(uint32_t msr)
+{
+	return (uint32_t) rdmsr64(msr);
+}
+
+static void mca_dump_32bit_state(void)
+{
+	kdb_printf("Extended Machine Check State:\n");
+	kdb_printf("  IA32_MCG_EAX:    0x%08x\n", rdmsr32(IA32_MCG_EAX));
+	kdb_printf("  IA32_MCG_EBX:    0x%08x\n", rdmsr32(IA32_MCG_EBX));
+	kdb_printf("  IA32_MCG_ECX:    0x%08x\n", rdmsr32(IA32_MCG_ECX));
+	kdb_printf("  IA32_MCG_EDX:    0x%08x\n", rdmsr32(IA32_MCG_EDX));
+	kdb_printf("  IA32_MCG_ESI:    0x%08x\n", rdmsr32(IA32_MCG_ESI));
+	kdb_printf("  IA32_MCG_EDI:    0x%08x\n", rdmsr32(IA32_MCG_EDI));
+	kdb_printf("  IA32_MCG_EBP:    0x%08x\n", rdmsr32(IA32_MCG_EBP));
+	kdb_printf("  IA32_MCG_ESP:    0x%08x\n", rdmsr32(IA32_MCG_ESP));
+	kdb_printf("  IA32_MCG_EFLAGS: 0x%08x\n", rdmsr32(IA32_MCG_EFLAGS));
+	kdb_printf("  IA32_MCG_EIP:    0x%08x\n", rdmsr32(IA32_MCG_EIP));
+	kdb_printf("  IA32_MCG_MISC:   0x%08x\n", rdmsr32(IA32_MCG_MISC));
+}
+
+static const char *mca_threshold_status[] = {
+	[THRESHOLD_STATUS_NO_TRACKING]	"No tracking",
+	[THRESHOLD_STATUS_GREEN]	"Green",
+	[THRESHOLD_STATUS_YELLOW]	"Yellow",
+	[THRESHOLD_STATUS_RESERVED]	"Reserved"
+};
+
+static void
+mca_dump_error_banks(void)
+{
+	unsigned int 		i;
+	ia32_mci_status_t	status;
+
+	kdb_printf("MCA error-reporting registers:\n");
+	for (i = 0; i < mca_error_bank_count; i++ ) {
+		status.u64 = rdmsr64(IA32_MCi_STATUS(i));
+		kdb_printf(
+			" IA32_MC%d_STATUS(0x%x): 0x%016qx %svalid\n",
+			i, IA32_MCi_STATUS(i), status.u64,
+			IF(!status.bits.val, "in"));
+		if (!status.bits.val)
+			continue;
+		kdb_printf(
+			"  MCA error code           : 0x%04x\n",
+			status.bits.mca_error);
+		kdb_printf(
+			"  Model specific error code: 0x%04x\n",
+			status.bits.model_specific_error);
+		if (!mca_threshold_status_present) {
+			kdb_printf(
+				"  Other information        : 0x%08x\n",
+				status.bits.other_information);
+		} else {
+			int	threshold = status.bits_tes_p.threshold;
+			kdb_printf(
+				"  Other information        : 0x%08x\n"
+			        "  Threshold-based status   : %s\n",
+				status.bits_tes_p.other_information,
+				(status.bits_tes_p.uc == 0) ?
+					mca_threshold_status[threshold] :
+					"Undefined");
+		}
+		kdb_printf(
+			"  Status bits:\n%s%s%s%s%s%s",
+			IF(status.bits.pcc,   "   Processor context corrupt\n"),
+			IF(status.bits.addrv, "   ADDR register valid\n"),
+			IF(status.bits.miscv, "   MISC register valid\n"),
+			IF(status.bits.en,    "   Error enabled\n"),
+			IF(status.bits.uc,    "   Uncorrected error\n"),
+			IF(status.bits.over,  "   Error overflow\n"));
+		if (status.bits.addrv)
+			kdb_printf(
+				"  IA32_MC%d_ADDR(0x%x): 0x%016qx\n",
+				i, IA32_MCi_ADDR(i), rdmsr64(IA32_MCi_ADDR(i)));
+		if (status.bits.miscv)
+			kdb_printf(
+				"  IA32_MC%d_MISC(0x%x): 0x%016qx\n",
+				i, IA32_MCi_MISC(i), rdmsr64(IA32_MCi_MISC(i)));
+	}
+}
+
+void
+mca_dump(void)
+{
+	ia32_mcg_status_t	status;
+
+	/*
+	 * Report machine-check capabilities:
+	 */
+	kdb_printf(
+		"Machine-check capabilities 0x%016qx:\n", ia32_mcg_cap.u64);
+	kdb_printf(
+		" %d error-reporting banks\n%s%s", mca_error_bank_count,
+		IF(mca_control_MSR_present,
+		   " control MSR present\n"),
+		IF(mca_threshold_status_present,
+		   " threshold-based error status present\n"));
+	if (mca_extended_MSRs_present)
+		kdb_printf(
+			" %d extended MSRs present\n", mca_extended_MSRs_count);
+ 
+	/*
+	 * Report machine-check status:
+	 */
+	status.u64 = rdmsr64(IA32_MCG_STATUS);
+	kdb_printf(
+		"Machine-check status 0x%016qx\n%s%s%s", status.u64,
+		IF(status.bits.ripv, " restart IP valid\n"),
+		IF(status.bits.eipv, " error IP valid\n"),
+		IF(status.bits.mcip, " machine-check in progress\n"));
+
+	/*
+	 * Dump error-reporting registers:
+	 */
+	mca_dump_error_banks();
+
+	/*
+	 * Dump any extended machine state:
+	 */
+	if (mca_extended_MSRs_present) {
+		if (cpu_mode_is64bit())
+			mca_dump_64bit_state();
+		else
+			mca_dump_32bit_state();
+	}
+}
diff --git a/osfmk/i386/machine_check.h b/osfmk/i386/machine_check.h
new file mode 100644
index 000000000..3ec287bbe
--- /dev/null
+++ b/osfmk/i386/machine_check.h
@@ -0,0 +1,196 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifdef KERNEL_PRIVATE
+#ifndef _I386_MACHINE_CHECK_H_
+#define _I386_MACHINE_CHECK_H_
+
+/*
+ * This header defines the machine check architecture for Pentium4 and Xeon.
+ */
+
+/*
+ * Macro BITS(n,m) returns the number of bits between bit(n) and bit(m),
+ * where (n>m). Macro BIT1(n) is cosmetic and returns 1.
+ */
+#define BITS(n,m)	((n)-(m)+1)
+#define BIT1(n)		(1)
+
+/*
+ * IA32 SDM 14.3.1 Machine-Check Global Control MSRs:
+ */
+#define IA32_MCG_CAP		(0x179)
+typedef union {
+     struct {
+	uint64_t	count			:BITS(7,0);
+	uint64_t	mcg_ctl_p		:BIT1(8);
+	uint64_t	mcg_ext_p		:BIT1(9);
+	uint64_t	reserved1		:BIT1(10);
+	uint64_t	mcg_tes_p		:BIT1(11);
+	uint64_t	reserved2		:BITS(15,12);
+	uint64_t	mcg_ext_cnt		:BITS(23,16);
+	uint64_t	reserved3		:BITS(63,24);
+     }		bits;
+     uint64_t	u64;
+} ia32_mcg_cap_t;
+
+#define IA32_MCG_STATUS		(0x17A)
+typedef union {
+     struct {
+	uint64_t	ripv			:BIT1(0);
+	uint64_t	eipv			:BIT1(1);
+	uint64_t	mcip			:BIT1(2);
+	uint64_t	reserved		:BITS(61,3);
+     }		bits;
+     uint64_t	u64;
+} ia32_mcg_status_t;
+
+#define IA32_MCG_CTL		(0x17B)
+typedef uint64_t	ia32_mcg_ctl_t;
+#define IA32_MCG_CTL_ENABLE	(0xFFFFFFFFFFFFFFFFULL)
+#define IA32_MCG_CTL_DISABLE	(0x0ULL)
+
+
+/*
+ * IA32 SDM 14.3.2 Error-Reporting Register Banks:
+ */
+#define IA32_MCi_CTL(i)		(0x400 + 4*(i))
+#define IA32_MCi_STATUS(i)	(0x401 + 4*(i))
+#define IA32_MCi_ADDR(i)	(0x402 + 4*(i))
+#define IA32_MCi_MISC(i)	(0x403 + 4*(i))
+
+#define IA32_MC0_CTL		IA32_MCi_CTL(0)
+#define IA32_MC0_STATUS		IA32_MCi_STATUS(0)
+#define IA32_MC0_ADDR		IA32_MCi_ADDR(0)
+#define IA32_MC0_MISC		IA32_MCi_MISC(0)
+
+#define IA32_MC1_CTL		IA32_MCi_CTL(1)
+#define IA32_MC1_STATUS		IA32_MCi_STATUS(1)
+#define IA32_MC1_ADDR		IA32_MCi_ADDR(1)
+#define IA32_MC1_MISC		IA32_MCi_MISC(1)
+
+#define IA32_MC2_CTL		IA32_MCi_CTL(2)
+#define IA32_MC2_STATUS		IA32_MCi_STATUS(2)
+#define IA32_MC2_ADDR		IA32_MCi_ADDR(2)
+#define IA32_MC2_MISC		IA32_MCi_MISC(2)
+
+#define IA32_MC3_CTL		IA32_MCi_CTL(3)
+#define IA32_MC3_STATUS		IA32_MCi_STATUS(3)
+#define IA32_MC3_ADDR		IA32_MCi_ADDR(3)
+#define IA32_MC3_MISC		IA32_MCi_MISC(3)
+
+#define IA32_MC4_CTL		IA32_MCi_CTL(4)
+#define IA32_MC4_STATUS		IA32_MCi_STATUS(4)
+#define IA32_MC4_ADDR		IA32_MCi_ADDR(4)
+#define IA32_MC4_MISC		IA32_MCi_MISC(4)
+
+typedef uint64_t	ia32_mci_ctl_t;
+#define IA32_MCi_CTL_EE(j)	(0x1ULL << (j))
+#define IA32_MCi_CTL_ENABLE_ALL	(0xFFFFFFFFFFFFFFFFULL)
+
+typedef union {
+     struct {
+	uint64_t	mca_error		:BITS(15,0);
+	uint64_t	model_specific_error	:BITS(31,16);
+	uint64_t	other_information	:BITS(56,32);
+	uint64_t	pcc			:BIT1(57);
+	uint64_t	addrv			:BIT1(58);
+	uint64_t	miscv			:BIT1(59);
+	uint64_t	en			:BIT1(60);
+	uint64_t	uc			:BIT1(61);
+	uint64_t	over			:BIT1(62);
+	uint64_t	val			:BIT1(63);
+     }		bits;
+     struct {		/* Variant if threshold-based error status present: */
+	uint64_t	mca_error		:BITS(15,0);
+	uint64_t	model_specific_error	:BITS(31,16);
+	uint64_t	other_information	:BITS(52,32);
+	uint64_t	threshold		:BITS(54,53);
+	uint64_t	reserved		:BITS(56,55);
+	uint64_t	pcc			:BIT1(57);
+	uint64_t	addrv			:BIT1(58);
+	uint64_t	miscv			:BIT1(59);
+	uint64_t	en			:BIT1(60);
+	uint64_t	uc			:BIT1(61);
+	uint64_t	over			:BIT1(62);
+	uint64_t	val			:BIT1(63);
+     }		bits_tes_p;
+     uint64_t	u64;
+} ia32_mci_status_t;
+
+/* Values for threshold_status if mcg_tes_p == 1 and uc == 0 */
+#define THRESHOLD_STATUS_NO_TRACKING	0
+#define THRESHOLD_STATUS_GREEN		1
+#define THRESHOLD_STATUS_YELLOW		2
+#define THRESHOLD_STATUS_RESERVED	3
+
+typedef uint64_t	ia32_mci_addr_t;
+typedef uint64_t	ia32_mci_misc_t;
+
+
+#define IA32_MCG_EAX		(0x180)
+#define IA32_MCG_EBX		(0x181)
+#define IA32_MCG_ECX		(0x182)
+#define IA32_MCG_EDX		(0x183)
+#define IA32_MCG_ESI		(0x184)
+#define IA32_MCG_EDI		(0x185)
+#define IA32_MCG_EBP		(0x186)
+#define IA32_MCG_ESP		(0x187)
+#define IA32_MCG_EFLAGS		(0x188)
+#define IA32_MCG_EIP		(0x189)
+#define IA32_MCG_MISC		(0x18A)
+
+#define IA32_MCG_RAX		(0x180)
+#define IA32_MCG_RBX		(0x181)
+#define IA32_MCG_RCX		(0x182)
+#define IA32_MCG_RDX		(0x183)
+#define IA32_MCG_RSI		(0x184)
+#define IA32_MCG_RDI		(0x185)
+#define IA32_MCG_RBP		(0x186)
+#define IA32_MCG_RSP		(0x187)
+#define IA32_MCG_RFLAGS		(0x188)
+#define IA32_MCG_RIP		(0x189)
+#define IA32_MCG_MISC		(0x18A)
+#define IA32_MCG_RESERVED1	(0x18B)
+#define IA32_MCG_RESERVED2	(0x18C)
+#define IA32_MCG_RESERVED3	(0x18D)
+#define IA32_MCG_RESERVED4	(0x18E)
+#define IA32_MCG_RESERVED5	(0x18F)
+#define IA32_MCG_R8		(0x190)
+#define IA32_MCG_R9		(0x191)
+#define IA32_MCG_R10		(0x192)
+#define IA32_MCG_R11		(0x193)
+#define IA32_MCG_R12		(0x194)
+#define IA32_MCG_R13		(0x195)
+#define IA32_MCG_R14		(0x196)
+#define IA32_MCG_R15		(0x197)
+
+extern void	mca_cpu_init(void);
+extern void	mca_dump(void);
+
+#endif	/* _I386_MACHINE_CHECK_H_ */
+#endif	/* KERNEL_PRIVATE */
diff --git a/osfmk/i386/machine_cpu.h b/osfmk/i386/machine_cpu.h
index b25a41f11..b3143b07e 100644
--- a/osfmk/i386/machine_cpu.h
+++ b/osfmk/i386/machine_cpu.h
@@ -38,9 +38,12 @@ __BEGIN_DECLS
 void	cpu_machine_init(
 	void);
 
-struct i386_interrupt_state;
 void	cpu_signal_handler(
-	struct i386_interrupt_state *regs);
+	x86_saved_state_t *regs);
+
+void	handle_pending_TLB_flushes(
+	void);
+
 
 kern_return_t cpu_register(
         int *slot_nump);
@@ -48,7 +51,7 @@ __END_DECLS
 
 static inline void cpu_halt(void)
 {
-	asm volatile( "cli; hlt" );
+	asm volatile( "wbinvd; cli; hlt" );
 }
 
 static inline void cpu_pause(void)
diff --git a/osfmk/i386/machine_routines.c b/osfmk/i386/machine_routines.c
index 37e1c4127..825df3b0d 100644
--- a/osfmk/i386/machine_routines.c
+++ b/osfmk/i386/machine_routines.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -41,15 +41,38 @@
 #include <i386/cpu_threads.h>
 #include <i386/pmap.h>
 #include <i386/misc_protos.h>
+#include <i386/pmCPU.h>
+#include <i386/proc_reg.h>
 #include <mach/vm_param.h>
+#if MACH_KDB
+#include <i386/db_machdep.h>
+#include <ddb/db_aout.h>
+#include <ddb/db_access.h>
+#include <ddb/db_sym.h>
+#include <ddb/db_variables.h>
+#include <ddb/db_command.h>
+#include <ddb/db_output.h>
+#include <ddb/db_expr.h>
+#endif
 
 #define MIN(a,b) ((a)<(b)? (a) : (b))
 
+#if DEBUG
+#define DBG(x...)	kprintf("DBG: " x)
+#else
+#define DBG(x...)
+#endif
+
 extern void	initialize_screen(Boot_Video *, unsigned int);
+extern thread_t	Shutdown_context(thread_t thread, void (*doshutdown)(processor_t),processor_t  processor);
 extern void 	wakeup(void *);
+extern unsigned KernelRelocOffset;
 
 static int max_cpus_initialized = 0;
 
+unsigned int	LockTimeOut = 12500000;
+unsigned int	MutexSpin = 0;
+
 #define MAX_CPUS_SET    0x1
 #define MAX_CPUS_WAIT   0x2
 
@@ -60,7 +83,7 @@ vm_offset_t ml_io_map(
 	vm_offset_t phys_addr, 
 	vm_size_t size)
 {
-	return(io_map(phys_addr,size));
+	return(io_map(phys_addr,size,VM_WIMG_IO));
 }
 
 /* boot memory allocation */
@@ -70,6 +93,21 @@ vm_offset_t ml_static_malloc(
 	return((vm_offset_t)NULL);
 }
 
+
+void ml_get_bouncepool_info(vm_offset_t *phys_addr, vm_size_t *size)
+{
+        *phys_addr = bounce_pool_base;
+	*size      = bounce_pool_size;
+}
+
+
+vm_offset_t
+ml_boot_ptovirt(
+	vm_offset_t paddr)
+{
+	return (vm_offset_t)((paddr-KernelRelocOffset) | LINEAR_KERNEL_ADDRESS);
+} 
+
 vm_offset_t
 ml_static_ptovirt(
 	vm_offset_t paddr)
@@ -90,7 +128,7 @@ ml_static_mfree(
 	vm_offset_t vaddr_cur;
 	ppnum_t ppn;
 
-	if (vaddr < VM_MIN_KERNEL_ADDRESS) return;
+//	if (vaddr < VM_MIN_KERNEL_ADDRESS) return;
 
 	assert((vaddr & (PAGE_SIZE-1)) == 0); /* must be page aligned */
 
@@ -106,6 +144,7 @@ ml_static_mfree(
 	}
 }
 
+
 /* virtual to physical on wired pages */
 vm_offset_t ml_vtophys(
 	vm_offset_t vaddr)
@@ -137,10 +176,21 @@ boolean_t ml_set_interrupts_enabled(boolean_t enable)
 
   __asm__ volatile("pushf; popl	%0" :  "=r" (flags));
 
- if (enable)
+  if (enable) {
+	ast_t		*myast;
+
+	myast = ast_pending();
+
+	if ( (get_preemption_level() == 0) &&  (*myast & AST_URGENT) ) {
 	__asm__ volatile("sti");
-  else
+          __asm__ volatile ("int $0xff");
+        } else {
+	  __asm__ volatile ("sti");
+	}
+  }
+  else {
 	__asm__ volatile("cli");
+  }
 
   return (flags & EFL_IF) != 0;
 }
@@ -197,12 +247,6 @@ void ml_install_interrupt_handler(
 	initialize_screen(0, kPEAcquireScreen);
 }
 
-static void
-cpu_idle(void)
-{
-	__asm__ volatile("sti; hlt": : :"memory");
-}
-void (*cpu_idle_handler)(void) = cpu_idle;
 
 void
 machine_idle(void)
@@ -215,12 +259,15 @@ machine_idle(void)
 	 * unless kernel param idlehalt is false and no other thread
 	 * in the same core is active - if so, don't halt so that this
 	 * core doesn't go into a low-power mode.
+	 * For 4/4, we set a null "active cr3" while idle.
 	 */
 	others_active = !atomic_decl_and_test(
 				(long *) &my_core->active_threads, 1);
 	if (idlehalt || others_active) {
 		DBGLOG(cpu_handle, cpu_number(), MP_IDLE);
-		cpu_idle_handler();
+		MARK_CPU_IDLE(cpu_number());
+		machine_idle_cstate();
+		MARK_CPU_ACTIVE(cpu_number());
 		DBGLOG(cpu_handle, cpu_number(), MP_UNIDLE);
 	} else {
 		__asm__ volatile("sti");
@@ -235,6 +282,16 @@ machine_signal_idle(
 	cpu_interrupt(PROCESSOR_DATA(processor, slot_num));
 }
 
+thread_t        
+machine_processor_shutdown(
+	thread_t               thread,
+	void                   (*doshutdown)(processor_t),
+	processor_t    processor)
+{
+        fpu_save_context(thread);
+	return(Shutdown_context(thread, doshutdown, processor));
+}
+
 kern_return_t
 ml_processor_register(
 	cpu_id_t	cpu_id,
@@ -263,7 +320,13 @@ ml_processor_register(
 	if (this_cpu_datap->cpu_console_buf == NULL)
 		goto failed;
 
+	this_cpu_datap->cpu_chud = chudxnu_cpu_alloc(boot_cpu);
+	if (this_cpu_datap->cpu_chud == NULL)
+		goto failed;
+
 	if (!boot_cpu) {
+		this_cpu_datap->cpu_core = cpu_thread_alloc(target_cpu);
+
 		this_cpu_datap->cpu_pmap = pmap_cpu_alloc(boot_cpu);
 		if (this_cpu_datap->cpu_pmap == NULL)
 			goto failed;
@@ -282,6 +345,7 @@ ml_processor_register(
 failed:
 	cpu_processor_free(this_cpu_datap->cpu_processor);
 	pmap_cpu_free(this_cpu_datap->cpu_pmap);
+	chudxnu_cpu_free(this_cpu_datap->cpu_chud);
 	console_cpu_free(this_cpu_datap->cpu_console_buf);
 	return KERN_FAILURE;
 }
@@ -296,11 +360,15 @@ ml_cpu_get_info(ml_cpu_info_t *cpu_infop)
 		return;
  
 	/*
-	 * Are we supporting XMM/SSE/SSE2?
+	 * Are we supporting MMX/SSE/SSE2/SSE3?
 	 * As distinct from whether the cpu has these capabilities.
 	 */
 	os_supports_sse = get_cr4() & CR4_XMM;
-	if ((cpuid_features() & CPUID_FEATURE_SSE2) && os_supports_sse)
+	if ((cpuid_features() & CPUID_FEATURE_MNI) && os_supports_sse)
+		cpu_infop->vector_unit = 6;
+	else if ((cpuid_features() & CPUID_FEATURE_SSE3) && os_supports_sse)
+		cpu_infop->vector_unit = 5;
+	else if ((cpuid_features() & CPUID_FEATURE_SSE2) && os_supports_sse)
 		cpu_infop->vector_unit = 4;
 	else if ((cpuid_features() & CPUID_FEATURE_SSE) && os_supports_sse)
 		cpu_infop->vector_unit = 3;
@@ -325,8 +393,8 @@ ml_cpu_get_info(ml_cpu_info_t *cpu_infop)
         }
 
         if (cpuid_infop->cache_size[L3U] > 0) {
-            cpu_infop->l2_settings = 1;
-            cpu_infop->l2_cache_size = cpuid_infop->cache_size[L3U];
+            cpu_infop->l3_settings = 1;
+            cpu_infop->l3_cache_size = cpuid_infop->cache_size[L3U];
         } else {
             cpu_infop->l3_settings = 0;
             cpu_infop->l3_cache_size = 0xFFFFFFFF;
@@ -371,6 +439,34 @@ ml_get_max_cpus(void)
         return(machine_info.max_cpus);
 }
 
+/*
+ *	Routine:        ml_init_lock_timeout
+ *	Function:
+ */
+void
+ml_init_lock_timeout(void)
+{
+	uint64_t	abstime;
+	uint32_t	mtxspin; 
+
+	/*
+	 * XXX As currently implemented for x86, LockTimeOut should be a
+	 * cycle (tsc) count not an absolute time (nanoseconds) -
+	 * but it's of the right order.
+	 */
+	nanoseconds_to_absolutetime(NSEC_PER_SEC>>2, &abstime);
+	LockTimeOut = (unsigned int)abstime;
+
+	if (PE_parse_boot_arg("mtxspin", &mtxspin)) {
+		if (mtxspin > USEC_PER_SEC>>4)
+			mtxspin =  USEC_PER_SEC>>4;
+		nanoseconds_to_absolutetime(mtxspin*NSEC_PER_USEC, &abstime);
+	} else {
+		nanoseconds_to_absolutetime(10*NSEC_PER_USEC, &abstime);
+	}
+	MutexSpin = (unsigned int)abstime;
+}
+
 /*
  * This is called from the machine-independent routine cpu_up()
  * to perform machine-dependent info updates. Defer to cpu_thread_init().
@@ -432,3 +528,134 @@ current_thread(void)
 {
   return(current_thread_fast());
 }
+
+/*
+ * Set the worst-case time for the C4 to C2 transition.
+ * The maxdelay parameter is in nanoseconds. 
+ */
+ 
+void
+ml_set_maxsnoop(uint32_t maxdelay)
+{
+	C4C2SnoopDelay = maxdelay;	/* Set the transition time */ 
+   	machine_nap_policy();		/* Adjust the current nap state */
+}
+
+
+/*
+ * Get the worst-case time for the C4 to C2 transition.  Returns nanoseconds.
+ */
+
+unsigned
+ml_get_maxsnoop(void)
+{
+	return C4C2SnoopDelay;		/* Set the transition time */
+}
+
+
+uint32_t
+ml_get_maxbusdelay(void)
+{
+    return maxBusDelay;
+}
+
+/*
+ * Set the maximum delay time allowed for snoop on the bus.
+ *
+ * Note that this value will be compared to the amount of time that it takes
+ * to transition from a non-snooping power state (C4) to a snooping state (C2).
+ * If maxBusDelay is less than C4C2SnoopDelay,
+ * we will not enter the lowest power state.
+ */
+ 
+void
+ml_set_maxbusdelay(uint32_t mdelay)
+{
+	maxBusDelay = mdelay;		/* Set the delay */
+	machine_nap_policy();		/* Adjust the current nap state */
+}
+
+
+boolean_t ml_is64bit(void) {
+
+        return (cpu_mode_is64bit());
+}
+
+
+boolean_t ml_thread_is64bit(thread_t thread) {
+  
+        return (thread_is_64bit(thread));
+}
+
+
+boolean_t ml_state_is64bit(void *saved_state) {
+
+	return is_saved_state64(saved_state);
+}
+
+void ml_cpu_set_ldt(int selector)
+{
+	/*
+	 * Avoid loading the LDT
+	 * if we're setting the KERNEL LDT and it's already set.
+	 */
+	if (selector == KERNEL_LDT &&
+	    current_cpu_datap()->cpu_ldt == KERNEL_LDT)
+		return;
+
+	/*
+ 	 * If 64bit this requires a mode switch (and back). 
+	 */
+	if (cpu_mode_is64bit())
+		ml_64bit_lldt(selector);
+	else
+		lldt(selector);
+	current_cpu_datap()->cpu_ldt = selector;	
+}
+
+void ml_fp_setvalid(boolean_t value)
+{
+        fp_setvalid(value);
+}
+
+#if MACH_KDB
+
+/*
+ *	Display the global msrs
+ * *		
+ *	ms
+ */
+void 
+db_msr(__unused db_expr_t addr,
+       __unused int have_addr,
+       __unused db_expr_t count,
+       __unused char *modif)
+{
+
+	uint32_t        i, msrlow, msrhigh;
+
+	/* Try all of the first 4096 msrs */
+	for (i = 0; i < 4096; i++) {
+		if (!rdmsr_carefully(i, &msrlow, &msrhigh)) {
+			db_printf("%08X - %08X.%08X\n", i, msrhigh, msrlow);
+		}
+	}
+
+	/* Try all of the 4096 msrs at 0x0C000000 */
+	for (i = 0; i < 4096; i++) {
+		if (!rdmsr_carefully(0x0C000000 | i, &msrlow, &msrhigh)) {
+			db_printf("%08X - %08X.%08X\n",
+				0x0C000000 | i, msrhigh, msrlow);
+		}
+	}
+
+	/* Try all of the 4096 msrs at 0xC0000000 */
+	for (i = 0; i < 4096; i++) {
+		if (!rdmsr_carefully(0xC0000000 | i, &msrlow, &msrhigh)) {
+			db_printf("%08X - %08X.%08X\n",
+				0xC0000000 | i, msrhigh, msrlow);
+		}
+	}
+}
+
+#endif
diff --git a/osfmk/i386/machine_routines.h b/osfmk/i386/machine_routines.h
index 82b0bc240..8187b3636 100644
--- a/osfmk/i386/machine_routines.h
+++ b/osfmk/i386/machine_routines.h
@@ -42,6 +42,24 @@
 
 __BEGIN_DECLS
 
+/* are we a 64 bit platform ? */
+
+boolean_t ml_is64bit(void);
+
+/* is this a 64bit thread? */
+
+boolean_t ml_thread_is64bit(thread_t);
+
+/* is this a 64bit thread? */
+
+boolean_t ml_state_is64bit(void *);
+
+/* set state of fpu save area for signal handling */
+
+void	ml_fp_setvalid(boolean_t);
+
+void	ml_cpu_set_ldt(int);
+
 /* Interrupt handling */
 
 /* Initialize Interrupts */
@@ -60,6 +78,7 @@ boolean_t ml_at_interrupt_context(void);
 void ml_cause_interrupt(void);
 
 void ml_get_timebase(unsigned long long *timestamp);
+void ml_init_lock_timeout(void); 
 
 /* Type for the Time Base Enable function */
 typedef void (*time_base_enable_t)(cpu_id_t cpu_id, boolean_t enable);
@@ -101,6 +120,12 @@ vm_offset_t
 ml_static_ptovirt(
 	vm_offset_t);
 
+#ifdef	XNU_KERNEL_PRIVATE
+vm_offset_t
+ml_boot_ptovirt(
+	vm_offset_t);
+#endif
+
 /* PCI config cycle probing */
 boolean_t ml_probe_read(
 	vm_offset_t paddr,
@@ -205,6 +230,15 @@ vm_offset_t ml_io_map(
 vm_offset_t ml_static_malloc(
 	vm_size_t size);
 
+
+extern uint32_t	bounce_pool_base;
+extern uint32_t	bounce_pool_size;
+
+void	ml_get_bouncepool_info(
+			       vm_offset_t *phys_addr,
+			       vm_size_t   *size);
+
+
 #endif /* PEXPERT_KERNEL_PRIVATE || MACH_KERNEL_PRIVATE  */
 
 /* Zero bytes starting at a physical address */
@@ -237,6 +271,14 @@ extern int	set_be_bit(void);
 extern int	clr_be_bit(void);
 extern int	be_tracing(void);
 
+extern void ml_set_maxsnoop(uint32_t maxdelay);
+extern unsigned ml_get_maxsnoop(void);
+extern void ml_set_maxbusdelay(uint32_t mdelay);
+extern uint32_t ml_get_maxbusdelay(void);
+extern void ml_hpet_cfg(uint32_t cpu, uint32_t hpetVect);
+
+extern uint64_t tmrCvt(uint64_t time, uint64_t conversion);
+
 #endif /* __APPLE_API_PRIVATE */
 
 __END_DECLS
diff --git a/osfmk/i386/machine_routines_asm.s b/osfmk/i386/machine_routines_asm.s
index 9c8bf0b99..021b59f34 100644
--- a/osfmk/i386/machine_routines_asm.s
+++ b/osfmk/i386/machine_routines_asm.s
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -25,7 +25,25 @@
  * 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
+ 
 #include <i386/asm.h>
+#include <i386/proc_reg.h>
+#include <i386/eflags.h>
+       
+#include <i386/postcode.h>
+#include <i386/apic.h>
+#include <assym.s>
+
+#define	PA(addr)	(addr)
+#define	VA(addr)	(addr)
+	
+/*
+ * GAS won't handle an intersegment jump with a relocatable offset.
+ */
+#define	LJMP(segment,address)	\
+	.byte	0xea		;\
+	.long	address		;\
+	.word	segment
 
 /*
 **      ml_get_timebase()
@@ -37,12 +55,148 @@
 */
 ENTRY(ml_get_timebase)
 
-        movl    S_ARG0, %ecx
+			movl    S_ARG0, %ecx
+			
+			rdtsc
+			
+			movl    %edx, 0(%ecx)
+			movl    %eax, 4(%ecx)
+			
+			ret
+
+
+/*
+ *  	Convert between various timer units 
+ *
+ *		uint64_t tmrCvt(uint64_t time, uint64_t *conversion)
+ *
+ *		This code converts 64-bit time units to other units.
+ *		For example, the TSC is converted to HPET units.
+ *
+ *		Time is a 64-bit integer that is some number of ticks.
+ *		Conversion is 64-bit fixed point number which is composed
+ *		of a 32 bit integer and a 32 bit fraction. 
+ *
+ *		The time ticks are multiplied by the conversion factor.  The
+ *		calculations are done as a 128-bit value but both the high
+ *		and low words are dropped.  The high word is overflow and the
+ *		low word is the fraction part of the result.
+ *
+ *		We return a 64-bit value.
+ *
+ *		Note that we can use this function to multiply 2 conversion factors.
+ *		We do this in order to calculate the multiplier used to convert
+ *		directly between any two units.
+ *
+ */
+
+			.globl	EXT(tmrCvt)
+			.align FALIGN
+
+LEXT(tmrCvt)
+
+			pushl	%ebp					// Save a volatile
+			movl	%esp,%ebp				// Get the parameters - 8
+			pushl	%ebx					// Save a volatile
+			pushl	%esi					// Save a volatile
+			pushl	%edi					// Save a volatile
+
+//			%ebp + 8	- low-order ts
+//			%ebp + 12	- high-order ts
+//			%ebp + 16	- low-order cvt
+//			%ebp + 20	- high-order cvt
+
+			movl	8(%ebp),%eax			// Get low-order ts
+			mull	16(%ebp)				// Multiply by low-order conversion
+			movl	%edx,%edi				// Need to save only the high order part
+			
+			movl	12(%ebp),%eax			// Get the high-order ts
+			mull	16(%ebp)				// Multiply by low-order conversion
+			addl	%eax,%edi				// Add in the overflow from the low x low calculation
+			adcl	$0,%edx					// Add in any overflow to high high part
+			movl	%edx,%esi				// Save high high part
+			
+//			We now have the upper 64 bits of the 96 bit multiply of ts and the low half of cvt
+//			in %esi:%edi
+
+			movl	8(%ebp),%eax			// Get low-order ts
+			mull	20(%ebp)				// Multiply by high-order conversion
+			movl	%eax,%ebx				// Need to save the low order part
+			movl	%edx,%ecx				// Need to save the high order part
+			
+			movl	12(%ebp),%eax			// Get the high-order ts
+			mull	20(%ebp)				// Multiply by high-order conversion
+			
+//			Now have %ecx:%ebx as low part of high low and %edx:%eax as high part of high high
+//			We don't care about the highest word since it is overflow
+			
+			addl	%edi,%ebx				// Add the low words
+			adcl	%ecx,%esi				// Add in the high plus carry from low
+			addl	%eax,%esi				// Add in the rest of the high
+			
+			movl	%ebx,%eax				// Pass back low word
+			movl	%esi,%edx				// and the high word
+			
+			popl	%edi					// Restore a volatile
+			popl	%esi					// Restore a volatile
+			popl	%ebx					// Restore a volatile
+			popl	%ebp					// Restore a volatile
+
+			ret								// Leave...
+
+			.globl	EXT(rtc_nanotime_store)
+			.align	FALIGN
+
+LEXT(rtc_nanotime_store)
+		push	%ebp
+		mov		%esp,%ebp
+
+		mov		32(%ebp),%edx
+
+		mov		8(%ebp),%eax
+		mov		%eax,RNT_TSC_BASE(%edx)
+		mov		12(%ebp),%eax
+		mov		%eax,RNT_TSC_BASE+4(%edx)
+
+		mov		24(%ebp),%eax
+		mov		%eax,RNT_SCALE(%edx)
+
+		mov		28(%ebp),%eax
+		mov		%eax,RNT_SHIFT(%edx)
+
+		mov		16(%ebp),%eax
+		mov		%eax,RNT_NS_BASE(%edx)
+		mov		20(%ebp),%eax
+		mov		%eax,RNT_NS_BASE+4(%edx)
+
+		pop		%ebp
+		ret
+
+			.globl	EXT(rtc_nanotime_load)
+			.align	FALIGN
+
+LEXT(rtc_nanotime_load)
+		push	%ebp
+		mov		%esp,%ebp
+
+		mov		8(%ebp),%ecx
+		mov		12(%ebp),%edx
+
+		mov		RNT_TSC_BASE(%ecx),%eax
+		mov		%eax,RNT_TSC_BASE(%edx)
+		mov		RNT_TSC_BASE+4(%ecx),%eax
+		mov		%eax,RNT_TSC_BASE+4(%edx)
 
-        rdtsc
+		mov		RNT_SCALE(%ecx),%eax
+		mov		%eax,RNT_SCALE(%edx)
 
-        movl    %edx, 0(%ecx)
-        movl    %eax, 4(%ecx)
+		mov		RNT_SHIFT(%ecx),%eax
+		mov		%eax,RNT_SHIFT(%edx)
 
-        ret
+		mov		RNT_NS_BASE(%ecx),%eax
+		mov		%eax,RNT_NS_BASE(%edx)
+		mov		RNT_NS_BASE+4(%ecx),%eax
+		mov		%eax,RNT_NS_BASE+4(%edx)
 
+		pop		%ebp
+		ret
diff --git a/osfmk/i386/misc_protos.h b/osfmk/i386/misc_protos.h
index 407a5fdab..e0f865324 100644
--- a/osfmk/i386/misc_protos.h
+++ b/osfmk/i386/misc_protos.h
@@ -29,11 +29,26 @@
  * @OSF_COPYRIGHT@
  */
 
+#ifndef	_I386_MISC_PROTOS_H_
+#define	_I386_MISC_PROTOS_H_
+
 #include <i386/thread.h>
 
-extern void		i386_preinit(void);
-extern void		i386_init(void);
-extern void		i386_vm_init(unsigned int, struct KernelBootArgs *);
+struct boot_args;
+struct cpu_data;
+
+extern void		i386_init(vm_offset_t);
+extern void		i386_macho_zerofill(void);
+extern void		i386_vm_init(
+				uint64_t		maxmem,
+				boolean_t 		IA32e,
+				struct boot_args	*args);
+extern void		cpu_IA32e_enable(struct cpu_data *);
+extern void		cpu_IA32e_disable(struct cpu_data *);
+extern void		ml_load_desc64(void);
+extern void		ml_64bit_wrmsr64(uint32_t msr, uint64_t value);
+extern void             cpu_window_init(int);
+extern void		ml_64bit_lldt(int);
 
 extern void		machine_startup(void);
 
@@ -42,6 +57,7 @@ extern void		picinit(void);
 extern void		interrupt_processor(
 				int		cpu);
 extern void		mp_probe_cpus(void);
+extern void		panic_io_port_read(void);
 extern void		remote_kdb(void);
 extern void		clear_kdb_intr(void);
 extern void             draw_panic_dialog(void);
@@ -50,6 +66,9 @@ extern void		cpu_shutdown(void);
 extern void		fix_desc(
 				void		* desc,
 				int		num_desc);
+extern void		fix_desc64(
+				void		* desc,
+				int		num_desc);
 extern void		cnpollc(
 				boolean_t	on);
 extern void		form_pic_mask(void);
@@ -63,9 +82,6 @@ extern void		blkclr(
 			       const char	*from,
 			       int		nbytes);
 
-extern void		kdb_kintr(void);
-extern void		kdb_console(void);
-
 extern unsigned int	div_scale(
 				unsigned int	dividend,
 				unsigned int	divisor,
@@ -90,11 +106,13 @@ extern void dcache_incoherent_io_store64(addr64_t pa, unsigned int count);
 extern processor_t	cpu_processor_alloc(boolean_t is_boot_cpu);
 extern void		cpu_processor_free(processor_t proc);
 
+extern void		*chudxnu_cpu_alloc(boolean_t is_boot_cpu);
+extern void		chudxnu_cpu_free(void *);
+
 extern void		sysclk_gettime_interrupts_disabled(
 				mach_timespec_t *cur_time);
 
-
-extern void	rtclock_intr(struct i386_interrupt_state *regs);
+extern void rtc_nanotime_init_commpage(void);
 
 extern void	rtc_sleep_wakeup(void);
 
@@ -104,5 +122,10 @@ extern void	rtc_clock_stepping(
 extern void	rtc_clock_stepped(
 			uint32_t new_frequency,
 			uint32_t old_frequency);
+extern void	rtc_clock_napped(
+			uint64_t);
 
 extern void     x86_lowmem_free(void);
+
+
+#endif	/* _I386_MISC_PROTOS_H_ */
diff --git a/osfmk/i386/mp.c b/osfmk/i386/mp.c
index 8fe553a7b..96b24e27e 100644
--- a/osfmk/i386/mp.c
+++ b/osfmk/i386/mp.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -45,6 +45,7 @@
 #include <kern/cpu_data.h>
 #include <kern/assert.h>
 #include <kern/machine.h>
+#include <kern/pms.h>
 
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>
@@ -57,7 +58,6 @@
 #include <i386/apic.h>
 #include <i386/ipl.h>
 #include <i386/fpu.h>
-#include <i386/pio.h>
 #include <i386/cpuid.h>
 #include <i386/proc_reg.h>
 #include <i386/machine_cpu.h>
@@ -67,6 +67,25 @@
 #include <i386/perfmon.h>
 #include <i386/cpu_threads.h>
 #include <i386/mp_desc.h>
+#include <i386/trap.h>
+#include <i386/machine_routines.h>
+#include <i386/pmCPU.h>
+#include <i386/hpet.h>
+
+#include <chud/chud_xnu.h>
+#include <chud/chud_xnu_private.h>
+
+#include <sys/kdebug.h>
+#if MACH_KDB
+#include <i386/db_machdep.h>
+#include <ddb/db_aout.h>
+#include <ddb/db_access.h>
+#include <ddb/db_sym.h>
+#include <ddb/db_variables.h>
+#include <ddb/db_command.h>
+#include <ddb/db_output.h>
+#include <ddb/db_expr.h>
+#endif
 
 #if	MP_DEBUG
 #define PAUSE		delay(1000000)
@@ -76,24 +95,6 @@
 #define PAUSE
 #endif	/* MP_DEBUG */
 
-/*
- * By default, use high vectors to leave vector space for systems
- * with multiple I/O APIC's. However some systems that boot with
- * local APIC disabled will hang in SMM when vectors greater than
- * 0x5F are used. Those systems are not expected to have I/O APIC
- * so 16 (0x50 - 0x40) vectors for legacy PIC support is perfect.
- */
-#define LAPIC_DEFAULT_INTERRUPT_BASE	0xD0
-#define LAPIC_REDUCED_INTERRUPT_BASE	0x50
-/*
- * Specific lapic interrupts are relative to this base:
- */ 
-#define LAPIC_PERFCNT_INTERRUPT		0xB
-#define LAPIC_TIMER_INTERRUPT		0xC
-#define LAPIC_SPURIOUS_INTERRUPT	0xD	
-#define LAPIC_INTERPROCESSOR_INTERRUPT	0xE
-#define LAPIC_ERROR_INTERRUPT		0xF
-
 /* Initialize lapic_id so cpu_number() works on non SMP systems */
 unsigned long	lapic_id_initdata = 0;
 unsigned long	lapic_id = (unsigned long)&lapic_id_initdata;
@@ -101,6 +102,7 @@ vm_offset_t	lapic_start;
 
 static i386_intr_func_t	lapic_timer_func;
 static i386_intr_func_t	lapic_pmi_func;
+static i386_intr_func_t	lapic_thermal_func;
 
 /* TRUE if local APIC was enabled by the OS not by the BIOS */
 static boolean_t lapic_os_enabled = FALSE;
@@ -110,9 +112,19 @@ int lapic_interrupt_base = LAPIC_DEFAULT_INTERRUPT_BASE;
 
 void 		slave_boot_init(void);
 
+#if MACH_KDB
+static void	mp_kdb_wait(void);
+volatile boolean_t	mp_kdb_trap = FALSE;
+volatile long	mp_kdb_ncpus = 0;
+#endif
+
 static void	mp_kdp_wait(void);
 static void	mp_rendezvous_action(void);
 
+static int	NMIInterruptHandler(void *regs);
+static boolean_t	cpu_signal_pending(int cpu, mp_event_t event);
+static void	cpu_NMI_interrupt(int cpu);
+
 boolean_t 	smp_initialized = FALSE;
 
 decl_simple_lock_data(,mp_kdp_lock);
@@ -120,13 +132,16 @@ decl_simple_lock_data(,mp_kdp_lock);
 decl_mutex_data(static, mp_cpu_boot_lock);
 
 /* Variables needed for MP rendezvous. */
+decl_simple_lock_data(,mp_rv_lock);
 static void		(*mp_rv_setup_func)(void *arg);
 static void		(*mp_rv_action_func)(void *arg);
 static void		(*mp_rv_teardown_func)(void *arg);
 static void		*mp_rv_func_arg;
 static int		mp_rv_ncpus;
-static long		mp_rv_waiters[2];
-decl_simple_lock_data(,mp_rv_lock);
+			/* Cache-aligned barriers: */
+static volatile long	mp_rv_entry    __attribute__((aligned(64)));
+static volatile long	mp_rv_exit     __attribute__((aligned(64)));
+static volatile long	mp_rv_complete __attribute__((aligned(64)));
 
 int		lapic_to_cpu[MAX_CPUS];
 int		cpu_to_lapic[MAX_CPUS];
@@ -149,6 +164,24 @@ lapic_cpu_map(int apic_id, int cpu)
 	lapic_to_cpu[apic_id] = cpu;
 }
 
+/*
+ * Retrieve the local apic ID a cpu.
+ *
+ * Returns the local apic ID for the given processor.
+ * If the processor does not exist or apic not configured, returns -1.
+ */
+
+uint32_t
+ml_get_apicid(uint32_t cpu)
+{
+	if(cpu >= (uint32_t)MAX_CPUS)
+		return 0xFFFFFFFF;	/* Return -1 if cpu too big */
+	
+	/* Return the apic ID (or -1 if not configured) */
+	return (uint32_t)cpu_to_lapic[cpu];
+
+}
+
 #ifdef MP_DEBUG
 static void
 lapic_cpu_map_dump(void)
@@ -175,18 +208,6 @@ lapic_cpu_map_dump(void)
 #define LAPIC_DUMP()
 #endif /* MP_DEBUG */
 
-#define LAPIC_REG(reg) \
-	(*((volatile int *)(lapic_start + LAPIC_##reg)))
-#define LAPIC_REG_OFFSET(reg,off) \
-	(*((volatile int *)(lapic_start + LAPIC_##reg + (off))))
-
-#define LAPIC_VECTOR(src) \
-	(lapic_interrupt_base + LAPIC_##src##_INTERRUPT)
-
-#define LAPIC_ISR_IS_SET(base,src) \
-	(LAPIC_REG_OFFSET(ISR_BASE,((base+LAPIC_##src##_INTERRUPT)/32)*0x10) & \
-		(1 <<((base + LAPIC_##src##_INTERRUPT)%32)))
-
 #if GPROF
 /*
  * Initialize dummy structs for profiling. These aren't used but
@@ -206,8 +227,6 @@ struct profile_vars *_profile_vars_cpus[MAX_CPUS] = { &_profile_vars };
 #define GPROF_INIT()
 #endif /* GPROF */
 
-extern void	master_up(void);
-
 void
 smp_init(void)
 {
@@ -241,17 +260,22 @@ smp_init(void)
 
 	/* Establish a map to the local apic */
 	lapic_start = vm_map_min(kernel_map);
-	result = vm_map_find_space(kernel_map, &lapic_start,
-				   round_page(LAPIC_SIZE), 0, &entry);
+	result = vm_map_find_space(kernel_map,
+				   (vm_map_address_t *) &lapic_start,
+				   round_page(LAPIC_SIZE), 0,
+				   VM_MAKE_TAG(VM_MEMORY_IOKIT), &entry);
 	if (result != KERN_SUCCESS) {
 		panic("smp_init: vm_map_find_entry FAILED (err=%d)", result);
 	}
 	vm_map_unlock(kernel_map);
+/* Map in the local APIC non-cacheable, as recommended by Intel
+ * in section 8.4.1 of the "System Programming Guide".
+ */
 	pmap_enter(pmap_kernel(),
 			lapic_start,
 			(ppnum_t) i386_btop(lapic_base),
-		   	VM_PROT_READ|VM_PROT_WRITE,
-			VM_WIMG_USE_DEFAULT,
+			VM_PROT_READ|VM_PROT_WRITE,
+			VM_WIMG_IO,
 			TRUE);
 	lapic_id = (unsigned long)(lapic_start + LAPIC_ID);
 
@@ -262,19 +286,16 @@ smp_init(void)
 	/* Set up the lapic_id <-> cpu_number map and add this boot processor */
 	lapic_cpu_map_init();
 	lapic_cpu_map((LAPIC_REG(ID)>>LAPIC_ID_SHIFT)&LAPIC_ID_MASK, 0);
+	kprintf("Boot cpu local APIC id 0x%x\n", cpu_to_lapic[0]);
 
 	lapic_init();
 
 	cpu_thread_init();
 
-	if (pmc_init() != KERN_SUCCESS)
-		printf("Performance counters not available\n");
-
 	GPROF_INIT();
 	DBGLOG_CPU_INIT(master_cpu);
 
 	slave_boot_init();
-	master_up();
 
 	smp_initialized = TRUE;
 
@@ -342,6 +363,11 @@ lapic_dump(void)
 		DM[(LAPIC_REG(LVT_PERFCNT)>>LAPIC_LVT_DM_SHIFT)&LAPIC_LVT_DM_MASK],
 		(LAPIC_REG(LVT_PERFCNT)&LAPIC_LVT_DS_PENDING)?"SendPending":"Idle",
 		BOOL(LAPIC_REG(LVT_PERFCNT)&LAPIC_LVT_MASKED));
+	kprintf("LVT_THERMAL: Vector 0x%02x [%s] %s %cmasked\n",
+		LAPIC_REG(LVT_THERMAL)&LAPIC_LVT_VECTOR_MASK,
+		DM[(LAPIC_REG(LVT_THERMAL)>>LAPIC_LVT_DM_SHIFT)&LAPIC_LVT_DM_MASK],
+		(LAPIC_REG(LVT_THERMAL)&LAPIC_LVT_DS_PENDING)?"SendPending":"Idle",
+		BOOL(LAPIC_REG(LVT_THERMAL)&LAPIC_LVT_MASKED));
 	kprintf("LVT_LINT0:   Vector 0x%02x [%s][%s][%s] %s %cmasked\n",
 		LAPIC_REG(LVT_LINT0)&LAPIC_LVT_VECTOR_MASK,
 		DM[(LAPIC_REG(LVT_LINT0)>>LAPIC_LVT_DM_SHIFT)&LAPIC_LVT_DM_MASK],
@@ -379,6 +405,26 @@ lapic_dump(void)
 	kprintf("\n");
 }
 
+#if MACH_KDB
+/*
+ *	Displays apic junk
+ *
+ *	da
+ */
+void 
+db_apic(__unused db_expr_t addr,
+	__unused int have_addr,
+	__unused db_expr_t count,
+	__unused char *modif)
+{
+
+	lapic_dump();
+
+	return;
+}
+
+#endif
+
 boolean_t
 lapic_probe(void)
 {
@@ -403,7 +449,7 @@ lapic_probe(void)
 		/*
 		 * Re-initialize cpu features info and re-check.
 		 */
-		set_cpu_model();
+		cpuid_set_info();
 		if (cpuid_features() & CPUID_FEATURE_APIC) {
 			printf("Local APIC discovered and enabled\n");
 			lapic_os_enabled = TRUE;
@@ -451,7 +497,7 @@ lapic_shutdown(void)
 	rdmsr(MSR_IA32_APIC_BASE, lo, hi);
 	lo &= ~MSR_IA32_APIC_BASE_ENABLE;
 	wrmsr(MSR_IA32_APIC_BASE, lo, hi);
-	set_cpu_model();
+	cpuid_set_info();
 
 	mp_enable_preemption();
 }
@@ -484,10 +530,12 @@ lapic_init(void)
 	/* Perfmon: unmasked */
 	LAPIC_REG(LVT_PERFCNT) = LAPIC_VECTOR(PERFCNT);
 
+	/* Thermal: unmasked */
+	LAPIC_REG(LVT_THERMAL) = LAPIC_VECTOR(THERMAL);
+
 	lapic_esr_clear();
 
 	LAPIC_REG(LVT_ERROR) = LAPIC_VECTOR(ERROR);
-
 }
 
 void
@@ -545,6 +593,12 @@ lapic_set_pmi_func(i386_intr_func_t func)
 	lapic_pmi_func = func;
 }
 
+void
+lapic_set_thermal_func(i386_intr_func_t func)
+{
+        lapic_thermal_func = func;
+}
+
 static inline void
 _lapic_end_of_interrupt(void)
 {
@@ -558,42 +612,77 @@ lapic_end_of_interrupt(void)
 }
 
 int
-lapic_interrupt(int interrupt, void *state)
+lapic_interrupt(int interrupt, x86_saved_state_t *state)
 {
+	int	retval = 0;
+
+	/* Did we just field an interruption for the HPET comparator? */
+	if(current_cpu_datap()->cpu_pmHpetVec == ((uint32_t)interrupt - 0x40)) {
+		/* Yes, go handle it... */
+		retval = HPETInterrupt();
+		/* Was it really handled? */
+		if(retval) {
+			/* If so, EOI the 'rupt */
+			_lapic_end_of_interrupt();
+			/*
+			 * and then leave,
+			 * indicating that this has been handled
+			 */
+			return 1;
+		}
+	}
+
 	interrupt -= lapic_interrupt_base;
-	if (interrupt < 0)
-		return 0;
+	if (interrupt < 0) {
+		if (interrupt == (LAPIC_NMI_INTERRUPT - lapic_interrupt_base)) {
+			retval = NMIInterruptHandler(state);
+			_lapic_end_of_interrupt();
+			return retval;
+		}
+		else
+			return 0;
+	}
 
 	switch(interrupt) {
 	case LAPIC_PERFCNT_INTERRUPT:
 		if (lapic_pmi_func != NULL)
-			(*lapic_pmi_func)(
-				(struct i386_interrupt_state *) state);
+			(*lapic_pmi_func)(NULL);
 		/* Clear interrupt masked */
 		LAPIC_REG(LVT_PERFCNT) = LAPIC_VECTOR(PERFCNT);
 		_lapic_end_of_interrupt();
-		return 1;
+		retval = 1;
+		break;
 	case LAPIC_TIMER_INTERRUPT:
 		_lapic_end_of_interrupt();
 		if (lapic_timer_func != NULL)
-			(*lapic_timer_func)(
-				(struct i386_interrupt_state *) state);
-		return 1;
+			(*lapic_timer_func)(state);
+		retval = 1;
+		break;
+	case LAPIC_THERMAL_INTERRUPT:
+		if (lapic_thermal_func != NULL)
+			(*lapic_thermal_func)(NULL);
+		_lapic_end_of_interrupt();
+		retval = 1;
+		break;
 	case LAPIC_ERROR_INTERRUPT:
 		lapic_dump();
 		panic("Local APIC error\n");
 		_lapic_end_of_interrupt();
-		return 1;
+		retval = 1;
+		break;
 	case LAPIC_SPURIOUS_INTERRUPT:
 		kprintf("SPIV\n");
 		/* No EOI required here */
-		return 1;
+		retval = 1;
+		break;
 	case LAPIC_INTERPROCESSOR_INTERRUPT:
-		cpu_signal_handler((struct i386_interrupt_state *) state);
 		_lapic_end_of_interrupt();
-		return 1;
+		cpu_signal_handler(state);
+		retval = 1;
+		break;
 	}
-	return 0;
+
+	return retval;
 }
 
 void
@@ -644,8 +733,14 @@ intel_startCPU(
 	DBG("intel_startCPU(%d) lapic_id=%d\n", slot_num, lapic);
 	DBG("IdlePTD(%p): 0x%x\n", &IdlePTD, (int) IdlePTD);
 
-	/* Initialize (or re-initialize) the descriptor tables for this cpu. */
-	mp_desc_init(cpu_datap(slot_num), FALSE);
+	/*
+	 * Initialize (or re-initialize) the descriptor tables for this cpu.
+	 * Propagate processor mode to slave.
+	 */
+	if (cpu_mode_is64bit())
+		cpu_desc_init64(cpu_datap(slot_num), FALSE);
+	else
+		cpu_desc_init(cpu_datap(slot_num), FALSE);
 
 	/* Serialize use of the slave boot stack. */
 	mutex_lock(&mp_cpu_boot_lock);
@@ -683,13 +778,13 @@ intel_startCPU(
 	mutex_unlock(&mp_cpu_boot_lock);
 
 	if (!cpu_datap(slot_num)->cpu_running) {
-		DBG("Failed to start CPU %02d\n", slot_num);
+		kprintf("Failed to start CPU %02d\n", slot_num);
 		printf("Failed to start CPU %02d, rebooting...\n", slot_num);
 		delay(1000000);
 		cpu_shutdown();
 		return KERN_SUCCESS;
 	} else {
-		DBG("Started CPU %02d\n", slot_num);
+		kprintf("Started cpu %d (lapic id %p)\n", slot_num, lapic);
 		printf("Started CPU %02d\n", slot_num);
 		return KERN_SUCCESS;
 	}
@@ -697,7 +792,7 @@ intel_startCPU(
 
 extern char	slave_boot_base[];
 extern char	slave_boot_end[];
-extern void	pstart(void);
+extern void	slave_pstart(void);
 
 void
 slave_boot_init(void)
@@ -715,7 +810,7 @@ slave_boot_init(void)
 	 * The slave boot code is responsible for switching to protected
 	 * mode and then jumping to the common startup, _start().
 	 */
-	bcopy_phys((addr64_t) kvtophys((vm_offset_t) slave_boot_base),
+	bcopy_phys(kvtophys((vm_offset_t) slave_boot_base),
 		   (addr64_t) MP_BOOT,
 		   slave_boot_end-slave_boot_base);
 
@@ -730,9 +825,9 @@ slave_boot_init(void)
 	 * common startup entry.
 	 */
 	DBG("writing 0x%x at phys 0x%x\n",
-		kvtophys((vm_offset_t) &pstart), MP_MACH_START+MP_BOOT);
+		kvtophys((vm_offset_t) &slave_pstart), MP_MACH_START+MP_BOOT);
 	ml_phys_write_word(MP_MACH_START+MP_BOOT,
-			   kvtophys((vm_offset_t) &pstart));
+			   (unsigned int)kvtophys((vm_offset_t) &slave_pstart));
 	
 	/* Flush caches */
 	__asm__("wbinvd");
@@ -747,7 +842,7 @@ MP_EVENT_NAME_DECL();
 #endif	/* MP_DEBUG */
 
 void
-cpu_signal_handler(__unused struct i386_interrupt_state *regs)
+cpu_signal_handler(x86_saved_state_t *regs)
 {
 	int		my_cpu;
 	volatile int	*my_word;
@@ -763,12 +858,19 @@ cpu_signal_handler(__unused struct i386_interrupt_state *regs)
 	do {
 #if	MACH_KDB && MACH_ASSERT
 		if (i-- <= 0)
-		    Debugger("cpu_signal_handler");
+		    Debugger("cpu_signal_handler: signals did not clear");
 #endif	/* MACH_KDB && MACH_ASSERT */
 #if	MACH_KDP
 		if (i_bit(MP_KDP, my_word)) {
 			DBGLOG(cpu_handle,my_cpu,MP_KDP);
 			i_bit_clear(MP_KDP, my_word);
+/* Ensure that the i386_kernel_state at the base of the
+ * current thread's stack (if any) is synchronized with the
+ * context at the moment of the interrupt, to facilitate
+ * access through the debugger.
+ * XXX 64-bit state?
+ */
+			sync_iss_to_iks(saved_state32(regs));
 			mp_kdp_wait();
 		} else
 #endif	/* MACH_KDP */
@@ -782,16 +884,20 @@ cpu_signal_handler(__unused struct i386_interrupt_state *regs)
 			ast_check(cpu_to_processor(my_cpu));
 #if	MACH_KDB
 		} else if (i_bit(MP_KDB, my_word)) {
-			extern kdb_is_slave[];
 
 			i_bit_clear(MP_KDB, my_word);
-			kdb_is_slave[my_cpu]++;
-			kdb_kintr();
+			current_cpu_datap()->cpu_kdb_is_slave++;
+			mp_kdb_wait();
+			current_cpu_datap()->cpu_kdb_is_slave--;
 #endif	/* MACH_KDB */
 		} else if (i_bit(MP_RENDEZVOUS, my_word)) {
 			DBGLOG(cpu_handle,my_cpu,MP_RENDEZVOUS);
 			i_bit_clear(MP_RENDEZVOUS, my_word);
 			mp_rendezvous_action();
+		} else if (i_bit(MP_CHUD, my_word)) {
+			DBGLOG(cpu_handle,my_cpu,MP_CHUD);
+			i_bit_clear(MP_CHUD, my_word);
+			chudxnu_cpu_signal_handler();
 		}
 	} while (*my_word);
 
@@ -799,19 +905,49 @@ cpu_signal_handler(__unused struct i386_interrupt_state *regs)
 
 }
 
+
+/* We want this to show up in backtraces, so mark it noinline
+ */
+static int __attribute__((noinline))
+NMIInterruptHandler(void *regs)
+{
+	boolean_t state = ml_set_interrupts_enabled(FALSE);
+	sync_iss_to_iks_unconditionally(regs);
+	mp_kdp_wait();
+	(void) ml_set_interrupts_enabled(state);
+	return 1;
+}
+
 #ifdef	MP_DEBUG
 extern int	max_lock_loops;
 #endif	/* MP_DEBUG */
+
+int trappedalready = 0;	/* (BRINGUP */
+
 void
 cpu_interrupt(int cpu)
 {
 	boolean_t	state;
+	
+	if(cpu_datap(cpu)->cpu_signals & 6) {	/* (BRINGUP) */
+		kprintf("cpu_interrupt: sending enter debugger signal (%08X) to cpu %d\n", cpu_datap(cpu)->cpu_signals, cpu);
+	}
 
 	if (smp_initialized) {
 
+#if MACH_KDB
+//		if(!trappedalready && (cpu_datap(cpu)->cpu_signals & 6)) {	/* (BRINGUP) */
+//			if(kdb_cpu != cpu_number()) {
+//				trappedalready = 1;
+//				panic("cpu_interrupt: sending enter debugger signal (%08X) to cpu %d and I do not own debugger, owner = %08X\n", 
+//					cpu_datap(cpu)->cpu_signals, cpu, kdb_cpu);
+//			}
+//		}
+#endif
+
 		/* Wait for previous interrupt to be delivered... */
 #ifdef	MP_DEBUG
-		int	pending_busy_count = 0;
+		int     pending_busy_count = 0;
 		while (LAPIC_REG(ICR) & LAPIC_ICR_DS_PENDING) {
 			if (++pending_busy_count > max_lock_loops)
 				panic("cpus_interrupt() deadlock\n");
@@ -831,18 +967,43 @@ cpu_interrupt(int cpu)
 
 }
 
+/*
+ * Send a true NMI via the local APIC to the specified CPU.
+ */
+static void
+cpu_NMI_interrupt(int cpu)
+{
+	boolean_t	state;
+
+	if (smp_initialized) {
+		state = ml_set_interrupts_enabled(FALSE);
+		LAPIC_REG(ICRD) =
+			cpu_to_lapic[cpu] << LAPIC_ICRD_DEST_SHIFT;
+/* The vector is ignored in this case, the other CPU will come in on the
+ * NMI vector.
+ */
+		LAPIC_REG(ICR)  =
+			LAPIC_VECTOR(INTERPROCESSOR) | LAPIC_ICR_DM_NMI;
+		(void) ml_set_interrupts_enabled(state);
+	}
+
+}
+
 void
 i386_signal_cpu(int cpu, mp_event_t event, mp_sync_t mode)
 {
 	volatile int	*signals = &cpu_datap(cpu)->cpu_signals;
 	uint64_t	tsc_timeout;
-	
 
+	
 	if (!cpu_datap(cpu)->cpu_running)
 		return;
 
-	DBGLOG(cpu_signal, cpu, event);
+	if (event == MP_TLB_FLUSH)
+	        KERNEL_DEBUG(0xef800020 | DBG_FUNC_START, cpu, 0, 0, 0, 0);
 
+	DBGLOG(cpu_signal, cpu, event);
+	
 	i_bit_set(event, signals);
 	cpu_interrupt(cpu);
 	if (mode == SYNC) {
@@ -857,6 +1018,8 @@ i386_signal_cpu(int cpu, mp_event_t event, mp_sync_t mode)
 			goto again;
 		}
 	}
+	if (event == MP_TLB_FLUSH)
+	        KERNEL_DEBUG(0xef800020 | DBG_FUNC_END, cpu, 0, 0, 0, 0);
 }
 
 void
@@ -907,19 +1070,28 @@ mp_rendezvous_action(void)
 	if (mp_rv_setup_func != NULL)
 		mp_rv_setup_func(mp_rv_func_arg);
 	/* spin on entry rendezvous */
-	atomic_incl(&mp_rv_waiters[0], 1);
-	while (*((volatile long *) &mp_rv_waiters[0]) < mp_rv_ncpus)
+	atomic_incl(&mp_rv_entry, 1);
+	while (mp_rv_entry < mp_rv_ncpus) {
+		boolean_t intr = ml_set_interrupts_enabled(FALSE);
+		/* poll for pesky tlb flushes */
+		handle_pending_TLB_flushes();
+		ml_set_interrupts_enabled(intr);
 		cpu_pause();
+	}
 	/* action function */
 	if (mp_rv_action_func != NULL)
 		mp_rv_action_func(mp_rv_func_arg);
 	/* spin on exit rendezvous */
-	atomic_incl(&mp_rv_waiters[1], 1);
-	while (*((volatile long *) &mp_rv_waiters[1]) < mp_rv_ncpus)
+	atomic_incl(&mp_rv_exit, 1);
+	while (mp_rv_exit < mp_rv_ncpus)
 		cpu_pause();
+
 	/* teardown function */
 	if (mp_rv_teardown_func != NULL)
 		mp_rv_teardown_func(mp_rv_func_arg);
+
+	/* Bump completion count */
+	atomic_incl(&mp_rv_complete, 1);
 }
 
 void
@@ -948,26 +1120,86 @@ mp_rendezvous(void (*setup_func)(void *),
 	mp_rv_teardown_func = teardown_func;
 	mp_rv_func_arg = arg;
 
-	mp_rv_waiters[0] = 0;		/* entry rendezvous count */
-	mp_rv_waiters[1] = 0;		/* exit  rendezvous count */
-	mp_rv_ncpus = i386_active_cpus();
+	mp_rv_entry    = 0;
+	mp_rv_exit     = 0;
+	mp_rv_complete = 0;
 
 	/*
 	 * signal other processors, which will call mp_rendezvous_action()
-	 * with interrupts disabled
 	 */
+	mp_rv_ncpus = i386_active_cpus();
 	i386_signal_cpus(MP_RENDEZVOUS, ASYNC);
 
 	/* call executor function on this cpu */
 	mp_rendezvous_action();
 
+	/*
+	 * Spin for everyone to complete.
+	 * This is necessary to ensure that all processors have proceeded
+	 * from the exit barrier before we release the rendezvous structure.
+	 */
+	while (mp_rv_complete < mp_rv_ncpus) {
+		cpu_pause();
+	}
+	
 	/* release lock */
 	simple_unlock(&mp_rv_lock);
 }
 
+void
+mp_rendezvous_break_lock(void)
+{
+	simple_lock_init(&mp_rv_lock, 0);
+}
+
+static void
+setup_disable_intrs(__unused void * param_not_used)
+{
+	/* disable interrupts before the first barrier */
+	boolean_t intr = ml_set_interrupts_enabled(FALSE);
+
+	current_cpu_datap()->cpu_iflag = intr;
+	DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__);
+}
+
+static void
+teardown_restore_intrs(__unused void * param_not_used)
+{
+	/* restore interrupt flag following MTRR changes */
+	ml_set_interrupts_enabled(current_cpu_datap()->cpu_iflag);
+	DBG("CPU%d: %s\n", get_cpu_number(), __FUNCTION__);
+}
+
+/*
+ * A wrapper to mp_rendezvous() to call action_func() with interrupts disabled.
+ * This is exported for use by kexts.
+ */
+void
+mp_rendezvous_no_intrs(
+	      void (*action_func)(void *),
+	      void *arg)
+{
+	mp_rendezvous(setup_disable_intrs,
+		      action_func,
+		      teardown_restore_intrs,
+		      arg);	
+}
+
+void
+handle_pending_TLB_flushes(void)
+{
+	volatile int	*my_word = &current_cpu_datap()->cpu_signals;
+
+        if (i_bit(MP_TLB_FLUSH, my_word)) {
+	        DBGLOG(cpu_handle, cpu_number(), MP_TLB_FLUSH);
+		i_bit_clear(MP_TLB_FLUSH, my_word);
+		pmap_update_interrupt();
+	}
+}
+
 #if	MACH_KDP
 volatile boolean_t	mp_kdp_trap = FALSE;
-long			mp_kdp_ncpus;
+volatile long		mp_kdp_ncpus;
 boolean_t		mp_kdp_state;
 
 
@@ -988,6 +1220,7 @@ mp_kdp_enter(void)
 	 */
 	mp_kdp_state = ml_set_interrupts_enabled(FALSE);
 	simple_lock(&mp_kdp_lock);
+
 	while (mp_kdp_trap) {
 		simple_unlock(&mp_kdp_lock);
 		DBG("mp_kdp_enter() race lost\n");
@@ -998,53 +1231,102 @@ mp_kdp_enter(void)
 	mp_kdp_trap = TRUE;
 	simple_unlock(&mp_kdp_lock);
 
-	/* Deliver a nudge to other cpus, counting how many */
+	/*
+	 * Deliver a nudge to other cpus, counting how many
+	 */
 	DBG("mp_kdp_enter() signaling other processors\n");
 	for (ncpus = 1, cpu = 0; cpu < real_ncpus; cpu++) {
 		if (cpu == my_cpu || !cpu_datap(cpu)->cpu_running)
 			continue;
 		ncpus++;
-		i386_signal_cpu(cpu, MP_KDP, ASYNC); 
+		i386_signal_cpu(cpu, MP_KDP, ASYNC);
 	}
-
-	/* Wait other processors to spin. */
+	/*
+	 * Wait other processors to synchronize
+	 */
 	DBG("mp_kdp_enter() waiting for (%d) processors to suspend\n", ncpus);
-	tsc_timeout = rdtsc64() + (1000*1000*1000);
-	while (*((volatile unsigned int *) &mp_kdp_ncpus) != ncpus
-		&& rdtsc64() < tsc_timeout) {
+
+	tsc_timeout = rdtsc64() + (ncpus * 100 * 1000 * 1000);
+
+	while (mp_kdp_ncpus != ncpus && rdtsc64() < tsc_timeout) {
+	        /*
+		 * A TLB shootdown request may be pending... this would
+		 * result in the requesting processor waiting in
+		 * PMAP_UPDATE_TLBS() until this processor deals with it.
+		 * Process it, so it can now enter mp_kdp_wait()
+		 */
+	        handle_pending_TLB_flushes();
 		cpu_pause();
 	}
+/* If we've timed out, and some processor(s) are still unresponsive,
+ * interrupt them with an NMI via the local APIC.
+ */
+	if (mp_kdp_ncpus != ncpus) {
+		for (cpu = 0; cpu < real_ncpus; cpu++) {
+			if (cpu == my_cpu || !cpu_datap(cpu)->cpu_running)
+				continue;
+			if (cpu_signal_pending(cpu, MP_KDP))
+			    cpu_NMI_interrupt(cpu);
+		}
+	}
+
 	DBG("mp_kdp_enter() %d processors done %s\n",
 		mp_kdp_ncpus, (mp_kdp_ncpus == ncpus) ? "OK" : "timed out");
+	
 	postcode(MP_KDP_ENTER);
 }
 
+static boolean_t
+cpu_signal_pending(int cpu, mp_event_t event)
+{
+	volatile int	*signals = &cpu_datap(cpu)->cpu_signals;
+	boolean_t retval = FALSE;
+
+	if (i_bit(event, signals))
+		retval = TRUE;
+	return retval;
+}
+
 static void
 mp_kdp_wait(void)
 {
-	boolean_t	state;
-
-	state = ml_set_interrupts_enabled(TRUE);
 	DBG("mp_kdp_wait()\n");
+
+	panic_io_port_read();
+
 	atomic_incl(&mp_kdp_ncpus, 1);
 	while (mp_kdp_trap) {
+	        /*
+		 * a TLB shootdown request may be pending... this would result in the requesting
+		 * processor waiting in PMAP_UPDATE_TLBS() until this processor deals with it.
+		 * Process it, so it can now enter mp_kdp_wait()
+		 */
+	        handle_pending_TLB_flushes();
+
 		cpu_pause();
 	}
-	atomic_decl(&mp_kdp_ncpus, 1);
+	atomic_decl((volatile long *)&mp_kdp_ncpus, 1);
 	DBG("mp_kdp_wait() done\n");
-	(void) ml_set_interrupts_enabled(state);
 }
 
 void
 mp_kdp_exit(void)
 {
 	DBG("mp_kdp_exit()\n");
-	atomic_decl(&mp_kdp_ncpus, 1);
+	atomic_decl((volatile long *)&mp_kdp_ncpus, 1);
 	mp_kdp_trap = FALSE;
+	__asm__ volatile("mfence");
 
 	/* Wait other processors to stop spinning. XXX needs timeout */
 	DBG("mp_kdp_exit() waiting for processors to resume\n");
-	while (*((volatile long *) &mp_kdp_ncpus) > 0) {
+	while (mp_kdp_ncpus > 0) {
+	        /*
+		 * a TLB shootdown request may be pending... this would result in the requesting
+		 * processor waiting in PMAP_UPDATE_TLBS() until this processor deals with it.
+		 * Process it, so it can now enter mp_kdp_wait()
+		 */
+	        handle_pending_TLB_flushes();
+
 		cpu_pause();
 	}
 	DBG("mp_kdp_exit() done\n");
@@ -1071,6 +1353,7 @@ cause_ast_check(
 	}
 }
 
+#if MACH_KDB
 /*
  * invoke kdb on slave processors 
  */
@@ -1080,14 +1363,55 @@ remote_kdb(void)
 {
 	unsigned int	my_cpu = cpu_number();
 	unsigned int	cpu;
+	int kdb_ncpus;
+	uint64_t tsc_timeout = 0;
 	
-	mp_disable_preemption();
-	for (cpu = 0; cpu < real_ncpus; cpu++) {
+	mp_kdb_trap = TRUE;
+	mp_kdb_ncpus = 1;
+	for (kdb_ncpus = 1, cpu = 0; cpu < real_ncpus; cpu++) {
 		if (cpu == my_cpu || !cpu_datap(cpu)->cpu_running)
 			continue;
-		i386_signal_cpu(cpu, MP_KDB, SYNC);
+		kdb_ncpus++;
+		i386_signal_cpu(cpu, MP_KDB, ASYNC);
 	}
-	mp_enable_preemption();
+	DBG("remote_kdb() waiting for (%d) processors to suspend\n",kdb_ncpus);
+
+	tsc_timeout = rdtsc64() + (kdb_ncpus * 100 * 1000 * 1000);
+
+	while (mp_kdb_ncpus != kdb_ncpus && rdtsc64() < tsc_timeout) {
+	        /*
+		 * a TLB shootdown request may be pending... this would result in the requesting
+		 * processor waiting in PMAP_UPDATE_TLBS() until this processor deals with it.
+		 * Process it, so it can now enter mp_kdp_wait()
+		 */
+	        handle_pending_TLB_flushes();
+
+		cpu_pause();
+	}
+	DBG("mp_kdp_enter() %d processors done %s\n",
+		mp_kdb_ncpus, (mp_kdb_ncpus == kdb_ncpus) ? "OK" : "timed out");
+}
+
+static void
+mp_kdb_wait(void)
+{
+	DBG("mp_kdb_wait()\n");
+
+	panic_io_port_read();
+
+	atomic_incl(&mp_kdb_ncpus, 1);
+	while (mp_kdb_trap) {
+	        /*
+		 * a TLB shootdown request may be pending... this would result in the requesting
+		 * processor waiting in PMAP_UPDATE_TLBS() until this processor deals with it.
+		 * Process it, so it can now enter mp_kdp_wait()
+		 */
+	        handle_pending_TLB_flushes();
+
+		cpu_pause();
+	}
+	atomic_decl((volatile long *)&mp_kdb_ncpus, 1);
+	DBG("mp_kdb_wait() done\n");
 }
 
 /*
@@ -1102,9 +1426,34 @@ clear_kdb_intr(void)
 	mp_enable_preemption();
 }
 
+void
+mp_kdb_exit(void)
+{
+	DBG("mp_kdb_exit()\n");
+	atomic_decl((volatile long *)&mp_kdb_ncpus, 1);
+	mp_kdb_trap = FALSE;
+	__asm__ volatile("mfence");
+
+	while (mp_kdb_ncpus > 0) {
+	        /*
+		 * a TLB shootdown request may be pending... this would result in the requesting
+		 * processor waiting in PMAP_UPDATE_TLBS() until this processor deals with it.
+		 * Process it, so it can now enter mp_kdp_wait()
+		 */
+	        handle_pending_TLB_flushes();
+
+		cpu_pause();
+	}
+	DBG("mp_kdb_exit() done\n");
+}
+
+#endif /* MACH_KDB */
+
 /*
  * i386_init_slave() is called from pstart.
  * We're in the cpu's interrupt stack with interrupts disabled.
+ * At this point we are in legacy mode. We need to switch on IA32e
+ * if the mode is set to 64-bits.
  */
 void
 i386_init_slave(void)
@@ -1117,16 +1466,30 @@ i386_init_slave(void)
 	DBG("i386_init_slave() CPU%d: phys (%d) active.\n",
 		get_cpu_number(), get_cpu_phys_number());
 
-	lapic_init();
+	assert(!ml_get_interrupts_enabled());
+	if (cpu_mode_is64bit()) {
+		cpu_IA32e_enable(current_cpu_datap());
+		cpu_desc_load64(current_cpu_datap());
+		fast_syscall_init64();
+	} else {
+		fast_syscall_init();
+	}
 
+	mca_cpu_init();
+
+	lapic_init();
 	LAPIC_DUMP();
 	LAPIC_CPU_MAP_DUMP();
 
+	init_fpu();
+
 	mtrr_update_cpu();
 
 	pat_init();
 
-	cpu_init();
+	cpu_thread_init();
+
+	cpu_init();	/* Sets cpu_running which starter cpu waits for */ 
 
 	slave_main();
 
@@ -1137,19 +1500,13 @@ void
 slave_machine_init(void)
 {
 	/*
- 	 * Here in process context.
+ 	 * Here in process context, but with interrupts disabled.
 	 */
 	DBG("slave_machine_init() CPU%d\n", get_cpu_number());
 
-	init_fpu();
-
-	cpu_thread_init();
-
-	pmc_init();
-
-	cpu_machine_init();
-
 	clock_init();
+
+	cpu_machine_init();		/* Interrupts enabled hereafter */
 }
 
 #undef cpu_number()
@@ -1204,44 +1561,5 @@ db_trap_hist(void)
 		
 }
 #endif	/* TRAP_DEBUG */
-
-void db_lapic(int cpu);
-unsigned int db_remote_read(int cpu, int reg);
-void db_ioapic(unsigned int);
-void kdb_console(void);
-
-void
-kdb_console(void)
-{
-}
-
-#define BOOLP(a) ((a)?' ':'!')
-
-static char *DM[8] = {
-	"Fixed",
-	"Lowest Priority",
-	"Invalid",
-	"Invalid",
-	"NMI",
-	"Reset",
-	"Invalid",
-	"ExtINT"};
-
-unsigned int
-db_remote_read(int cpu, int reg)
-{
-	return -1;
-}
-
-void
-db_lapic(int cpu)
-{
-}
-
-void
-db_ioapic(unsigned int ind)
-{
-}
-
 #endif	/* MACH_KDB */
 
diff --git a/osfmk/i386/mp.h b/osfmk/i386/mp.h
index 30a6d941b..13905aa67 100644
--- a/osfmk/i386/mp.h
+++ b/osfmk/i386/mp.h
@@ -77,6 +77,7 @@
 #include <sys/cdefs.h>
 #include <mach/boolean.h>
 #include <mach/kern_return.h>
+#include <mach/i386/thread_status.h>
 
 __BEGIN_DECLS
 
@@ -91,12 +92,13 @@ extern void lapic_shutdown(void);
 extern void lapic_smm_restore(void);
 extern boolean_t lapic_probe(void);
 extern void lapic_dump(void);
-extern int  lapic_interrupt(int interrupt, void *state);
+extern int  lapic_interrupt(int interrupt, x86_saved_state_t *state);
 extern void lapic_end_of_interrupt(void);
 extern int  lapic_to_cpu[];
 extern int  cpu_to_lapic[];
 extern int  lapic_interrupt_base;
 extern void lapic_cpu_map(int lapic, int cpu_num);
+extern uint32_t ml_get_apicid(uint32_t cpu);
 
 extern void lapic_set_timer(
 		boolean_t		interrupt,
@@ -113,9 +115,50 @@ extern void lapic_get_timer(
 typedef	void (*i386_intr_func_t)(void *);
 extern void lapic_set_timer_func(i386_intr_func_t func);
 extern void lapic_set_pmi_func(i386_intr_func_t func);
+extern void lapic_set_thermal_func(i386_intr_func_t func);
 
 __END_DECLS
 
+/*
+ * By default, use high vectors to leave vector space for systems
+ * with multiple I/O APIC's. However some systems that boot with
+ * local APIC disabled will hang in SMM when vectors greater than
+ * 0x5F are used. Those systems are not expected to have I/O APIC
+ * so 16 (0x50 - 0x40) vectors for legacy PIC support is perfect.
+ */
+#define LAPIC_DEFAULT_INTERRUPT_BASE	0xD0
+#define LAPIC_REDUCED_INTERRUPT_BASE	0x50
+/*
+ * Specific lapic interrupts are relative to this base
+ * in priority order from high to low:
+ */
+
+#define LAPIC_PERFCNT_INTERRUPT		0xF
+#define LAPIC_TIMER_INTERRUPT		0xE
+#define LAPIC_INTERPROCESSOR_INTERRUPT	0xD
+#define LAPIC_THERMAL_INTERRUPT		0xC
+#define LAPIC_ERROR_INTERRUPT		0xB
+#define LAPIC_SPURIOUS_INTERRUPT	0xA
+/* The vector field is ignored for NMI interrupts via the LAPIC
+ * or otherwise, so this is not an offset from the interrupt
+ * base.
+ */
+#define LAPIC_NMI_INTERRUPT		0x2
+
+#define LAPIC_REG(reg) \
+	(*((volatile uint32_t *)(lapic_start + LAPIC_##reg)))
+#define LAPIC_REG_OFFSET(reg,off) \
+	(*((volatile uint32_t *)(lapic_start + LAPIC_##reg + (off))))
+
+#define LAPIC_VECTOR(src) \
+	(lapic_interrupt_base + LAPIC_##src##_INTERRUPT)
+
+#define LAPIC_ISR_IS_SET(base,src) \
+	(LAPIC_REG_OFFSET(ISR_BASE,((base+LAPIC_##src##_INTERRUPT)/32)*0x10) & \
+		(1 <<((base + LAPIC_##src##_INTERRUPT)%32)))
+
+extern vm_offset_t	lapic_start;
+
 #endif	/* ASSEMBLER */
 
 #define CPU_NUMBER(r)				\
@@ -146,20 +189,28 @@ extern	void	console_cpu_free(void *console_buf);
 
 extern	int	kdb_cpu;		/* current cpu running kdb	*/
 extern	int	kdb_debug;
-extern	int	kdb_is_slave[];
 extern	int	kdb_active[];
 
 extern	volatile boolean_t mp_kdp_trap;
 extern	void	mp_kdp_enter(void);
 extern	void	mp_kdp_exit(void);
 
+#if MACH_KDB
+extern void mp_kdb_exit(void);
+#endif
+
 /*
  * All cpu rendezvous:
  */
-extern void mp_rendezvous(void (*setup_func)(void *),
-			  void (*action_func)(void *),
-			  void (*teardown_func)(void *),
-			  void *arg);
+extern void mp_rendezvous(
+		void (*setup_func)(void *),
+		void (*action_func)(void *),
+		void (*teardown_func)(void *),
+		void *arg);
+extern void mp_rendezvous_no_intrs(
+		void (*action_func)(void *),
+		void *arg);
+extern void mp_rendezvous_break_lock(void);
 
 __END_DECLS
 
diff --git a/osfmk/i386/mp_desc.c b/osfmk/i386/mp_desc.c
index f136838b1..64f460e69 100644
--- a/osfmk/i386/mp_desc.c
+++ b/osfmk/i386/mp_desc.c
@@ -61,14 +61,18 @@
 #include <kern/cpu_number.h>
 #include <kern/kalloc.h>
 #include <kern/cpu_data.h>
+#include <mach/mach_types.h>
 #include <mach/machine.h>
+#include <mach/vm_map.h>
 #include <vm/vm_kern.h>
+#include <vm/vm_map.h>
 
 #include <i386/mp_desc.h>
 #include <i386/lock.h>
 #include <i386/misc_protos.h>
 #include <i386/mp.h>
 #include <i386/pmap.h>
+#include <i386/cpu_threads.h>
 
 #include <kern/misc_protos.h>
 
@@ -83,8 +87,8 @@
 /*
  * First cpu`s interrupt stack.
  */
-extern char		intstack[];	/* bottom */
-extern char		eintstack[];	/* top */
+extern uint32_t		low_intstack[];	/* bottom */
+extern uint32_t		low_eintstack[];	/* top */
 
 /*
  * Per-cpu data area pointers.
@@ -98,6 +102,15 @@ decl_simple_lock_data(,cpu_lock);	/* protects real_ncpus */
 unsigned int	real_ncpus = 1;
 unsigned int	max_ncpus = MAX_CPUS;
 
+extern void *hi_remap_text;
+#define HI_TEXT(lo_text)	\
+	(((uint32_t)&lo_text - (uint32_t)&hi_remap_text) + HIGH_MEM_BASE)
+
+extern void	hi_sysenter(void);
+extern void	hi64_sysenter(void);
+extern void	hi64_syscall(void);
+
+
 /*
  * Multiprocessor i386/i486 systems use a separate copy of the
  * GDT, IDT, LDT, and kernel TSS per processor.  The first three
@@ -113,13 +126,14 @@ unsigned int	max_ncpus = MAX_CPUS;
 
 struct fake_descriptor ldt_desc_pattern = {
 	(unsigned int) 0,
-	LDTSZ * sizeof(struct fake_descriptor) - 1,
+	LDTSZ_MIN * sizeof(struct fake_descriptor) - 1,
 	0,
 	ACC_P|ACC_PL_K|ACC_LDT
 };
+
 struct fake_descriptor tss_desc_pattern = {
 	(unsigned int) 0,
-	sizeof(struct i386_tss),
+	sizeof(struct i386_tss) - 1,
 	0,
 	ACC_P|ACC_PL_K|ACC_TSS
 };
@@ -131,53 +145,133 @@ struct fake_descriptor cpudata_desc_pattern = {
 	ACC_P|ACC_PL_K|ACC_DATA_W
 };
 
+struct fake_descriptor userwindow_desc_pattern = {
+	(unsigned int) 0,
+	((NBPDE * NCOPY_WINDOWS) / PAGE_SIZE) - 1,
+	SZ_32 | SZ_G,
+	ACC_P|ACC_PL_U|ACC_DATA_W
+};
+
+struct fake_descriptor physwindow_desc_pattern = {
+	(unsigned int) 0,
+	PAGE_SIZE - 1,
+	SZ_32,
+	ACC_P|ACC_PL_K|ACC_DATA_W
+};
+
+/*
+ * This is the expanded, 64-bit variant of the kernel LDT descriptor.
+ * When switching to 64-bit mode this replaces KERNEL_LDT entry
+ * and the following empty slot. This enables the LDT to be referenced
+ * in the uber-space remapping window on the kernel.
+ */
+struct fake_descriptor64 kernel_ldt_desc64 = {
+	FAKE_UBER64(&master_ldt),
+	LDTSZ_MIN*sizeof(struct fake_descriptor)-1,
+	0,
+	ACC_P|ACC_PL_K|ACC_LDT,
+	0
+};
+
+/*
+ * This is the expanded, 64-bit variant of the kernel TSS descriptor.
+ * It is follows pattern of the KERNEL_LDT.
+ */
+struct fake_descriptor64 kernel_tss_desc64 = {
+	FAKE_UBER64(&master_ktss64),
+	sizeof(struct x86_64_tss)-1,
+	0,
+	ACC_P|ACC_PL_K|ACC_TSS,
+	0
+};
+
 void
-mp_desc_init(
+cpu_desc_init(
 	cpu_data_t	*cdp,
 	boolean_t	is_boot_cpu)
 {
-	struct mp_desc_table	*mpt = cdp->cpu_desc_tablep;
-	cpu_desc_index_t	*cdt = &cdp->cpu_desc_index;
+	cpu_desc_table_t	*cdt = cdp->cpu_desc_tablep;
+	cpu_desc_index_t	*cdi = &cdp->cpu_desc_index;
 
 	if (is_boot_cpu) {
 	    /*
 	     * Master CPU uses the tables built at boot time.
-	     * Just set the TSS and GDT pointers.
+	     * Just set the index pointers to the high shared-mapping space.
+	     * Note that the sysenter stack uses empty space above the ktss
+	     * in the HIGH_FIXED_KTSS page. In this case we don't map the
+	     * the real master_sstk in low memory.
 	     */
-	    cdt->cdi_ktss = &ktss;
+	    cdi->cdi_ktss = (struct i386_tss *)
+				pmap_index_to_virt(HIGH_FIXED_KTSS) ;
+	    cdi->cdi_sstk  = (vm_offset_t) (cdi->cdi_ktss + 1) +
+				(vm_offset_t) &master_sstk.top -
+				(vm_offset_t) &master_sstk;
 #if	MACH_KDB
-	    cdt->cdi_dbtss = &dbtss;
+	    cdi->cdi_dbtss = (struct i386_tss *)
+				pmap_index_to_virt(HIGH_FIXED_DBTSS);
 #endif	/* MACH_KDB */
-	    cdt->cdi_gdt = gdt;
-	    cdt->cdi_idt = idt;
-	    cdt->cdi_ldt = ldt;
+	    cdi->cdi_gdt = (struct fake_descriptor *)
+				pmap_index_to_virt(HIGH_FIXED_GDT);
+	    cdi->cdi_idt = (struct fake_descriptor *)
+				pmap_index_to_virt(HIGH_FIXED_IDT);
+	    cdi->cdi_ldt = (struct fake_descriptor *)
+				pmap_index_to_virt(HIGH_FIXED_LDT_BEGIN);
 
 	} else {
 
-	    cdt->cdi_ktss = &mpt->ktss;
-	    cdt->cdi_gdt = mpt->gdt;
-	    cdt->cdi_idt = mpt->idt;
-	    cdt->cdi_ldt = mpt->ldt;
+	    vm_offset_t	cpu_hi_desc;
+
+	    cpu_hi_desc = pmap_cpu_high_shared_remap(cdp->cpu_number,
+						     HIGH_CPU_DESC,
+						     (vm_offset_t) cdt, 1);
+
+	    /*
+	     * Per-cpu GDT, IDT, LDT, KTSS descriptors are allocated in one
+	     * block (cpu_desc_table) and double-mapped into high shared space
+	     * in one page window.
+	     * Also, a transient stack for the fast sysenter path. The top of
+	     * which is set at context switch time to point to the PCB using
+	     * the high address.
+	     */
+	    cdi->cdi_gdt  = (struct fake_descriptor *) (cpu_hi_desc +
+				offsetof(cpu_desc_table_t, gdt[0]));
+	    cdi->cdi_idt  = (struct fake_descriptor *) (cpu_hi_desc +
+				offsetof(cpu_desc_table_t, idt[0]));
+	    cdi->cdi_ktss = (struct i386_tss *) (cpu_hi_desc +
+				offsetof(cpu_desc_table_t, ktss));
+	    cdi->cdi_sstk = cpu_hi_desc +
+				offsetof(cpu_desc_table_t, sstk.top);
+				
+	    /*
+	     * LDT descriptors are mapped into a seperate area.
+	     */
+	    cdi->cdi_ldt  = (struct fake_descriptor *)
+				pmap_cpu_high_shared_remap(
+				    cdp->cpu_number,
+				    HIGH_CPU_LDT_BEGIN,
+				    (vm_offset_t) cdp->cpu_ldtp,
+				    HIGH_CPU_LDT_END - HIGH_CPU_LDT_BEGIN + 1);
 
 	    /*
 	     * Copy the tables
 	     */
-	    bcopy((char *)idt,
-		  (char *)mpt->idt,
-		  sizeof(idt));
-	    bcopy((char *)gdt,
-		  (char *)mpt->gdt,
-		  sizeof(gdt));
-	    bcopy((char *)ldt,
-		  (char *)mpt->ldt,
-		  sizeof(ldt));
-	    bzero((char *)&mpt->ktss,
+	    bcopy((char *)master_idt,
+		  (char *)cdt->idt,
+		  sizeof(master_idt));
+	    bcopy((char *)master_gdt,
+		  (char *)cdt->gdt,
+		  sizeof(master_gdt));
+	    bcopy((char *)master_ldt,
+		  (char *)cdp->cpu_ldtp,
+		  sizeof(master_ldt));
+	    bzero((char *)&cdt->ktss,
 		  sizeof(struct i386_tss));
 
 #if	MACH_KDB
-	    cdt->cdi_dbtss = &dbtss;
-	    bcopy((char *)&dbtss,
-		  (char *)&mpt->dbtss,
+	    cdi->cdi_dbtss = (struct i386_tss *) (cpu_hi_desc +
+				offsetof(cpu_desc_table_t, dbtss));
+	    bcopy((char *)&master_dbtss,
+		  (char *)&cdt->dbtss,
 		  sizeof(struct i386_tss));
 #endif	/* MACH_KDB */
 
@@ -185,32 +279,196 @@ mp_desc_init(
 	     * Fix up the entries in the GDT to point to
 	     * this LDT and this TSS.
 	     */
-	    mpt->gdt[sel_idx(KERNEL_LDT)] = ldt_desc_pattern;
-	    mpt->gdt[sel_idx(KERNEL_LDT)].offset = (vm_offset_t) mpt->ldt;
-	    fix_desc(&mpt->gdt[sel_idx(KERNEL_LDT)], 1);
+	    cdt->gdt[sel_idx(KERNEL_LDT)] = ldt_desc_pattern;
+	    cdt->gdt[sel_idx(KERNEL_LDT)].offset = (vm_offset_t) cdi->cdi_ldt;
+	    fix_desc(&cdt->gdt[sel_idx(KERNEL_LDT)], 1);
+
+	    cdt->gdt[sel_idx(USER_LDT)] = ldt_desc_pattern;
+	    cdt->gdt[sel_idx(USER_LDT)].offset = (vm_offset_t) cdi->cdi_ldt;
+	    fix_desc(&cdt->gdt[sel_idx(USER_LDT)], 1);
 
-	    mpt->gdt[sel_idx(KERNEL_TSS)] = tss_desc_pattern;
-	    mpt->gdt[sel_idx(KERNEL_TSS)].offset = (vm_offset_t) &mpt->ktss;
-	    fix_desc(&mpt->gdt[sel_idx(KERNEL_TSS)], 1);
+	    cdt->gdt[sel_idx(KERNEL_TSS)] = tss_desc_pattern;
+	    cdt->gdt[sel_idx(KERNEL_TSS)].offset = (vm_offset_t) cdi->cdi_ktss;
+	    fix_desc(&cdt->gdt[sel_idx(KERNEL_TSS)], 1);
 
-	    mpt->gdt[sel_idx(CPU_DATA_GS)] = cpudata_desc_pattern;
-	    mpt->gdt[sel_idx(CPU_DATA_GS)].offset = (vm_offset_t) cdp;
-	    fix_desc(&mpt->gdt[sel_idx(CPU_DATA_GS)], 1);
+	    cdt->gdt[sel_idx(CPU_DATA_GS)] = cpudata_desc_pattern;
+	    cdt->gdt[sel_idx(CPU_DATA_GS)].offset = (vm_offset_t) cdp;
+	    fix_desc(&cdt->gdt[sel_idx(CPU_DATA_GS)], 1);
 
 #if	MACH_KDB
-	    mpt->gdt[sel_idx(DEBUG_TSS)] = tss_desc_pattern;
-	    mpt->gdt[sel_idx(DEBUG_TSS)].offset = (vm_offset_t) &mpt->dbtss;
-	    fix_desc(&mpt->gdt[sel_idx(DEBUG_TSS)], 1);
-
-	    mpt->dbtss.esp0 = (int)(db_task_stack_store +
-		    (INTSTACK_SIZE * (cpu + 1)) - sizeof (natural_t));
-	    mpt->dbtss.esp = mpt->dbtss.esp0;
-	    mpt->dbtss.eip = (int)&db_task_start;
+	    cdt->gdt[sel_idx(DEBUG_TSS)] = tss_desc_pattern;
+	    cdt->gdt[sel_idx(DEBUG_TSS)].offset = (vm_offset_t) cdi->cdi_dbtss;
+	    fix_desc(&cdt->gdt[sel_idx(DEBUG_TSS)], 1);
+
+	    cdt->dbtss.esp0 = (int)(db_task_stack_store +
+		    (INTSTACK_SIZE * (cdp->cpu_number)) - sizeof (natural_t));
+	    cdt->dbtss.esp = cdt->dbtss.esp0;
+	    cdt->dbtss.eip = (int)&db_task_start;
 #endif	/* MACH_KDB */
 
-	    mpt->ktss.ss0 = KERNEL_DS;
-	    mpt->ktss.io_bit_map_offset = 0x0FFF;	/* no IO bitmap */
+	    cdt->ktss.ss0 = KERNEL_DS;
+	    cdt->ktss.io_bit_map_offset = 0x0FFF;	/* no IO bitmap */
+
+	    cpu_window_init(cdp->cpu_number);
+
 	}
+
+}
+
+void
+cpu_desc_init64(
+	cpu_data_t	*cdp,
+	boolean_t	is_boot_cpu)
+{
+	cpu_desc_table64_t	*cdt = (cpu_desc_table64_t *)
+					cdp->cpu_desc_tablep;
+	cpu_desc_index_t	*cdi = &cdp->cpu_desc_index;
+
+	if (is_boot_cpu) {
+	    /*
+	     * Master CPU uses the tables built at boot time.
+	     * Just set the index pointers to the low memory space.
+	     * Note that in 64-bit mode these are addressed in the
+	     * double-mapped window (uber-space).
+	     */
+	    cdi->cdi_ktss = (struct i386_tss *) &master_ktss64;
+	    cdi->cdi_sstk = (vm_offset_t) &master_sstk.top;
+	    cdi->cdi_gdt  = master_gdt;
+	    cdi->cdi_idt  = (struct fake_descriptor *) &master_idt64;
+	    cdi->cdi_ldt  = (struct fake_descriptor *) &master_ldt;
+
+	    /* Replace the expanded LDT and TSS slots in the GDT: */
+	    *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_LDT)] =
+		kernel_ldt_desc64;
+	    *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_TSS)] =
+		kernel_tss_desc64;
+
+	    /*
+	     * Fix up the expanded descriptors for 64-bit.
+	     */
+	    fix_desc64((void *) &master_idt64, IDTSZ);
+	    fix_desc64((void *) &master_gdt[sel_idx(KERNEL_LDT)], 1);
+	    fix_desc64((void *) &master_gdt[sel_idx(KERNEL_TSS)], 1);
+
+	    /*
+	     * Set the double-fault stack as IST1 in the 64-bit TSS 
+	     */
+	    master_ktss64.ist1 = UBER64(df_task_stack_end);
+
+	} else {
+	    /*
+	     * Per-cpu GDT, IDT, KTSS descriptors are allocated in kernel 
+	     * heap (cpu_desc_table) and double-mapped in uber-space (over 4GB).
+	     * LDT descriptors are mapped into a separate area.
+	     */
+	    cdi->cdi_gdt  = cdt->gdt;
+	    cdi->cdi_idt  = (struct fake_descriptor *) cdt->idt;
+	    cdi->cdi_ktss = (struct i386_tss *) &cdt->ktss;
+	    cdi->cdi_sstk = (vm_offset_t) &cdt->sstk.top;
+	    cdi->cdi_ldt  = cdp->cpu_ldtp;
+
+	    /*
+	     * Copy the tables
+	     */
+	    bcopy((char *)master_idt64,
+		  (char *)cdt->idt,
+		  sizeof(master_idt64));
+	    bcopy((char *)master_gdt,
+		  (char *)cdt->gdt,
+		  sizeof(master_gdt));
+	    bcopy((char *)master_ldt,
+		  (char *)cdp->cpu_ldtp,
+		  sizeof(master_ldt));
+	    bcopy((char *)&master_ktss64,
+		  (char *)&cdt->ktss,
+		  sizeof(struct x86_64_tss));
+
+	    /*
+	     * Fix up the entries in the GDT to point to
+	     * this LDT and this TSS.
+	     */
+	    kernel_ldt_desc64.offset[0] = (vm_offset_t) cdi->cdi_ldt;
+	    *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_LDT)] =
+	    		kernel_ldt_desc64;
+	    fix_desc64(&cdt->gdt[sel_idx(KERNEL_LDT)], 1);
+
+	    kernel_ldt_desc64.offset[0] = (vm_offset_t) cdi->cdi_ldt;
+	    *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(USER_LDT)] =
+	    		kernel_ldt_desc64;
+	    fix_desc64(&cdt->gdt[sel_idx(USER_LDT)], 1);
+
+	    kernel_tss_desc64.offset[0] = (vm_offset_t) cdi->cdi_ktss;
+	    *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_TSS)] =
+	    		kernel_tss_desc64;
+	    fix_desc64(&cdt->gdt[sel_idx(KERNEL_TSS)], 1);
+
+	    cdt->gdt[sel_idx(CPU_DATA_GS)] = cpudata_desc_pattern;
+	    cdt->gdt[sel_idx(CPU_DATA_GS)].offset = (vm_offset_t) cdp;
+	    fix_desc(&cdt->gdt[sel_idx(CPU_DATA_GS)], 1);
+
+	    /* Set double-fault stack as IST1 */
+	    cdt->ktss.ist1 = UBER64(cdt->dfstk + sizeof(cdt->dfstk));
+
+	    /*
+	     * Allocate copyio windows.
+	     */
+	    cpu_window_init(cdp->cpu_number);
+
+	}
+
+	/* Require that the top of the sysenter stack is 16-byte aligned */
+	if ((cdi->cdi_sstk % 16) != 0)
+		panic("cpu_desc_init64() sysenter stack not 16-byte aligned");
+}
+
+/*
+ * Set MSRs for sysenter/sysexit for 64-bit.
+ */
+void
+fast_syscall_init64(void)
+{
+	wrmsr64(MSR_IA32_SYSENTER_CS, SYSENTER_CS); 
+	wrmsr64(MSR_IA32_SYSENTER_EIP, UBER64(hi64_sysenter));
+	wrmsr64(MSR_IA32_SYSENTER_ESP, UBER64(current_sstk()));
+
+	/* Enable syscall/sysret */
+	wrmsr64(MSR_IA32_EFER, rdmsr64(MSR_IA32_EFER) | MSR_IA32_EFER_SCE);
+
+	/*
+	 * MSRs for 64-bit syscall/sysret
+	 * Note USER_CS because sysret uses this + 16 when returning to
+	 * 64-bit code.
+	 */
+	wrmsr64(MSR_IA32_LSTAR, UBER64(hi64_syscall));
+	wrmsr64(MSR_IA32_STAR, (((uint64_t)USER_CS)     << 48) |
+			       (((uint64_t)KERNEL64_CS) << 32));
+	/*
+	 * Emulate eflags cleared by sysenter but note that
+	 * we also clear the trace trap to avoid the complications
+	 * of single-stepping into a syscall. We also clear
+	 * the nested task bit to avoid a spurious "task switch"
+	 * on IRET.
+	 */
+	wrmsr64(MSR_IA32_FMASK, EFL_DF|EFL_IF|EFL_TF|EFL_NT);
+
+	/*
+	 * Set the Kermel GS base MSR to point to per-cpu data in uber-space.
+	 * The uber-space handler (hi64_syscall) uses the swapgs instruction.
+	 */
+	wrmsr64(MSR_IA32_KERNEL_GS_BASE, UBER64(current_cpu_datap()));
+	kprintf("fast_syscall_init64() KERNEL_GS_BASE=0x%016llx\n",
+		rdmsr64(MSR_IA32_KERNEL_GS_BASE));
+}
+
+/*
+ * Set MSRs for sysenter/sysexit
+ */
+void
+fast_syscall_init(void)
+{
+	wrmsr(MSR_IA32_SYSENTER_CS, SYSENTER_CS, 0); 
+	wrmsr(MSR_IA32_SYSENTER_EIP, HI_TEXT(hi_sysenter), 0);
+	wrmsr(MSR_IA32_SYSENTER_ESP, current_sstk(), 0);
 }
 
 cpu_data_t *
@@ -227,8 +485,10 @@ cpu_data_alloc(boolean_t is_boot_cpu)
 			cdp->cpu_processor = cpu_processor_alloc(TRUE);
 			cdp->cpu_pmap = pmap_cpu_alloc(TRUE);
 			cdp->cpu_this = cdp;
-			cdp->cpu_int_stack_top = (vm_offset_t) eintstack;
-			mp_desc_init(cdp, TRUE);
+			cdp->cpu_is64bit = FALSE;
+			cdp->cpu_int_stack_top = (vm_offset_t) low_eintstack;
+			cpu_desc_init(cdp, TRUE);
+			fast_syscall_init();
 		}
 		return cdp;
 	}
@@ -249,6 +509,9 @@ cpu_data_alloc(boolean_t is_boot_cpu)
 	bzero((void*) cdp, sizeof(cpu_data_t));
 	cdp->cpu_this = cdp;
 
+	/* Propagate mode */
+	cdp->cpu_is64bit = cpu_mode_is64bit();
+
 	/*
 	 * Allocate interrupt stack:
 	 */
@@ -264,15 +527,28 @@ cpu_data_alloc(boolean_t is_boot_cpu)
 
 	/*
 	 * Allocate descriptor table:
+	 * Size depends on cpu mode.
 	 */
 	ret = kmem_alloc(kernel_map, 
 			 (vm_offset_t *) &cdp->cpu_desc_tablep,
-			 sizeof(struct mp_desc_table));
+			 cdp->cpu_is64bit ? sizeof(cpu_desc_table64_t)
+					  : sizeof(cpu_desc_table_t));
 	if (ret != KERN_SUCCESS) {
 		printf("cpu_data_alloc() desc_table failed, ret=%d\n", ret);
 		goto abort;
 	}
 
+	/*
+	 * Allocate LDT
+	 */
+	ret = kmem_alloc(kernel_map, 
+			 (vm_offset_t *) &cdp->cpu_ldtp,
+			 sizeof(struct real_descriptor) * LDTSZ);
+	if (ret != KERN_SUCCESS) {
+		printf("cpu_data_alloc() ldt failed, ret=%d\n", ret);
+		goto abort;
+	}
+
 	simple_lock(&cpu_lock);
 	if (real_ncpus >= max_ncpus) {
 		simple_unlock(&cpu_lock);
@@ -282,10 +558,11 @@ cpu_data_alloc(boolean_t is_boot_cpu)
 	cdp->cpu_number = real_ncpus;
 	real_ncpus++;
 	simple_unlock(&cpu_lock);
-	
+
 	kprintf("cpu_data_alloc(%d) 0x%x desc_table: 0x%x "
+		"ldt: 0x%x "
 		"int_stack: 0x%x-0x%x\n",
-		cdp->cpu_number, cdp, cdp->cpu_desc_tablep,
+		cdp->cpu_number, cdp, cdp->cpu_desc_tablep, cdp->cpu_ldtp,
 		cdp->cpu_int_stack_top - INTSTACK_SIZE, cdp->cpu_int_stack_top);
 
 	return cdp;
@@ -319,3 +596,128 @@ valid_user_segment_selectors(uint16_t cs,
 	       valid_user_data_selector(gs);
 }
 
+
+static vm_offset_t user_window_base = 0;
+static vm_offset_t phys_window_base = 0;
+
+void
+cpu_window_init(int cpu)
+{
+	cpu_data_t		*cdp = cpu_data_ptr[cpu];
+	cpu_desc_index_t	*cdi;
+        vm_offset_t 		user_window;
+        vm_offset_t 		phys_window;
+        vm_offset_t 		vaddr;
+	int			num_cpus;
+
+	num_cpus = ml_get_max_cpus();
+
+	if (cpu >= num_cpus)
+	        panic("copy_window_init: cpu > num_cpus");
+
+	if (user_window_base == 0) {
+
+	        if (vm_allocate(kernel_map, &vaddr,
+				(NBPDE * NCOPY_WINDOWS * num_cpus) + NBPDE,
+				VM_FLAGS_ANYWHERE) != KERN_SUCCESS)
+		        panic("copy_window_init: "
+				"couldn't allocate user map window");
+
+		/*
+		 * window must start on a page table boundary
+		 * in the virtual address space
+		 */
+		user_window_base = (vaddr + (NBPDE - 1)) & ~(NBPDE - 1);
+
+		/*
+		 * get rid of any allocation leading up to our
+		 * starting boundary
+		 */
+		vm_deallocate(kernel_map, vaddr, user_window_base - vaddr);
+
+		/*
+		 * get rid of tail that we don't need
+		 */
+		user_window = user_window_base +
+					(NBPDE * NCOPY_WINDOWS * num_cpus);
+
+		vm_deallocate(kernel_map, user_window,
+				(vaddr +
+				 ((NBPDE * NCOPY_WINDOWS * num_cpus) + NBPDE)) -
+				 user_window);
+
+	        if (vm_allocate(kernel_map, &phys_window_base,
+				PAGE_SIZE * num_cpus, VM_FLAGS_ANYWHERE)
+					!= KERN_SUCCESS)
+		        panic("copy_window_init: "
+				"couldn't allocate phys map window");
+	}
+
+	user_window = user_window_base + (cpu * NCOPY_WINDOWS * NBPDE);
+	phys_window = phys_window_base + (cpu * PAGE_SIZE);
+
+	cdi = &cdp->cpu_desc_index;
+	  
+	cdp->cpu_copywindow_base = user_window;
+	cdp->cpu_copywindow_pdp  = pmap_pde(kernel_pmap, user_window);
+
+	cdi->cdi_gdt[sel_idx(USER_WINDOW_SEL)] = userwindow_desc_pattern;
+	cdi->cdi_gdt[sel_idx(USER_WINDOW_SEL)].offset = user_window;
+
+	fix_desc(&cdi->cdi_gdt[sel_idx(USER_WINDOW_SEL)], 1);
+
+	cdp->cpu_physwindow_base = phys_window;
+
+	/*
+	 * make sure the page that encompasses the
+	 * pte pointer we're interested in actually
+	 * exists in the page table
+	 */
+	pmap_expand(kernel_pmap, phys_window);
+
+	cdp->cpu_physwindow_ptep = vtopte(phys_window);
+
+	cdi->cdi_gdt[sel_idx(PHYS_WINDOW_SEL)] = physwindow_desc_pattern;
+	cdi->cdi_gdt[sel_idx(PHYS_WINDOW_SEL)].offset = phys_window;
+
+	fix_desc(&cdi->cdi_gdt[sel_idx(PHYS_WINDOW_SEL)], 1);
+}
+
+
+typedef struct {
+	uint16_t	length;
+	uint32_t	offset[2];
+} __attribute__((__packed__)) table_descriptor64_t;
+
+extern	table_descriptor64_t	gdtptr64;
+extern	table_descriptor64_t	idtptr64;
+/*
+ * Load the segment descriptor tables for the current processor.
+ */
+void
+cpu_desc_load64(cpu_data_t *cdp)
+{
+	cpu_desc_index_t	*cdi = &cdp->cpu_desc_index;
+
+ 	/*
+	 * Load up the new descriptors etc
+	 * ml_load_desc64() expects these global pseudo-descriptors:
+	 *   gdtptr64 -> master_gdt
+	 *   idtptr64 -> master_idt64
+	 * These are 10-byte descriptors with 64-bit addresses into
+	 * uber-space.
+	 */
+	gdtptr64.length = sizeof(master_gdt) - 1;
+	gdtptr64.offset[0] = (uint32_t) cdi->cdi_gdt;
+	gdtptr64.offset[1] = KERNEL_UBER_BASE_HI32;
+	idtptr64.length = sizeof(master_idt64) - 1;
+	idtptr64.offset[0] = (uint32_t) cdi->cdi_idt;
+	idtptr64.offset[1] = KERNEL_UBER_BASE_HI32;
+
+	/* Make sure busy bit is cleared in the TSS */
+	gdt_desc_p(KERNEL_TSS)->access &= ~ACC_TSS_BUSY;
+	
+	ml_load_desc64();
+
+	kprintf("64-bit descriptor tables loaded\n");
+}
diff --git a/osfmk/i386/mp_desc.h b/osfmk/i386/mp_desc.h
index ca147e689..8f591cfcc 100644
--- a/osfmk/i386/mp_desc.h
+++ b/osfmk/i386/mp_desc.h
@@ -83,26 +83,47 @@ __BEGIN_DECLS
  * Note that dbtss could be conditionalized on MACH_KDB, but
  * doing so increases misconfiguration risk.
  */
-struct mp_desc_table {
-	struct fake_descriptor	idt[IDTSZ];	/* IDT */
-	struct fake_descriptor	gdt[GDTSZ];	/* GDT */
-	struct fake_descriptor	ldt[LDTSZ];	/* LDT */
-	struct i386_tss		ktss;
-	struct i386_tss		dbtss;
-};
+typedef struct cpu_desc_table {
+	struct fake_descriptor	idt[IDTSZ] __attribute__ ((aligned (16)));
+	struct fake_descriptor	gdt[GDTSZ] __attribute__ ((aligned (16)));
+	struct i386_tss		ktss       __attribute__ ((aligned (16)));
+	struct i386_tss		dbtss      __attribute__ ((aligned (16)));
+	struct sysenter_stack	sstk;
+} cpu_desc_table_t;
+
+typedef struct cpu_desc_table64 {
+	struct fake_descriptor64 idt[IDTSZ]      __attribute__ ((aligned (16)));
+	struct fake_descriptor	gdt[GDTSZ]       __attribute__ ((aligned (16)));
+	struct x86_64_tss	ktss             __attribute__ ((aligned (16)));
+	struct sysenter_stack	sstk	         __attribute__ ((aligned (16)));
+	uint8_t			dfstk[PAGE_SIZE] __attribute__ ((aligned (16)));
+} cpu_desc_table64_t;
 
 #define	current_gdt()	(current_cpu_datap()->cpu_desc_index.cdi_gdt)
 #define	current_idt()	(current_cpu_datap()->cpu_desc_index.cdi_idt)
 #define	current_ldt()	(current_cpu_datap()->cpu_desc_index.cdi_ldt)
 #define	current_ktss()	(current_cpu_datap()->cpu_desc_index.cdi_ktss)
 #define	current_dbtss()	(current_cpu_datap()->cpu_desc_index.cdi_dbtss)
+#define	current_sstk()	(current_cpu_datap()->cpu_desc_index.cdi_sstk)
+
+#define	current_ktss64() ((struct x86_64_tss *) current_ktss())
+#define	current_sstk64() ((addr64_t *) current_sstk())
 
 #define	gdt_desc_p(sel) \
 	((struct real_descriptor *)&current_gdt()[sel_idx(sel)])
 #define	ldt_desc_p(sel) \
 	((struct real_descriptor *)&current_ldt()[sel_idx(sel)])
 
-extern void	mp_desc_init(cpu_data_t *cdp, boolean_t is_boot_cpu);
+extern void	cpu_desc_init(
+			cpu_data_t	*cdp,
+			boolean_t	is_boot_cpu);
+extern void	cpu_desc_init64(
+			cpu_data_t	*cdp,
+			boolean_t	is_boot_cpu);
+extern void	cpu_desc_load64(
+			cpu_data_t	*cdp);
+extern void	fast_syscall_init(void);
+extern void	fast_syscall_init64(void);
 
 static inline boolean_t
 valid_user_data_selector(uint16_t selector)
diff --git a/osfmk/i386/mp_events.h b/osfmk/i386/mp_events.h
index c2b9e23d7..fac0a85ea 100644
--- a/osfmk/i386/mp_events.h
+++ b/osfmk/i386/mp_events.h
@@ -42,6 +42,7 @@ typedef enum {
 	MP_RENDEZVOUS,
 	MP_IDLE,
 	MP_UNIDLE,
+	MP_CHUD,
 	MP_LAST
 } mp_event_t;
 
@@ -54,6 +55,7 @@ const char *mp_event_name[] = {	\
 	"MP_RENDEZVOUS",	\
 	"MP_IDLE",		\
 	"MP_UNIDLE",		\
+	"MP_CHUD",		\
 	"MP_LAST"		\
 }
 	
diff --git a/osfmk/i386/mp_slave_boot.s b/osfmk/i386/mp_slave_boot.s
index 6fb8fdddc..15a9d811c 100644
--- a/osfmk/i386/mp_slave_boot.s
+++ b/osfmk/i386/mp_slave_boot.s
@@ -67,75 +67,72 @@
 	.text	
 	.align	12		// Page align for single bcopy_phys()
 
-#define	LJMP(segment,address)	\
-	.byte	0xea		;\
-	.long	address-EXT(slave_boot_base)		;\
+#define	operand_size_prefix	.byte 0x66
+#define	address_size_prefix	.byte 0x67
+
+#define	LJMP(segment,address)			\
+	operand_size_prefix			;\
+	.byte	0xea				;\
+	.long	address-EXT(slave_boot_base)	;\
 	.word	segment
 
-#define	LGDT(address)	\
-	.word	0x010f ;\
-	.byte	0x15 ;\
+#define	LGDT(address)				\
+	address_size_prefix			;\
+	operand_size_prefix			;\
+	.word	0x010f				;\
+	.byte	0x15				;\
 	.long	address-EXT(slave_boot_base)
 
 Entry(slave_boot_base)
 	/* code is loaded at 0x0:0x1000 */
 	/* ljmp to the next instruction to set up %cs */
-	data16
-	LJMP(MP_BOOTSEG, EXT(slave_pstart))
+	LJMP(MP_BOOTSEG, EXT(slave_rstart))
 
-Entry(slave_pstart)
+Entry(slave_rstart)
 	/* set up %ds */
 	mov	%cs, %ax
 	mov	%ax, %ds
 
-	POSTCODE(SLAVE_PSTART_ENTRY);
+	POSTCODE(SLAVE_RSTART_ENTRY);
 
 	/* set up %ss and %esp */
-	data16
-	mov	$(MP_BOOTSEG), %eax
+	mov	%cs, %ax
 	mov	%ax, %ss
-	data16
 	mov	$(MP_BOOTSTACK), %esp
 
 	/*set up %es */
 	mov	%ax, %es
 
 	/* change to protected mode */
-	data16
-	call	EXT(real_to_prot)
+	operand_size_prefix
+	call	EXT(slave_real_to_prot)
 
 	push	MP_MACH_START
-	call	EXT(startprog)
+	call	EXT(slave_startprog)
 
 /*
- real_to_prot()
+ slave_real_to_prot()
  	transfer from real mode to protected mode.
 */
 
-Entry(real_to_prot)
+Entry(slave_real_to_prot)
 	/* guarantee that interrupt is disabled when in prot mode */
 	cli
 
-	POSTCODE(REAL_TO_PROT_ENTRY);
+	POSTCODE(SLAVE_REAL_TO_PROT_ENTRY);
 
 	/* load the gdtr */
-	addr16
-	data16
 	LGDT(EXT(gdtr))
 
 	/* load the gdtr */
 	/* set the PE bit of CR0 */
 	mov	%cr0, %eax
-
-	data16
 	or	$(CR0_PE_ON), %eax
 	mov	%eax, %cr0 
 
 	/* make intrasegment jump to flush the processor pipeline and */
 	/* reload CS register */
-	data16
 	LJMP(0x08, xprot)
-
 xprot:
 	
 	/* we are in USE32 mode now */
@@ -145,20 +142,20 @@ xprot:
 	movw	%ax, %ss
 	movw	%ax, %es
 
-	POSTCODE(REAL_TO_PROT_EXIT);
+	POSTCODE(SLAVE_REAL_TO_PROT_EXIT);
 
 	ret
 
 /*
- startprog(phyaddr)
+ slave_startprog(phyaddr)
 	start the program on protected mode where phyaddr is the entry point
 */
 
-Entry(startprog)
+Entry(slave_startprog)
 	push	%ebp
 	movl	%esp, %ebp
 	
-	POSTCODE(STARTPROG_ENTRY);
+	POSTCODE(SLAVE_STARTPROG_ENTRY);
 
 	movl	0x8(%ebp), %ecx		/* entry offset  */
 	movl	$0x28, %ebx		/* segment */
@@ -170,7 +167,7 @@ Entry(startprog)
 	movw	%bx, %ds
 	movw	%bx, %es
 
-	POSTCODE(STARTPROG_EXIT);
+	POSTCODE(SLAVE_STARTPROG_EXIT);
 
 	lret
 
@@ -204,7 +201,7 @@ Entry(Gdt)
 	.byte	0,0x9e,0x0,0
 
 	.word	0xffff,0	/* 0x20 : init data */
-	.byte	0,0x93,0xcf,0
+	.byte	0,0x9f,0xcf,0
 
 	.word	0xffff,0	/* 0x28 : init code */
 	.byte	0,0x9f,0xcf,0
diff --git a/osfmk/i386/pcb.c b/osfmk/i386/pcb.c
index 31b34218f..ea386618e 100644
--- a/osfmk/i386/pcb.c
+++ b/osfmk/i386/pcb.c
@@ -96,24 +96,37 @@
 #include <i386/iopb_entries.h>
 #include <i386/mp_desc.h>
 #include <i386/cpu_data.h>
-
+#include <i386/machine_routines.h>
 
 /*
  * Maps state flavor to number of words in the state:
  */
-__private_extern__
 unsigned int _MachineStateCount[] = {
-	/* FLAVOR_LIST */ 0,
-	i386_NEW_THREAD_STATE_COUNT,
-	i386_FLOAT_STATE_COUNT,
-	i386_ISA_PORT_MAP_STATE_COUNT,
-	i386_V86_ASSIST_STATE_COUNT,
-	i386_REGS_SEGS_STATE_COUNT,
-	i386_THREAD_SYSCALL_STATE_COUNT,
-	/* THREAD_STATE_NONE */ 0,
-	i386_SAVED_STATE_COUNT,
+	/* FLAVOR_LIST */
+        0,
+	x86_THREAD_STATE32_COUNT,
+	x86_FLOAT_STATE32_COUNT,
+	x86_EXCEPTION_STATE32_COUNT,
+	x86_THREAD_STATE64_COUNT,
+	x86_FLOAT_STATE64_COUNT,
+	x86_EXCEPTION_STATE64_COUNT,
+	x86_THREAD_STATE_COUNT,
+	x86_FLOAT_STATE_COUNT,
+	x86_EXCEPTION_STATE_COUNT,
+	0,
+	x86_SAVED_STATE32_COUNT,
+	x86_SAVED_STATE64_COUNT,
+	x86_DEBUG_STATE32_COUNT,
+	x86_DEBUG_STATE64_COUNT,
+	x86_DEBUG_STATE_COUNT
 };
 
+zone_t		iss_zone32;		/* zone for 32bit saved_state area */
+zone_t		iss_zone64;		/* zone for 64bit saved_state area */
+zone_t		ids_zone32;		/* zone for 32bit debug_state area */
+zone_t		ids_zone64;		/* zone for 64bit debug_state area */
+
+
 /* Forward */
 
 void		act_machine_throughcall(thread_t thr_act);
@@ -129,6 +142,279 @@ extern void		Thread_continue(void);
 extern void		Load_context(
 				thread_t			thread);
 
+
+static void
+get_exception_state32(thread_t thread, x86_exception_state32_t *es);
+
+static void
+get_exception_state64(thread_t thread, x86_exception_state64_t *es);
+
+static void
+get_thread_state32(thread_t thread, x86_thread_state32_t *ts);
+
+static void
+get_thread_state64(thread_t thread, x86_thread_state64_t *ts);
+
+static int
+set_thread_state32(thread_t thread, x86_thread_state32_t *ts);
+
+static int
+set_thread_state64(thread_t thread, x86_thread_state64_t *ts);
+
+/*
+ * Don't let an illegal value for dr7 get set.	Specifically,
+ * check for undefined settings.  Setting these bit patterns
+ * result in undefined behaviour and can lead to an unexpected
+ * TRCTRAP.
+ */
+static boolean_t
+dr7_is_valid(uint32_t *dr7)
+{
+	int i;
+	uint32_t mask1, mask2;
+
+	/*
+	 * If the DE bit is set in CR4, R/W0-3 can be pattern
+	 * "10B" to indicate i/o reads and write
+	 */
+	if (!(get_cr4() & CR4_DE))
+		for (i = 0, mask1 = 0x3<<16, mask2 = 0x2<<16; i < 4; 
+				i++, mask1 <<= 4, mask2 <<= 4)
+			if ((*dr7 & mask1) == mask2)
+				return (FALSE);
+
+	/*
+	 * len0-3 pattern "10B" is ok for len on 64-bit.
+	 */
+	if (current_cpu_datap()->cpu_is64bit == TRUE)
+		for (i = 0, mask1 = 0x3<<18, mask2 = 0x2<<18; i < 4; 
+				i++, mask1 <<= 4, mask2 <<= 4)
+			if ((*dr7 & mask1) == mask2)
+				return (FALSE);
+
+	/*
+	 * if we are doing an instruction execution break (indicated
+	 * by r/w[x] being "00B"), then the len[x] must also be set
+	 * to "00B"
+	 */
+	for (i = 0; i < 4; i++)
+		if (((((*dr7 >> (16 + i*4))) & 0x3) == 0) &&
+				((((*dr7 >> (18 + i*4))) & 0x3) != 0))
+			return (FALSE);
+
+	/*
+	 * Intel docs have these bits fixed.
+	 */
+	*dr7 |= 0x1 << 10; /* set bit 10 to 1 */
+	*dr7 &= ~(0x1 << 11); /* set bit 11 to 0 */
+	*dr7 &= ~(0x1 << 12); /* set bit 12 to 0 */
+	*dr7 &= ~(0x1 << 14); /* set bit 14 to 0 */
+	*dr7 &= ~(0x1 << 15); /* set bit 15 to 0 */
+
+	/*
+	 * We don't allow anything to set the global breakpoints.
+	 */
+
+	if (*dr7 & 0x2)
+		return (FALSE);
+
+	if (*dr7 & (0x2<<2))
+		return (FALSE);
+
+	if (*dr7 & (0x2<<4))
+		return (FALSE);
+
+	if (*dr7 & (0x2<<6))
+		return (FALSE);
+
+	return (TRUE);
+}
+
+static inline void
+set_live_debug_state32(cpu_data_t *cdp, x86_debug_state32_t *ds)
+{
+	__asm__ volatile ("movl %0,%%db0" : :"r" (ds->dr0));
+	__asm__ volatile ("movl %0,%%db1" : :"r" (ds->dr1));
+	__asm__ volatile ("movl %0,%%db2" : :"r" (ds->dr2));
+	__asm__ volatile ("movl %0,%%db3" : :"r" (ds->dr3));
+	if (cpu_mode_is64bit())
+		cdp->cpu_dr7 = ds->dr7;
+}
+
+extern void set_64bit_debug_regs(x86_debug_state64_t *ds);
+
+static inline void
+set_live_debug_state64(cpu_data_t *cdp, x86_debug_state64_t *ds)
+{
+	/*
+	 * We need to enter 64-bit mode in order to set the full
+	 * width of these registers
+	 */
+	set_64bit_debug_regs(ds);
+	cdp->cpu_dr7 = ds->dr7;
+}
+
+static kern_return_t
+set_debug_state32(thread_t thread, x86_debug_state32_t *ds)
+{
+	x86_debug_state32_t *ids;
+	pcb_t pcb;
+
+	pcb = thread->machine.pcb;
+	ids = pcb->ids;
+
+	if (ids == NULL) {
+		ids = zalloc(ids_zone32);
+		bzero(ids, sizeof *ids);
+
+		simple_lock(&pcb->lock);
+		/* make sure it wasn't already alloc()'d elsewhere */
+		if (pcb->ids == NULL) {
+			pcb->ids = ids;
+			simple_unlock(&pcb->lock);
+		} else {
+			simple_unlock(&pcb->lock);
+			zfree(ids_zone32, ids);
+		}
+	}
+
+	if (!dr7_is_valid(&ds->dr7))
+		goto err;
+
+	/*
+	 * Only allow local breakpoints and make sure they are not
+	 * in the trampoline code.
+	 */
+
+	if (ds->dr7 & 0x1)
+		if (ds->dr0 >= (unsigned long)HIGH_MEM_BASE)
+			goto err;
+
+	if (ds->dr7 & (0x1<<2))
+		if (ds->dr1 >= (unsigned long)HIGH_MEM_BASE)
+			goto err;
+
+	if (ds->dr7 & (0x1<<4))
+		if (ds->dr2 >= (unsigned long)HIGH_MEM_BASE)
+			goto err;
+
+	if (ds->dr7 & (0x1<<6))
+		if (ds->dr3 >= (unsigned long)HIGH_MEM_BASE)
+			goto err;
+
+	ids->dr0 = ds->dr0;
+	ids->dr1 = ds->dr1;
+	ids->dr2 = ds->dr2;
+	ids->dr3 = ds->dr3;
+	ids->dr6 = ds->dr6;
+	ids->dr7 = ds->dr7;
+
+	return (KERN_SUCCESS);
+
+err:
+	return (KERN_INVALID_ARGUMENT);
+}
+
+static kern_return_t
+set_debug_state64(thread_t thread, x86_debug_state64_t *ds)
+{
+	x86_debug_state64_t *ids;
+	pcb_t pcb;
+
+	pcb = thread->machine.pcb;
+	ids = pcb->ids;
+
+	if (ids == NULL) {
+		ids = zalloc(ids_zone64);
+		bzero(ids, sizeof *ids);
+
+		simple_lock(&pcb->lock);
+		/* make sure it wasn't already alloc()'d elsewhere */
+		if (pcb->ids == NULL) {
+			pcb->ids = ids;
+			simple_unlock(&pcb->lock);
+		} else {
+			simple_unlock(&pcb->lock);
+			zfree(ids_zone64, ids);
+		}
+	}
+
+	if (!dr7_is_valid((uint32_t *)&ds->dr7))
+		goto err;
+
+	/*
+	 * Don't allow the user to set debug addresses above their max
+	 * value
+	 */
+	if (ds->dr7 & 0x1)
+		if (ds->dr0 >= VM_MAX_PAGE_ADDRESS)
+			goto err;
+
+	if (ds->dr7 & (0x1<<2))
+		if (ds->dr1 >= VM_MAX_PAGE_ADDRESS)
+			goto err;
+
+	if (ds->dr7 & (0x1<<4))
+		if (ds->dr2 >= VM_MAX_PAGE_ADDRESS)
+			goto err;
+
+	if (ds->dr7 & (0x1<<6))
+		if (ds->dr3 >= VM_MAX_PAGE_ADDRESS)
+			goto err;
+
+	ids->dr0 = ds->dr0;
+	ids->dr1 = ds->dr1;
+	ids->dr2 = ds->dr2;
+	ids->dr3 = ds->dr3;
+	ids->dr6 = ds->dr6;
+	ids->dr7 = ds->dr7;
+
+	return (KERN_SUCCESS);
+
+err:
+	return (KERN_INVALID_ARGUMENT);
+}
+
+static void
+get_debug_state32(thread_t thread, x86_debug_state32_t *ds)
+{
+	x86_debug_state32_t *saved_state;
+
+	saved_state = thread->machine.pcb->ids;
+
+	if (saved_state) {
+		ds->dr0 = saved_state->dr0;
+		ds->dr1 = saved_state->dr1;
+		ds->dr2 = saved_state->dr2;
+		ds->dr3 = saved_state->dr3;
+		ds->dr4 = saved_state->dr4;
+		ds->dr5 = saved_state->dr5;
+		ds->dr6 = saved_state->dr6;
+		ds->dr7 = saved_state->dr7;
+	} else
+		bzero(ds, sizeof *ds);
+}
+
+static void
+get_debug_state64(thread_t thread, x86_debug_state64_t *ds)
+{
+	x86_debug_state64_t *saved_state;
+
+	saved_state = (x86_debug_state64_t *)thread->machine.pcb->ids;
+
+	if (saved_state) {
+		ds->dr0 = saved_state->dr0;
+		ds->dr1 = saved_state->dr1;
+		ds->dr2 = saved_state->dr2;
+		ds->dr3 = saved_state->dr3;
+		ds->dr4 = saved_state->dr4;
+		ds->dr5 = saved_state->dr5;
+		ds->dr6 = saved_state->dr6;
+		ds->dr7 = saved_state->dr7;
+	} else
+		bzero(ds, sizeof *ds);
+}
+
 /*
  * consider_machine_collect:
  *
@@ -145,92 +431,151 @@ consider_machine_adjust(void)
 }
 
 
-// DEBUG
-int	DEBUG_kldt = 0;
-int	DEBUG_uldt = 0;
 
 static void
 act_machine_switch_pcb( thread_t new )
 {
-	pcb_t			pcb = new->machine.pcb;
-	int			mycpu;
-	register iopb_tss_t	tss = pcb->ims.io_tss;
+        pcb_t			pcb = new->machine.pcb;
+	struct real_descriptor	*ldtp;
 	vm_offset_t		pcb_stack_top;
-	register user_ldt_t	uldt = pcb->ims.ldt;
+	vm_offset_t		hi_pcb_stack_top;
+        vm_offset_t		hi_iss;
+	cpu_data_t              *cdp = current_cpu_datap();
 
-        assert(new->kernel_stack != 0);
-        STACK_IEL(new->kernel_stack)->saved_state =
-                &new->machine.pcb->iss;
+	assert(new->kernel_stack != 0);
+	STACK_IEL(new->kernel_stack)->saved_state = pcb->iss;
 
-	/*
-	 *	Save a pointer to the top of the "kernel" stack -
-	 *	actually the place in the PCB where a trap into
-	 *	kernel mode will push the registers.
-	 *	The location depends on V8086 mode.  If we are
-	 *	not in V8086 mode, then a trap into the kernel
-	 *	won`t save the v86 segments, so we leave room.
-	 */
+	if (!cpu_mode_is64bit()) {
+		x86_saved_state32_tagged_t	*hi_iss32;
 
-	pcb_stack_top = (pcb->iss.efl & EFL_VM)
-			? (int) (&pcb->iss + 1)
-			: (int) (&pcb->iss.v86_segs);
-
-	mp_disable_preemption();
-	mycpu = cpu_number();
-
-	if (tss == 0) {
-	    /*
-	     *	No per-thread IO permissions.
-	     *	Use standard kernel TSS.
-	     */
-	    if (!(gdt_desc_p(KERNEL_TSS)->access & ACC_TSS_BUSY))
-		set_tr(KERNEL_TSS);
-	    current_ktss()->esp0 = pcb_stack_top;
-	}
-	else {
-	    /*
-	     * Set the IO permissions.  Use this thread`s TSS.
-	     */
-	    *gdt_desc_p(USER_TSS)
-	    	= *(struct real_descriptor *)tss->iopb_desc;
-	    tss->tss.esp0 = pcb_stack_top;
-	    set_tr(USER_TSS);
-	    gdt_desc_p(KERNEL_TSS)->access &= ~ ACC_TSS_BUSY;
-	}
+		/*
+		 *	Save a pointer to the top of the "kernel" stack -
+		 *	actually the place in the PCB where a trap into
+		 *	kernel mode will push the registers.
+		 */
+		hi_iss = (vm_offset_t)((unsigned long)
+			pmap_cpu_high_map_vaddr(cpu_number(), HIGH_CPU_ISS0) |
+			((unsigned long)pcb->iss & PAGE_MASK));
+	 
+		cdp->cpu_hi_iss = (void *)hi_iss;
 
-	/*
-	 * Set the thread`s LDT or LDT entry.
-	 */
-	if (uldt == 0) {
-	    struct real_descriptor *ldtp;
-	    /*
-	     * Use system LDT.
-	     */
-            // Set up the tasks specific ldt entries if extant
-	    ldtp = (struct real_descriptor *)current_ldt();
-	    ldtp[sel_idx(USER_CTHREAD)] = pcb->cthread_desc;
-            if (pcb->uldt_selector != 0)
-                ldtp[sel_idx(pcb->uldt_selector)] = pcb->uldt_desc;
-	    set_ldt(KERNEL_LDT);
-	}
-	else {
-	    /*
-	     * Thread has its own LDT. // THIS SHOULD BE REMOVED!!!!
-	     */
-	    *gdt_desc_p(USER_LDT) = uldt->desc;
-	    set_ldt(USER_LDT);
-            /*debug*/
-	    if ((DEBUG_uldt++ % 0x7fff) == 0)
-		printf("KERNEL----> setting user ldt");
+		pmap_high_map(pcb->iss_pte0, HIGH_CPU_ISS0);
+		pmap_high_map(pcb->iss_pte1, HIGH_CPU_ISS1);
+
+		hi_iss32 = (x86_saved_state32_tagged_t *) hi_iss;
+		assert(hi_iss32->tag == x86_SAVED_STATE32);
+
+		hi_pcb_stack_top = (int) (hi_iss32 + 1);
+
+		/*
+		 * For fast syscall, top of interrupt stack points to pcb stack
+		 */
+		*(vm_offset_t *) current_sstk() = hi_pcb_stack_top;
+
+		current_ktss()->esp0 = hi_pcb_stack_top;
+/* XXX: This check is performed against the thread save state flavor rather than the
+ * task's 64-bit feature flag because of the thread/task 64-bit state divergence
+ * that can arise in task_set_64bit() on x86. When that is addressed, we can
+ * revert to checking the task 64 bit feature flag. The assert below is retained
+ * for that reason.
+ */
+	} else if (is_saved_state64(pcb->iss)) {
+	        x86_saved_state64_tagged_t	*iss64;
+		vm_offset_t			isf;
+
+		assert(is_saved_state64(pcb->iss));
+						   
+		iss64 = (x86_saved_state64_tagged_t *) pcb->iss;
 	
+		/*
+		 * Set pointer to PCB's interrupt stack frame in cpu data.
+		 * Used by syscall and double-fault trap handlers.
+		 */
+		isf = (vm_offset_t) &iss64->state.isf;
+		cdp->cpu_uber.cu_isf = UBER64(isf);
+		pcb_stack_top = (vm_offset_t) (iss64 + 1);
+		/* require 16-byte alignment */
+		assert((pcb_stack_top & 0xF) == 0);
+		/* Interrupt stack is pcb */
+		current_ktss64()->rsp0 = UBER64(pcb_stack_top);
+
+		/*
+		 * Top of temporary sysenter stack points to pcb stack.
+		 * Although this is not normally used by 64-bit users,
+		 * it needs to be set in case a sysenter is attempted.
+		 */
+		*current_sstk64() = UBER64(pcb_stack_top);
+
+		cdp->cpu_task_map = new->map->pmap->pm_kernel_cr3 ?
+					TASK_MAP_64BIT_SHARED : TASK_MAP_64BIT;
+
+		/*
+		 * Enable the 64-bit user code segment, USER64_CS.
+		 */
+		ldt_desc_p(USER64_CS)->access |= ACC_PL_U;
+
+	} else {
+		x86_saved_state_compat32_t	*iss32compat;
+		vm_offset_t			isf;
+
+		assert(is_saved_state32(pcb->iss));
+		iss32compat = (x86_saved_state_compat32_t *) pcb->iss;
+
+		pcb_stack_top = (int) (iss32compat + 1);
+		/* require 16-byte alignment */
+		assert((pcb_stack_top & 0xF) == 0);
+
+		/*
+		 * Set pointer to PCB's interrupt stack frame in cpu data.
+		 * Used by debug trap handler.
+		 */
+		isf = (vm_offset_t) &iss32compat->isf64;
+		cdp->cpu_uber.cu_isf = UBER64(isf);
+
+		/* Top of temporary sysenter stack points to pcb stack */
+		*current_sstk64() = UBER64(pcb_stack_top);
+
+		/* Interrupt stack is pcb */
+		current_ktss64()->rsp0 = UBER64(pcb_stack_top);
+
+		cdp->cpu_task_map = TASK_MAP_32BIT;
+
+		/*
+		 * Disable USER64_CS
+		 */
+		ldt_desc_p(USER64_CS)->access &= ~ACC_PL_U;
 	}
 
-	mp_enable_preemption();
 	/*
-	 * Load the floating-point context, if necessary.
+	 * Set the thread`s cthread (a.k.a pthread)
+	 * For 32-bit user this involves setting the USER_CTHREAD
+	 * descriptor in the LDT to point to the cthread data.
+	 * The involves copying in the pre-initialized descriptor.
+	 */ 
+	ldtp = (struct real_descriptor *)current_ldt();
+	ldtp[sel_idx(USER_CTHREAD)] = pcb->cthread_desc;
+	if (pcb->uldt_selector != 0)
+		ldtp[sel_idx(pcb->uldt_selector)] = pcb->uldt_desc;
+	/*
+	 * For 64-bit, we additionally set the 64-bit User GS base
+	 * address. On return to 64-bit user, the GS.Base MSR will be written.
 	 */
-	fpu_load_context(pcb);
+	cdp->cpu_uber.cu_user_gs_base = pcb->cthread_self;
 
+	/*
+	 * Set the thread`s LDT or LDT entry.
+	 */
+	if (new->task == TASK_NULL || new->task->i386_ldt == 0) {
+		/*
+		 * Use system LDT.
+		 */
+	       	ml_cpu_set_ldt(KERNEL_LDT);
+	} else {
+		/*
+		 * Task has its own LDT.
+		 */
+		user_ldt_set(new);
+	}
 }
 
 /*
@@ -240,6 +585,7 @@ void
 machine_load_context(
 	thread_t		new)
 {
+	new->machine.specFlags |= OnProc;
 	act_machine_switch_pcb(new);
 	Load_context(new);
 }
@@ -264,23 +610,22 @@ machine_switch_context(
 	 */
 	fpu_save_context(old);
 
+	old->machine.specFlags &= ~OnProc;
+	new->machine.specFlags |= OnProc;
+
 	/*
 	 *	Switch address maps if need be, even if not switching tasks.
 	 *	(A server activation may be "borrowing" a client map.)
 	 */
-    {
-	int	mycpu = cpu_number();
-
-	PMAP_SWITCH_CONTEXT(old, new, mycpu)
-    }
+	PMAP_SWITCH_CONTEXT(old, new, cpu_number())
 
 	/*
 	 *	Load the rest of the user state for the new thread
 	 */
 	act_machine_switch_pcb(new);
-	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED) | DBG_FUNC_NONE,
-		     (int)old, (int)new, old->sched_pri, new->sched_pri, 0);
-	old->continuation = NULL;
+	KERNEL_DEBUG_CONSTANT(
+		MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED) | DBG_FUNC_NONE,
+		(int)old, (int)new, old->sched_pri, new->sched_pri, 0);
 	return(Switch_context(old, continuation, new));
 }
 
@@ -303,12 +648,226 @@ kern_return_t
 machine_thread_state_initialize(
 	thread_t thread)
 {
-#pragma unused (thread)
+        /*
+	 * If there's an fpu save area, free it.
+	 * The initialized state will then be lazily faulted-in, if required.
+	 * And if we're target, re-arm the no-fpu trap.
+	 */
+        if (thread->machine.pcb->ifps) {
+	        (void) fpu_set_fxstate(thread, NULL);
 
-    return  KERN_SUCCESS;
+		if (thread == current_thread())
+		        clear_fpu();
+	}
+	return KERN_SUCCESS;
 }
- 
- 
+
+uint32_t
+get_eflags_exportmask(void)
+{
+	return EFL_USER_SET;
+}
+
+/*
+ * x86_SAVED_STATE32	 - internal save/restore general register state on 32/64 bit processors
+ *			   for 32bit tasks only
+ * x86_SAVED_STATE64	 - internal save/restore general register state on 64 bit processors
+ *			   for 64bit tasks only
+ * x86_THREAD_STATE32	 - external set/get general register state on 32/64 bit processors
+ *			   for 32bit tasks only
+ * x86_THREAD_STATE64	 - external set/get general register state on 64 bit processors
+ *			   for 64bit tasks only
+ * x86_SAVED_STATE	 - external set/get general register state on 32/64 bit processors
+ *			   for either 32bit or 64bit tasks
+ * x86_FLOAT_STATE32	 - internal/external save/restore float and xmm state on 32/64 bit processors
+ *			   for 32bit tasks only
+ * x86_FLOAT_STATE64	 - internal/external save/restore float and xmm state on 64 bit processors
+ *			   for 64bit tasks only
+ * x86_FLOAT_STATE	 - external save/restore float and xmm state on 32/64 bit processors
+ *			   for either 32bit or 64bit tasks
+ * x86_EXCEPTION_STATE32 - external get exception state on 32/64 bit processors
+ *			   for 32bit tasks only
+ * x86_EXCEPTION_STATE64 - external get exception state on 64 bit processors
+ *			   for 64bit tasks only
+ * x86_EXCEPTION_STATE   - external get exception state on 323/64 bit processors
+ *			   for either 32bit or 64bit tasks
+ */
+
+
+static void
+get_exception_state64(thread_t thread, x86_exception_state64_t *es)
+{
+        x86_saved_state64_t *saved_state;
+
+        saved_state = USER_REGS64(thread);
+
+	es->trapno = saved_state->isf.trapno;
+	es->err = saved_state->isf.err;
+	es->faultvaddr = saved_state->cr2;
+}		
+
+static void
+get_exception_state32(thread_t thread, x86_exception_state32_t *es)
+{
+        x86_saved_state32_t *saved_state;
+
+        saved_state = USER_REGS32(thread);
+
+	es->trapno = saved_state->trapno;
+	es->err = saved_state->err;
+	es->faultvaddr = saved_state->cr2;
+}		
+
+
+static int
+set_thread_state32(thread_t thread, x86_thread_state32_t *ts)
+{
+        x86_saved_state32_t	*saved_state;
+
+	saved_state = USER_REGS32(thread);
+
+	/*
+	 * Scrub segment selector values:
+	 */
+	if (ts->cs != USER_CS) ts->cs = USER_CS;
+	if (ts->ss == 0) ts->ss = USER_DS;
+	if (ts->ds == 0) ts->ds = USER_DS;
+	if (ts->es == 0) ts->es = USER_DS;
+
+	/* Check segment selectors are safe */
+	if (!valid_user_segment_selectors(ts->cs,
+					  ts->ss,
+					  ts->ds,
+					  ts->es,
+					  ts->fs,
+					  ts->gs))
+		return(KERN_INVALID_ARGUMENT);
+
+	saved_state->eax = ts->eax;
+	saved_state->ebx = ts->ebx;
+	saved_state->ecx = ts->ecx;
+	saved_state->edx = ts->edx;
+	saved_state->edi = ts->edi;
+	saved_state->esi = ts->esi;
+	saved_state->ebp = ts->ebp;
+	saved_state->uesp = ts->esp;
+	saved_state->efl = (ts->eflags & ~EFL_USER_CLEAR) | EFL_USER_SET;
+	saved_state->eip = ts->eip;
+	saved_state->cs = ts->cs;
+	saved_state->ss = ts->ss;
+	saved_state->ds = ts->ds;
+	saved_state->es = ts->es;
+	saved_state->fs = ts->fs;
+	saved_state->gs = ts->gs;
+
+	/*
+	 * If the trace trap bit is being set,
+	 * ensure that the user returns via iret
+	 * - which is signaled thusly:
+	 */
+	if ((saved_state->efl & EFL_TF) && saved_state->cs == SYSENTER_CS)
+		saved_state->cs = SYSENTER_TF_CS;
+
+	return(KERN_SUCCESS);
+}
+
+static int
+set_thread_state64(thread_t thread, x86_thread_state64_t *ts)
+{
+        x86_saved_state64_t	*saved_state;
+
+	saved_state = USER_REGS64(thread);
+
+	if (!IS_USERADDR64_CANONICAL(ts->rsp) ||
+	    !IS_USERADDR64_CANONICAL(ts->rip))
+		return(KERN_INVALID_ARGUMENT);
+
+	saved_state->r8 = ts->r8;
+	saved_state->r9 = ts->r9;
+	saved_state->r10 = ts->r10;
+	saved_state->r11 = ts->r11;
+	saved_state->r12 = ts->r12;
+	saved_state->r13 = ts->r13;
+	saved_state->r14 = ts->r14;
+	saved_state->r15 = ts->r15;
+	saved_state->rax = ts->rax;
+	saved_state->rax = ts->rax;
+	saved_state->rbx = ts->rbx;
+	saved_state->rcx = ts->rcx;
+	saved_state->rdx = ts->rdx;
+	saved_state->rdi = ts->rdi;
+	saved_state->rsi = ts->rsi;
+	saved_state->rbp = ts->rbp;
+	saved_state->isf.rsp = ts->rsp;
+	saved_state->isf.rflags = (ts->rflags & ~EFL_USER_CLEAR) | EFL_USER_SET;
+	saved_state->isf.rip = ts->rip;
+	saved_state->isf.cs = USER64_CS;
+	saved_state->fs = ts->fs;
+	saved_state->gs = ts->gs;
+
+	return(KERN_SUCCESS);
+}
+
+
+
+static void
+get_thread_state32(thread_t thread, x86_thread_state32_t *ts)
+{
+        x86_saved_state32_t	*saved_state;
+
+	saved_state = USER_REGS32(thread);
+
+	ts->eax = saved_state->eax;
+	ts->ebx = saved_state->ebx;
+	ts->ecx = saved_state->ecx;
+	ts->edx = saved_state->edx;
+	ts->edi = saved_state->edi;
+	ts->esi = saved_state->esi;
+	ts->ebp = saved_state->ebp;
+	ts->esp = saved_state->uesp;
+	ts->eflags = saved_state->efl;
+	ts->eip = saved_state->eip;
+	ts->cs = saved_state->cs;
+	ts->ss = saved_state->ss;
+	ts->ds = saved_state->ds;
+	ts->es = saved_state->es;
+	ts->fs = saved_state->fs;
+	ts->gs = saved_state->gs;
+}
+
+
+static void
+get_thread_state64(thread_t thread, x86_thread_state64_t *ts)
+{
+        x86_saved_state64_t	*saved_state;
+
+	saved_state = USER_REGS64(thread);
+
+	ts->r8 = saved_state->r8;
+	ts->r9 = saved_state->r9;
+	ts->r10 = saved_state->r10;
+	ts->r11 = saved_state->r11;
+	ts->r12 = saved_state->r12;
+	ts->r13 = saved_state->r13;
+	ts->r14 = saved_state->r14;
+	ts->r15 = saved_state->r15;
+	ts->rax = saved_state->rax;
+	ts->rbx = saved_state->rbx;
+	ts->rcx = saved_state->rcx;
+	ts->rdx = saved_state->rdx;
+	ts->rdi = saved_state->rdi;
+	ts->rsi = saved_state->rsi;
+	ts->rbp = saved_state->rbp;
+	ts->rsp = saved_state->isf.rsp;
+	ts->rflags = saved_state->isf.rflags;
+	ts->rip = saved_state->isf.rip;
+	ts->cs = saved_state->isf.cs;
+	ts->fs = saved_state->fs;
+	ts->gs = saved_state->gs;
+}
+
+
+
 /*
  *	act_machine_set_state:
  *
@@ -322,48 +881,29 @@ machine_thread_set_state(
 	thread_state_t tstate,
 	mach_msg_type_number_t count)
 {
-	int kernel_act = 0;
-
-	switch (flavor) {
-	    case THREAD_SYSCALL_STATE:
-	    {
-		register struct thread_syscall_state *state;
-		register struct i386_saved_state *saved_state = USER_REGS(thr_act);
-
-		state = (struct thread_syscall_state *) tstate;
-		saved_state->eax = state->eax;
-		saved_state->edx = state->edx;
-		if (kernel_act)
-			saved_state->efl = state->efl;
-		else
-			saved_state->efl = (state->efl & ~EFL_USER_CLEAR) | EFL_USER_SET;
-		saved_state->eip = state->eip;
-		saved_state->uesp = state->esp;
-		break;
-	    }
 
-	    case i386_SAVED_STATE:
+	switch (flavor)
+	{
+	    case x86_SAVED_STATE32:
 	    {
-		register struct i386_saved_state	*state;
-		register struct i386_saved_state	*saved_state;
+		x86_saved_state32_t	*state;
+		x86_saved_state32_t	*saved_state;
 
-		if (count < i386_SAVED_STATE_COUNT) {
-		    return(KERN_INVALID_ARGUMENT);
-		}
+		if (count < x86_SAVED_STATE32_COUNT)
+		        return(KERN_INVALID_ARGUMENT);
 
-		state = (struct i386_saved_state *) tstate;
+		state = (x86_saved_state32_t *) tstate;
 
 		/* Check segment selectors are safe */
-		if (!kernel_act &&
-		    !valid_user_segment_selectors(state->cs,
+		if (!valid_user_segment_selectors(state->cs,
 						  state->ss,
 						  state->ds,
 						  state->es,
 						  state->fs,
 						  state->gs))
-		    return KERN_INVALID_ARGUMENT;
+		        return KERN_INVALID_ARGUMENT;
 		
-		saved_state = USER_REGS(thr_act);
+		saved_state = USER_REGS32(thr_act);
 
 		/*
 		 * General registers
@@ -377,271 +917,248 @@ machine_thread_set_state(
 		saved_state->ecx = state->ecx;
 		saved_state->eax = state->eax;
 		saved_state->eip = state->eip;
-		if (kernel_act)
-			saved_state->efl = state->efl;
-		else
-			saved_state->efl = (state->efl & ~EFL_USER_CLEAR)
-						| EFL_USER_SET;
+
+		saved_state->efl = (state->efl & ~EFL_USER_CLEAR) | EFL_USER_SET;
 
 		/*
-		 * Segment registers.  Set differently in V8086 mode.
+		 * If the trace trap bit is being set,
+		 * ensure that the user returns via iret
+		 * - which is signaled thusly:
 		 */
-		if (state->efl & EFL_VM) {
-		    /*
-		     * Set V8086 mode segment registers.
-		     */
-		    saved_state->cs = state->cs & 0xffff;
-		    saved_state->ss = state->ss & 0xffff;
-		    saved_state->v86_segs.v86_ds = state->ds & 0xffff;
-		    saved_state->v86_segs.v86_es = state->es & 0xffff;
-		    saved_state->v86_segs.v86_fs = state->fs & 0xffff;
-		    saved_state->v86_segs.v86_gs = state->gs & 0xffff;
-
-		    /*
-		     * Zero protected mode segment registers.
-		     */
-		    saved_state->ds = 0;
-		    saved_state->es = 0;
-		    saved_state->fs = 0;
-		    saved_state->gs = 0;
-
-		    if (thr_act->machine.pcb->ims.v86s.int_table) {
-			/*
-			 * Hardware assist on.
-			 */
-			thr_act->machine.pcb->ims.v86s.flags =
-			    state->efl & (EFL_TF | EFL_IF);
-		    }
-		}
-		else if (kernel_act) {
-		    /*
-		     * 386 mode.  Set segment registers for flat
-		     * 32-bit address space.
-		     */
-		  saved_state->cs = KERNEL_CS;
-		  saved_state->ss = KERNEL_DS;
-		  saved_state->ds = KERNEL_DS;
-		  saved_state->es = KERNEL_DS;
-		  saved_state->fs = KERNEL_DS;
-		  saved_state->gs = CPU_DATA_GS;
-		}
-		else {
-		    /*
-		     * User setting segment registers.
-		     * Code and stack selectors have already been
-		     * checked.  Others will be reset by 'iret'
-		     * if they are not valid.
-		     */
-		    saved_state->cs = state->cs;
-		    saved_state->ss = state->ss;
-		    saved_state->ds = state->ds;
-		    saved_state->es = state->es;
-		    saved_state->fs = state->fs;
-		    saved_state->gs = state->gs;
-		}
+		if ((saved_state->efl & EFL_TF) && state->cs == SYSENTER_CS)
+			state->cs = SYSENTER_TF_CS;
+
+		/*
+		 * User setting segment registers.
+		 * Code and stack selectors have already been
+		 * checked.  Others will be reset by 'iret'
+		 * if they are not valid.
+		 */
+		saved_state->cs = state->cs;
+		saved_state->ss = state->ss;
+		saved_state->ds = state->ds;
+		saved_state->es = state->es;
+		saved_state->fs = state->fs;
+		saved_state->gs = state->gs;
 		break;
 	    }
 
-	    case i386_NEW_THREAD_STATE:
-	    case i386_REGS_SEGS_STATE:
+	    case x86_SAVED_STATE64:
 	    {
-		register struct i386_new_thread_state	*state;
-		register struct i386_saved_state	*saved_state;
+		x86_saved_state64_t	*state;
+		x86_saved_state64_t	*saved_state;
 
-		if (count < i386_NEW_THREAD_STATE_COUNT) {
-		    return(KERN_INVALID_ARGUMENT);
-		}
+		if (count < x86_SAVED_STATE64_COUNT)
+		        return(KERN_INVALID_ARGUMENT);
+
+		state = (x86_saved_state64_t *) tstate;
 
-		state = (struct i386_new_thread_state *) tstate;
-
-		if (flavor == i386_REGS_SEGS_STATE) {
-		    /*
-		     * Code and stack selectors must not be null,
-		     * and must have user protection levels.
-		     * Only the low 16 bits are valid.
-		     */
-		    state->cs &= 0xffff;
-		    state->ss &= 0xffff;
-		    state->ds &= 0xffff;
-		    state->es &= 0xffff;
-		    state->fs &= 0xffff;
-		    state->gs &= 0xffff;
-
-		    if (!kernel_act &&
-			!valid_user_segment_selectors(state->cs,
-						      state->ss,
-						      state->ds,
-						      state->es,
-						      state->fs,
-						      state->gs))
+		/* Verify that the supplied code segment selector is
+		 * valid. In 64-bit mode, the FS and GS segment overrides
+		 * use the FS.base and GS.base MSRs to calculate
+		 * base addresses, and the trampolines don't directly
+		 * restore the segment registers--hence they are no
+		 * longer relevant for validation.
+		 */
+		if (!valid_user_code_selector(state->isf.cs))
+		        return KERN_INVALID_ARGUMENT;
+		
+		/* Check pc and stack are canonical addresses */
+		if (!IS_USERADDR64_CANONICAL(state->isf.rsp) ||
+		    !IS_USERADDR64_CANONICAL(state->isf.rip))
 			return KERN_INVALID_ARGUMENT;
-		}
 
-		saved_state = USER_REGS(thr_act);
+		saved_state = USER_REGS64(thr_act);
 
 		/*
 		 * General registers
 		 */
-		saved_state->edi = state->edi;
-		saved_state->esi = state->esi;
-		saved_state->ebp = state->ebp;
-		saved_state->uesp = state->uesp;
-		saved_state->ebx = state->ebx;
-		saved_state->edx = state->edx;
-		saved_state->ecx = state->ecx;
-		saved_state->eax = state->eax;
-		saved_state->eip = state->eip;
-		if (kernel_act)
-			saved_state->efl = state->efl;
-		else
-			saved_state->efl = (state->efl & ~EFL_USER_CLEAR)
-						| EFL_USER_SET;
-
-		/*
-		 * Segment registers.  Set differently in V8086 mode.
+		saved_state->r8 = state->r8;
+		saved_state->r9 = state->r9;
+		saved_state->r10 = state->r10;
+		saved_state->r11 = state->r11;
+		saved_state->r12 = state->r12;
+		saved_state->r13 = state->r13;
+		saved_state->r14 = state->r14;
+		saved_state->r15 = state->r15;
+		saved_state->rdi = state->rdi;
+		saved_state->rsi = state->rsi;
+		saved_state->rbp = state->rbp;
+		saved_state->rbx = state->rbx;
+		saved_state->rdx = state->rdx;
+		saved_state->rcx = state->rcx;
+		saved_state->rax = state->rax;
+		saved_state->isf.rsp = state->isf.rsp;
+		saved_state->isf.rip = state->isf.rip;
+
+		saved_state->isf.rflags = (state->isf.rflags & ~EFL_USER_CLEAR) | EFL_USER_SET;
+
+	        /*
+		 * User setting segment registers.
+		 * Code and stack selectors have already been
+		 * checked.  Others will be reset by 'sys'
+		 * if they are not valid.
 		 */
-		if (state->efl & EFL_VM) {
-		    /*
-		     * Set V8086 mode segment registers.
-		     */
-		    saved_state->cs = state->cs & 0xffff;
-		    saved_state->ss = state->ss & 0xffff;
-		    saved_state->v86_segs.v86_ds = state->ds & 0xffff;
-		    saved_state->v86_segs.v86_es = state->es & 0xffff;
-		    saved_state->v86_segs.v86_fs = state->fs & 0xffff;
-		    saved_state->v86_segs.v86_gs = state->gs & 0xffff;
-
-		    /*
-		     * Zero protected mode segment registers.
-		     */
-		    saved_state->ds = 0;
-		    saved_state->es = 0;
-		    saved_state->fs = 0;
-		    saved_state->gs = 0;
-
-		    if (thr_act->machine.pcb->ims.v86s.int_table) {
-			/*
-			 * Hardware assist on.
-			 */
-			thr_act->machine.pcb->ims.v86s.flags =
-			    state->efl & (EFL_TF | EFL_IF);
-		    }
-		}
-		else if (flavor == i386_NEW_THREAD_STATE && kernel_act) {
-		    /*
-		     * 386 mode.  Set segment registers for flat
-		     * 32-bit address space.
-		     */
-		  saved_state->cs = KERNEL_CS;
-		  saved_state->ss = KERNEL_DS;
-		  saved_state->ds = KERNEL_DS;
-		  saved_state->es = KERNEL_DS;
-		  saved_state->fs = KERNEL_DS;
-		  saved_state->gs = CPU_DATA_GS;
-		}
-		else {
-		    /*
-		     * User setting segment registers.
-		     * Code and stack selectors have already been
-		     * checked.  Others will be reset by 'iret'
-		     * if they are not valid.
-		     */
-		    saved_state->cs = state->cs;
-		    saved_state->ss = state->ss;
-		    saved_state->ds = state->ds;
-		    saved_state->es = state->es;
-		    saved_state->fs = state->fs;
-		    saved_state->gs = state->gs;
-		}
+	        saved_state->isf.cs = state->isf.cs;
+		saved_state->isf.ss = state->isf.ss;
+		saved_state->fs = state->fs;
+		saved_state->gs = state->gs;
+
 		break;
 	    }
 
-	    case i386_FLOAT_STATE: {
-		if (count < i386_old_FLOAT_STATE_COUNT)
+	    case x86_FLOAT_STATE32:
+	    {
+		if (count != x86_FLOAT_STATE32_COUNT)
+			return(KERN_INVALID_ARGUMENT);
+
+		if (thread_is_64bit(thr_act))
 			return(KERN_INVALID_ARGUMENT);
-                if (count < i386_FLOAT_STATE_COUNT)
-                    return fpu_set_state(thr_act,(struct i386_float_state*)tstate);
-                else return fpu_set_fxstate(thr_act,(struct i386_float_state*)tstate);
+
+		return fpu_set_fxstate(thr_act, tstate);
 	    }
 
-	    /*
-	     * Temporary - replace by i386_io_map
-	     */
-	    case i386_ISA_PORT_MAP_STATE: {
-		if (count < i386_ISA_PORT_MAP_STATE_COUNT)
+	    case x86_FLOAT_STATE64:
+	    {
+		if (count != x86_FLOAT_STATE64_COUNT)
 			return(KERN_INVALID_ARGUMENT);
 
-		break;
+		if ( !thread_is_64bit(thr_act))
+			return(KERN_INVALID_ARGUMENT);
+
+		return fpu_set_fxstate(thr_act, tstate);
 	    }
 
-	    case i386_V86_ASSIST_STATE:
+	    case x86_FLOAT_STATE:
 	    {
-		register struct i386_v86_assist_state *state;
-		vm_offset_t	int_table;
-		int		int_count;
+	        x86_float_state_t	*state;
 
-		if (count < i386_V86_ASSIST_STATE_COUNT)
-		    return KERN_INVALID_ARGUMENT;
+		if (count != x86_FLOAT_STATE_COUNT)
+			return(KERN_INVALID_ARGUMENT);
 
-		state = (struct i386_v86_assist_state *) tstate;
-		int_table = state->int_table;
-		int_count = state->int_count;
+		state = (x86_float_state_t *)tstate;
 
-		if (int_table >= VM_MAX_ADDRESS ||
-		    int_table +
-			int_count * sizeof(struct v86_interrupt_table)
-			    > VM_MAX_ADDRESS)
-		    return KERN_INVALID_ARGUMENT;
+		if (state->fsh.flavor == x86_FLOAT_STATE64 && state->fsh.count == x86_FLOAT_STATE64_COUNT &&
+		    thread_is_64bit(thr_act)) {
+		        return fpu_set_fxstate(thr_act, (thread_state_t)&state->ufs.fs64);
+		}
+		if (state->fsh.flavor == x86_FLOAT_STATE32 && state->fsh.count == x86_FLOAT_STATE32_COUNT &&
+		    !thread_is_64bit(thr_act)) {
+		        return fpu_set_fxstate(thr_act, (thread_state_t)&state->ufs.fs32);
+		}
+		return(KERN_INVALID_ARGUMENT);
+	    }
 
-		thr_act->machine.pcb->ims.v86s.int_table = int_table;
-		thr_act->machine.pcb->ims.v86s.int_count = int_count;
 
-		thr_act->machine.pcb->ims.v86s.flags =
-			USER_REGS(thr_act)->efl & (EFL_TF | EFL_IF);
+
+	    case OLD_i386_THREAD_STATE: 
+	    case x86_THREAD_STATE32: 
+	    {
+		if (count != x86_THREAD_STATE32_COUNT)
+			return(KERN_INVALID_ARGUMENT);
+
+		if (thread_is_64bit(thr_act))
+			return(KERN_INVALID_ARGUMENT);
+		
+		return set_thread_state32(thr_act, (x86_thread_state32_t *)tstate);
+	    }
+
+	    case x86_THREAD_STATE64: 
+	    {
+		if (count != x86_THREAD_STATE64_COUNT)
+			return(KERN_INVALID_ARGUMENT);
+
+		if ( !thread_is_64bit(thr_act))
+			return(KERN_INVALID_ARGUMENT);
+
+		return set_thread_state64(thr_act, (x86_thread_state64_t *)tstate);
+	    }
+
+	    case x86_THREAD_STATE: 
+	    {
+	        x86_thread_state_t	*state;
+
+		if (count != x86_THREAD_STATE_COUNT)
+			return(KERN_INVALID_ARGUMENT);
+
+		state = (x86_thread_state_t *)tstate;
+
+		if (state->tsh.flavor == x86_THREAD_STATE64 && state->tsh.count == x86_THREAD_STATE64_COUNT &&
+		    thread_is_64bit(thr_act)) {
+		        return set_thread_state64(thr_act, &state->uts.ts64);
+		} else if (state->tsh.flavor == x86_THREAD_STATE32 && state->tsh.count == x86_THREAD_STATE32_COUNT &&
+			   !thread_is_64bit(thr_act)) {
+		        return set_thread_state32(thr_act, &state->uts.ts32);
+		} else
+		        return(KERN_INVALID_ARGUMENT);
+
 		break;
 	    }
+	case x86_DEBUG_STATE32:
+	{
+		x86_debug_state32_t *state;
+		kern_return_t ret;
 
-	case i386_THREAD_STATE: {
-		struct i386_saved_state	*saved_state;
-		i386_thread_state_t	*state25;
-
-		saved_state = USER_REGS(thr_act);
-		state25 = (i386_thread_state_t *)tstate;
-
-	    	saved_state->eax = state25->eax;
-	    	saved_state->ebx = state25->ebx;
-	    	saved_state->ecx = state25->ecx;
-	    	saved_state->edx = state25->edx;
-	    	saved_state->edi = state25->edi;
-	    	saved_state->esi = state25->esi;
-	    	saved_state->ebp = state25->ebp;
-		saved_state->uesp = state25->esp;
-		saved_state->efl = (state25->eflags & ~EFL_USER_CLEAR)
-						| EFL_USER_SET;
-	    	saved_state->eip = state25->eip;
-		saved_state->cs = USER_CS;	/* FIXME? */
-		saved_state->ss = USER_DS;
-		saved_state->ds = USER_DS;
-		saved_state->es = USER_DS;
-		saved_state->fs = state25->fs;
-		saved_state->gs = state25->gs;
+		if (thread_is_64bit(thr_act))
+			return(KERN_INVALID_ARGUMENT);
+
+		state = (x86_debug_state32_t *)tstate;
+
+		ret = set_debug_state32(thr_act, state);
+
+		return ret;
 	}
-		break;
+	case x86_DEBUG_STATE64:
+	{
+		x86_debug_state64_t *state;
+		kern_return_t ret;
 
-	    default:
+		if (!thread_is_64bit(thr_act))
+			return(KERN_INVALID_ARGUMENT);
+
+		state = (x86_debug_state64_t *)tstate;
+
+		ret = set_debug_state64(thr_act, state);
+
+		return ret;
+	}
+	case x86_DEBUG_STATE:
+	{
+		x86_debug_state_t *state;
+		kern_return_t ret = KERN_INVALID_ARGUMENT;
+
+		if (count != x86_DEBUG_STATE_COUNT)
+			return (KERN_INVALID_ARGUMENT);
+
+		state = (x86_debug_state_t *)tstate;
+		if (state->dsh.flavor == x86_DEBUG_STATE64 &&
+				state->dsh.count == x86_DEBUG_STATE64_COUNT &&
+				thread_is_64bit(thr_act)) {
+			ret = set_debug_state64(thr_act, &state->uds.ds64);
+		}
+		else
+			if (state->dsh.flavor == x86_DEBUG_STATE32 &&
+			    state->dsh.count == x86_DEBUG_STATE32_COUNT &&
+			    !thread_is_64bit(thr_act)) {
+				ret = set_debug_state32(thr_act, &state->uds.ds32);
+		}
+		return ret;
+	}
+	default:
 		return(KERN_INVALID_ARGUMENT);
 	}
 
 	return(KERN_SUCCESS);
 }
 
+
+
 /*
  *	thread_getstatus:
  *
  *	Get the status of the specified thread.
  */
 
-
 kern_return_t
 machine_thread_get_state(
 	thread_t thr_act,
@@ -651,227 +1168,300 @@ machine_thread_get_state(
 {
 	switch (flavor)  {
 
-	    case i386_SAVED_STATE:
+	    case THREAD_STATE_FLAVOR_LIST:
+	    {
+		if (*count < 3)
+		        return (KERN_INVALID_ARGUMENT);
+
+	        tstate[0] = i386_THREAD_STATE;
+		tstate[1] = i386_FLOAT_STATE;
+		tstate[2] = i386_EXCEPTION_STATE;
+
+		*count = 3;
+		break;
+	    }
+
+	    case THREAD_STATE_FLAVOR_LIST_NEW:
+	    {
+		if (*count < 4)
+		        return (KERN_INVALID_ARGUMENT);
+
+	        tstate[0] = x86_THREAD_STATE;
+		tstate[1] = x86_FLOAT_STATE;
+		tstate[2] = x86_EXCEPTION_STATE;
+		tstate[3] = x86_DEBUG_STATE;
+
+		*count = 4;
+		break;
+	    }
+
+	    case x86_SAVED_STATE32:
 	    {
-		register struct i386_saved_state	*state;
-		register struct i386_saved_state	*saved_state;
+		x86_saved_state32_t	*state;
+		x86_saved_state32_t	*saved_state;
 
-		if (*count < i386_SAVED_STATE_COUNT)
-		    return(KERN_INVALID_ARGUMENT);
+		if (*count < x86_SAVED_STATE32_COUNT)
+		        return(KERN_INVALID_ARGUMENT);
 
-		state = (struct i386_saved_state *) tstate;
-		saved_state = USER_REGS(thr_act);
+		state = (x86_saved_state32_t *) tstate;
+		saved_state = USER_REGS32(thr_act);
 
 		/*
 		 * First, copy everything:
 		 */
 		*state = *saved_state;
+		state->ds = saved_state->ds & 0xffff;
+		state->es = saved_state->es & 0xffff;
+		state->fs = saved_state->fs & 0xffff;
+		state->gs = saved_state->gs & 0xffff;
 
-		if (saved_state->efl & EFL_VM) {
-		    /*
-		     * V8086 mode.
-		     */
-		    state->ds = saved_state->v86_segs.v86_ds & 0xffff;
-		    state->es = saved_state->v86_segs.v86_es & 0xffff;
-		    state->fs = saved_state->v86_segs.v86_fs & 0xffff;
-		    state->gs = saved_state->v86_segs.v86_gs & 0xffff;
-
-		    if (thr_act->machine.pcb->ims.v86s.int_table) {
-			/*
-			 * Hardware assist on
-			 */
-			if ((thr_act->machine.pcb->ims.v86s.flags &
-					(EFL_IF|V86_IF_PENDING)) == 0)
-			    state->efl &= ~EFL_IF;
-		    }
-		}
-		else {
-		    /*
-		     * 386 mode.
-		     */
-		    state->ds = saved_state->ds & 0xffff;
-		    state->es = saved_state->es & 0xffff;
-		    state->fs = saved_state->fs & 0xffff;
-		    state->gs = saved_state->gs & 0xffff;
-		}
-		*count = i386_SAVED_STATE_COUNT;
+		*count = x86_SAVED_STATE32_COUNT;
 		break;
 	    }
 
-	    case i386_NEW_THREAD_STATE:
-	    case i386_REGS_SEGS_STATE:
+	    case x86_SAVED_STATE64:
 	    {
-		register struct i386_new_thread_state	*state;
-		register struct i386_saved_state	*saved_state;
+		x86_saved_state64_t	*state;
+		x86_saved_state64_t	*saved_state;
 
-		if (*count < i386_NEW_THREAD_STATE_COUNT)
-		    return(KERN_INVALID_ARGUMENT);
+		if (*count < x86_SAVED_STATE64_COUNT)
+		        return(KERN_INVALID_ARGUMENT);
 
-		state = (struct i386_new_thread_state *) tstate;
-		saved_state = USER_REGS(thr_act);
+		state = (x86_saved_state64_t *)tstate;
+		saved_state = USER_REGS64(thr_act);
 
 		/*
-		 * General registers.
+		 * First, copy everything:
 		 */
-		state->edi = saved_state->edi;
-		state->esi = saved_state->esi;
-		state->ebp = saved_state->ebp;
-		state->ebx = saved_state->ebx;
-		state->edx = saved_state->edx;
-		state->ecx = saved_state->ecx;
-		state->eax = saved_state->eax;
-		state->eip = saved_state->eip;
-		state->efl = saved_state->efl;
-		state->uesp = saved_state->uesp;
-
-		state->cs = saved_state->cs;
-		state->ss = saved_state->ss;
-		if (saved_state->efl & EFL_VM) {
-		    /*
-		     * V8086 mode.
-		     */
-		    state->ds = saved_state->v86_segs.v86_ds & 0xffff;
-		    state->es = saved_state->v86_segs.v86_es & 0xffff;
-		    state->fs = saved_state->v86_segs.v86_fs & 0xffff;
-		    state->gs = saved_state->v86_segs.v86_gs & 0xffff;
-
-		    if (thr_act->machine.pcb->ims.v86s.int_table) {
-			/*
-			 * Hardware assist on
-			 */
-			if ((thr_act->machine.pcb->ims.v86s.flags &
-					(EFL_IF|V86_IF_PENDING)) == 0)
-			    state->efl &= ~EFL_IF;
-		    }
-		}
-		else {
-		    /*
-		     * 386 mode.
-		     */
-		    state->ds = saved_state->ds & 0xffff;
-		    state->es = saved_state->es & 0xffff;
-		    state->fs = saved_state->fs & 0xffff;
-		    state->gs = saved_state->gs & 0xffff;
-		}
-		*count = i386_NEW_THREAD_STATE_COUNT;
+		*state = *saved_state;
+		state->fs = saved_state->fs & 0xffff;
+		state->gs = saved_state->gs & 0xffff;
+
+		*count = x86_SAVED_STATE64_COUNT;
 		break;
 	    }
 
-	    case THREAD_SYSCALL_STATE:
+	    case x86_FLOAT_STATE32:
 	    {
-		register struct thread_syscall_state *state;
-		register struct i386_saved_state *saved_state = USER_REGS(thr_act);
-
-		state = (struct thread_syscall_state *) tstate;
-		state->eax = saved_state->eax;
-		state->edx = saved_state->edx;
-		state->efl = saved_state->efl;
-		state->eip = saved_state->eip;
-		state->esp = saved_state->uesp;
-		*count = i386_THREAD_SYSCALL_STATE_COUNT;
-		break;
+		if (*count < x86_FLOAT_STATE32_COUNT) 
+			return(KERN_INVALID_ARGUMENT);
+
+		if (thread_is_64bit(thr_act))
+			return(KERN_INVALID_ARGUMENT);
+
+		*count = x86_FLOAT_STATE32_COUNT;
+
+		return fpu_get_fxstate(thr_act, tstate);
 	    }
 
-	    case THREAD_STATE_FLAVOR_LIST:
-		if (*count < 5)
-		    return (KERN_INVALID_ARGUMENT);
-		tstate[0] = i386_NEW_THREAD_STATE;
-		tstate[1] = i386_FLOAT_STATE;
-		tstate[2] = i386_ISA_PORT_MAP_STATE;
-		tstate[3] = i386_V86_ASSIST_STATE;
-		tstate[4] = THREAD_SYSCALL_STATE;
-		*count = 5;
+	    case x86_FLOAT_STATE64:
+	    {
+		if (*count < x86_FLOAT_STATE64_COUNT) 
+			return(KERN_INVALID_ARGUMENT);
+
+		if ( !thread_is_64bit(thr_act))
+			return(KERN_INVALID_ARGUMENT);
+
+		*count = x86_FLOAT_STATE64_COUNT;
+
+		return fpu_get_fxstate(thr_act, tstate);
+	    }
+
+	    case x86_FLOAT_STATE:
+	    {
+	        x86_float_state_t	*state;
+		kern_return_t		kret;
+
+		if (*count < x86_FLOAT_STATE_COUNT)
+			return(KERN_INVALID_ARGUMENT);
+
+		state = (x86_float_state_t *)tstate;
+
+		/*
+		 * no need to bzero... currently 
+		 * x86_FLOAT_STATE64_COUNT == x86_FLOAT_STATE32_COUNT
+		 */
+		if (thread_is_64bit(thr_act)) {
+		        state->fsh.flavor = x86_FLOAT_STATE64;
+		        state->fsh.count  = x86_FLOAT_STATE64_COUNT;
+
+			kret = fpu_get_fxstate(thr_act, (thread_state_t)&state->ufs.fs64);
+		} else {
+		        state->fsh.flavor = x86_FLOAT_STATE32;
+			state->fsh.count  = x86_FLOAT_STATE32_COUNT;
+
+			kret = fpu_get_fxstate(thr_act, (thread_state_t)&state->ufs.fs32);
+		}
+		*count = x86_FLOAT_STATE_COUNT;
+
+		return(kret);
+	    }
+
+
+	    case OLD_i386_THREAD_STATE: 
+	    case x86_THREAD_STATE32: 
+	    {
+		if (*count < x86_THREAD_STATE32_COUNT)
+			return(KERN_INVALID_ARGUMENT);
+
+		if (thread_is_64bit(thr_act))
+		        return(KERN_INVALID_ARGUMENT);
+
+		*count = x86_THREAD_STATE32_COUNT;
+
+		get_thread_state32(thr_act, (x86_thread_state32_t *)tstate);
 		break;
+	    }
 
-	    case i386_FLOAT_STATE: {
-		if (*count < i386_old_FLOAT_STATE_COUNT)
+	    case x86_THREAD_STATE64:
+	    {
+		if (*count < x86_THREAD_STATE64_COUNT)
 			return(KERN_INVALID_ARGUMENT);
-                if (*count< i386_FLOAT_STATE_COUNT) {
-                    *count = i386_old_FLOAT_STATE_COUNT;
-                    return fpu_get_state(thr_act,(struct i386_float_state *)tstate);
-                } else {
-                    *count = i386_FLOAT_STATE_COUNT;
-                    return fpu_get_fxstate(thr_act,(struct i386_float_state *)tstate);
-                }
+
+                if ( !thread_is_64bit(thr_act))
+		        return(KERN_INVALID_ARGUMENT);
+
+		*count = x86_THREAD_STATE64_COUNT;
+
+		get_thread_state64(thr_act, (x86_thread_state64_t *)tstate);
+		break;
 	    }
 
-	    /*
-	     * Temporary - replace by i386_io_map
-	     */
-	    case i386_ISA_PORT_MAP_STATE: {
-		register struct i386_isa_port_map_state *state;
-		register iopb_tss_t tss;
+	    case x86_THREAD_STATE:
+	    {
+		x86_thread_state_t 	*state;
 
-		if (*count < i386_ISA_PORT_MAP_STATE_COUNT)
+		if (*count < x86_THREAD_STATE_COUNT)
 			return(KERN_INVALID_ARGUMENT);
 
-		state = (struct i386_isa_port_map_state *) tstate;
-		tss = thr_act->machine.pcb->ims.io_tss;
+		state = (x86_thread_state_t *)tstate;
 
-		if (tss == 0) {
-		    unsigned int i;
+		bzero((char *)state, sizeof(x86_thread_state_t));
 
-		    /*
-		     *	The thread has no ktss, so no IO permissions.
-		     */
+		if (thread_is_64bit(thr_act)) {
+			state->tsh.flavor = x86_THREAD_STATE64;
+			state->tsh.count  = x86_THREAD_STATE64_COUNT;
 
-		    for (i = 0; i < sizeof state->pm; i++)
-			state->pm[i] = 0xff;
+		        get_thread_state64(thr_act, &state->uts.ts64);
 		} else {
-		    /*
-		     *	The thread has its own ktss.
-		     */
+			state->tsh.flavor = x86_THREAD_STATE32;
+			state->tsh.count  = x86_THREAD_STATE32_COUNT;
 
-		    bcopy((char *) tss->bitmap,
-			  (char *) state->pm,
-			  sizeof state->pm);
+		        get_thread_state32(thr_act, &state->uts.ts32);
 		}
+		*count = x86_THREAD_STATE_COUNT;
+
+		break;
+	    }
+
+
+	    case x86_EXCEPTION_STATE32:
+	    {
+		if (*count < x86_EXCEPTION_STATE32_COUNT)
+			return(KERN_INVALID_ARGUMENT);
+
+		if (thread_is_64bit(thr_act))
+			return(KERN_INVALID_ARGUMENT);
+
+		*count = x86_EXCEPTION_STATE32_COUNT;
 
-		*count = i386_ISA_PORT_MAP_STATE_COUNT;
+		get_exception_state32(thr_act, (x86_exception_state32_t *)tstate);
 		break;
 	    }
 
-	    case i386_V86_ASSIST_STATE:
+	    case x86_EXCEPTION_STATE64:
 	    {
-		register struct i386_v86_assist_state *state;
+		if (*count < x86_EXCEPTION_STATE64_COUNT)
+			return(KERN_INVALID_ARGUMENT);
 
-		if (*count < i386_V86_ASSIST_STATE_COUNT)
-		    return KERN_INVALID_ARGUMENT;
+		if ( !thread_is_64bit(thr_act))
+			return(KERN_INVALID_ARGUMENT);
 
-		state = (struct i386_v86_assist_state *) tstate;
-		state->int_table = thr_act->machine.pcb->ims.v86s.int_table;
-		state->int_count = thr_act->machine.pcb->ims.v86s.int_count;
+		*count = x86_EXCEPTION_STATE64_COUNT;
 
-		*count = i386_V86_ASSIST_STATE_COUNT;
+		get_exception_state64(thr_act, (x86_exception_state64_t *)tstate);
 		break;
 	    }
 
-	case i386_THREAD_STATE: {
-		struct i386_saved_state	*saved_state;
-		i386_thread_state_t	*state;
-
-		saved_state = USER_REGS(thr_act);
-		state = (i386_thread_state_t *)tstate;
-
-	    	state->eax = saved_state->eax;
-	    	state->ebx = saved_state->ebx;
-	    	state->ecx = saved_state->ecx;
-	    	state->edx = saved_state->edx;
-	    	state->edi = saved_state->edi;
-	    	state->esi = saved_state->esi;
-	    	state->ebp = saved_state->ebp;
-		state->esp = saved_state->uesp;
-		state->eflags = saved_state->efl;
-	    	state->eip = saved_state->eip;
-		state->cs = saved_state->cs;
-		state->ss = saved_state->ss;
-		state->ds = saved_state->ds;
-		state->es = saved_state->es;
-		state->fs = saved_state->fs;
-		state->gs = saved_state->gs;
+	    case x86_EXCEPTION_STATE:
+	    {
+		x86_exception_state_t 	*state;
+
+		if (*count < x86_EXCEPTION_STATE_COUNT)
+			return(KERN_INVALID_ARGUMENT);
+
+		state = (x86_exception_state_t *)tstate;
+
+		bzero((char *)state, sizeof(x86_exception_state_t));
+
+		if (thread_is_64bit(thr_act)) {
+			state->esh.flavor = x86_EXCEPTION_STATE64;
+			state->esh.count  = x86_EXCEPTION_STATE64_COUNT;
+
+		        get_exception_state64(thr_act, &state->ues.es64);
+		} else {
+			state->esh.flavor = x86_EXCEPTION_STATE32;
+			state->esh.count  = x86_EXCEPTION_STATE32_COUNT;
+
+		        get_exception_state32(thr_act, &state->ues.es32);
+		}
+		*count = x86_EXCEPTION_STATE_COUNT;
+
+		break;
+	}
+	case x86_DEBUG_STATE32:
+	{
+		if (*count < x86_DEBUG_STATE32_COUNT)
+			return(KERN_INVALID_ARGUMENT);
+
+		if (thread_is_64bit(thr_act))
+			return(KERN_INVALID_ARGUMENT);
+
+		get_debug_state32(thr_act, (x86_debug_state32_t *)tstate);
+
+		*count = x86_DEBUG_STATE32_COUNT;
+
+		break;
+	}
+	case x86_DEBUG_STATE64:
+	{
+		if (*count < x86_DEBUG_STATE64_COUNT)
+			return(KERN_INVALID_ARGUMENT);
+		
+		if (!thread_is_64bit(thr_act))
+			return(KERN_INVALID_ARGUMENT);
+
+		get_debug_state64(thr_act, (x86_debug_state64_t *)tstate);
+
+		*count = x86_DEBUG_STATE64_COUNT;
+
 		break;
 	}
+	case x86_DEBUG_STATE:
+	{
+		x86_debug_state_t   *state;
+
+		if (*count < x86_DEBUG_STATE_COUNT)
+			return(KERN_INVALID_ARGUMENT);
+
+		state = (x86_debug_state_t *)tstate;
+
+		bzero(state, sizeof *state);
+
+		if (thread_is_64bit(thr_act)) {
+			state->dsh.flavor = x86_DEBUG_STATE64;
+			state->dsh.count  = x86_DEBUG_STATE64_COUNT;
+
+			get_debug_state64(thr_act, &state->uds.ds64);
+		} else {
+			state->dsh.flavor = x86_DEBUG_STATE32;
+			state->dsh.count  = x86_DEBUG_STATE32_COUNT;
 
+			get_debug_state32(thr_act, &state->uds.ds32);
+		}
+		*count = x86_DEBUG_STATE_COUNT;
+		break;
+	}
 	    default:
 		return(KERN_INVALID_ARGUMENT);
 	}
@@ -879,43 +1469,211 @@ machine_thread_get_state(
 	return(KERN_SUCCESS);
 }
 
+kern_return_t
+machine_thread_get_kern_state(
+		thread_t		thread,
+		thread_flavor_t		flavor,
+		thread_state_t		tstate,
+		mach_msg_type_number_t	*count)
+{
+
+	/*
+	 * This works only for an interrupted kernel thread
+	 */
+	if (thread != current_thread() || current_cpu_datap()->cpu_int_state == NULL)
+		return KERN_FAILURE;
+
+	switch(flavor) {
+        
+		case x86_THREAD_STATE32: 
+		{
+
+			x86_thread_state32_t	*state;
+			x86_saved_state32_t	*saved_state;
+
+			if (*count < x86_THREAD_STATE32_COUNT)
+				return(KERN_INVALID_ARGUMENT);
+			
+			state = (x86_thread_state32_t *)tstate;
+
+			assert(is_saved_state32(current_cpu_datap()->cpu_int_state));
+			saved_state = saved_state32(current_cpu_datap()->cpu_int_state);
+			/*
+			 * General registers.
+			 */
+			state->eax = saved_state->eax;
+			state->ebx = saved_state->ebx;
+			state->ecx = saved_state->ecx;
+			state->edx = saved_state->edx;
+			state->edi = saved_state->edi;
+			state->esi = saved_state->esi;
+			state->ebp = saved_state->ebp;
+			state->esp = saved_state->uesp;
+			state->eflags = saved_state->efl;
+			state->eip = saved_state->eip;
+			state->cs = saved_state->cs;
+			state->ss = saved_state->ss;
+			state->ds = saved_state->ds & 0xffff;
+			state->es = saved_state->es & 0xffff;
+			state->fs = saved_state->fs & 0xffff;
+			state->gs = saved_state->gs & 0xffff;
+			
+			*count = x86_THREAD_STATE32_COUNT;
+
+			return KERN_SUCCESS;
+		}
+		break;	// for completeness
+
+		case x86_THREAD_STATE:
+		{
+			// wrap a 32 bit thread state into a 32/64bit clean thread state
+			x86_thread_state_t		*state;
+			x86_saved_state32_t		*saved_state;
+
+			if(*count < x86_THREAD_STATE_COUNT)
+				return (KERN_INVALID_ARGUMENT);
+
+			state = (x86_thread_state_t *)tstate;
+			assert(is_saved_state32(current_cpu_datap()->cpu_int_state));
+			saved_state = saved_state32(current_cpu_datap()->cpu_int_state);
+
+			state->tsh.flavor = x86_THREAD_STATE32;
+			state->tsh.count = x86_THREAD_STATE32_COUNT;
+			
+			/* 
+			 * General registers.
+			 */
+
+			state->uts.ts32.eax = saved_state->eax;
+			state->uts.ts32.ebx = saved_state->ebx;
+			state->uts.ts32.ecx = saved_state->ecx;
+			state->uts.ts32.edx = saved_state->edx;
+			state->uts.ts32.edi = saved_state->edi;
+			state->uts.ts32.esi = saved_state->esi;
+			state->uts.ts32.ebp = saved_state->ebp;
+			state->uts.ts32.esp = saved_state->uesp;
+			state->uts.ts32.eflags = saved_state->efl;
+			state->uts.ts32.eip = saved_state->eip;
+			state->uts.ts32.cs = saved_state->cs;
+			state->uts.ts32.ss = saved_state->ss;
+			state->uts.ts32.ds = saved_state->ds & 0xffff;
+			state->uts.ts32.es = saved_state->es & 0xffff;
+			state->uts.ts32.fs = saved_state->fs & 0xffff;
+			state->uts.ts32.gs = saved_state->gs & 0xffff;
+	
+			*count = x86_THREAD_STATE_COUNT;
+			return KERN_SUCCESS;
+		}
+		break;
+	}	
+	return KERN_FAILURE;
+}
+
+
 /*
  * Initialize the machine-dependent state for a new thread.
  */
 kern_return_t
 machine_thread_create(
 	thread_t		thread,
-	__unused task_t		task)
+	task_t			task)
 {
-	pcb_t	pcb = &thread->machine.xxx_pcb;
+	pcb_t			pcb = &thread->machine.xxx_pcb;
+	struct real_descriptor	*ldtp;
+	pmap_paddr_t		paddr;
+	x86_saved_state_t	*iss;
 
-	thread->machine.pcb = pcb;
+	inval_copy_windows(thread);
 
-	simple_lock_init(&pcb->lock, 0);
+	thread->machine.physwindow_pte = 0;
+	thread->machine.physwindow_busy = 0;
 
-	/*
-	 *	Guarantee that the bootstrapped thread will be in user
-	 *	mode.
-	 */
-	pcb->iss.cs = USER_CS;
-	pcb->iss.ss = USER_DS;
-	pcb->iss.ds = USER_DS;
-	pcb->iss.es = USER_DS;
-	pcb->iss.fs = USER_DS;
-	pcb->iss.gs = USER_DS;
-	pcb->iss.efl = EFL_USER_SET;
-	{
-	  struct real_descriptor *ldtp;
-	  ldtp = (struct real_descriptor *)ldt;
-	  pcb->cthread_desc = ldtp[sel_idx(USER_DS)];
-          pcb->uldt_desc = ldtp[sel_idx(USER_DS)];
-          pcb->uldt_selector = 0;
+        if (task_has_64BitAddr(task)) {
+		x86_sframe64_t		*sf64;
+
+	        sf64 = (x86_sframe64_t *)zalloc(iss_zone64);
+
+		if (sf64 == NULL)
+		        panic("iss_zone64");
+		pcb->sf = (void *)sf64;
+
+		bzero((char *)sf64, sizeof(x86_sframe64_t));
+
+		iss = (x86_saved_state_t *) &sf64->ssf;
+		iss->flavor = x86_SAVED_STATE64;
+		/*
+		 *	Guarantee that the bootstrapped thread will be in user
+		 *	mode.
+		 */
+		iss->ss_64.isf.rflags = EFL_USER_SET;
+		iss->ss_64.isf.cs = USER64_CS;
+		iss->ss_64.isf.ss = USER_DS;
+		iss->ss_64.fs = USER_DS;
+		iss->ss_64.gs = USER_DS;
+	} else {
+		if (cpu_mode_is64bit()) {
+			x86_sframe_compat32_t	   *sfc32;
+
+			sfc32 = (x86_sframe_compat32_t *)zalloc(iss_zone32);
+			if (sfc32 == NULL)
+			        panic("iss_zone32");
+			pcb->sf = (void *)sfc32;
+
+			bzero((char *)sfc32, sizeof(x86_sframe_compat32_t));
+
+			iss = (x86_saved_state_t *) &sfc32->ssf.iss32;
+			iss->flavor = x86_SAVED_STATE32;
+#if DEBUG
+			{
+			x86_saved_state_compat32_t *xssc;
+
+			xssc  = (x86_saved_state_compat32_t *) iss;
+			xssc->pad_for_16byte_alignment[0] = 0x64326432;
+			xssc->pad_for_16byte_alignment[1] = 0x64326432;
+			}
+#endif
+		} else {
+		        x86_sframe32_t	*sf32;
+
+			sf32 = (x86_sframe32_t *)zalloc(iss_zone32);
+
+			if (sf32 == NULL)
+			        panic("iss_zone32");
+			pcb->sf = (void *)sf32;
+
+			bzero((char *)sf32, sizeof(x86_sframe32_t));
+
+			iss = (x86_saved_state_t *) &sf32->ssf;
+			iss->flavor = x86_SAVED_STATE32;
+		}
+		/*
+		 *	Guarantee that the bootstrapped thread will be in user
+		 *	mode.
+		 */
+		iss->ss_32.cs = USER_CS;
+		iss->ss_32.ss = USER_DS;
+		iss->ss_32.ds = USER_DS;
+		iss->ss_32.es = USER_DS;
+		iss->ss_32.fs = USER_DS;
+		iss->ss_32.gs = USER_DS;
+		iss->ss_32.efl = EFL_USER_SET;
 	}
+	pcb->iss = iss;
 
-	/*
-	 *      Allocate a kernel stack per thread.
-	 */
-	stack_alloc(thread);
+	thread->machine.pcb = pcb;
+	simple_lock_init(&pcb->lock, 0);
+
+	ldtp = (struct real_descriptor *)pmap_index_to_virt(HIGH_FIXED_LDT_BEGIN);
+	pcb->cthread_desc = ldtp[sel_idx(USER_DS)];
+	pcb->uldt_desc = ldtp[sel_idx(USER_DS)];
+	pcb->uldt_selector = 0;
+
+	pcb->iss_pte0 = (uint64_t)pte_kernel_rw(kvtophys((vm_offset_t)pcb->iss));
+
+	if (0 == (paddr = pa_to_pte(kvtophys((vm_offset_t)(pcb->iss) + PAGE_SIZE))))
+	        pcb->iss_pte1 = INTEL_PTE_INVALID;
+	else
+	        pcb->iss_pte1 = (uint64_t)pte_kernel_rw(paddr);
 
 	return(KERN_SUCCESS);
 }
@@ -931,15 +1689,47 @@ machine_thread_destroy(
 
 	assert(pcb);
         
-	if (pcb->ims.io_tss != 0)
-		iopb_destroy(pcb->ims.io_tss);
-	if (pcb->ims.ifps != 0)
-		fpu_free(pcb->ims.ifps);
-	if (pcb->ims.ldt != 0)
-		user_ldt_free(pcb->ims.ldt);
+	if (pcb->ifps != 0)
+		fpu_free(pcb->ifps);
+	if (pcb->sf != 0) {
+	        if (thread_is_64bit(thread))
+		        zfree(iss_zone64, pcb->sf);
+		else
+		        zfree(iss_zone32, pcb->sf);
+		pcb->sf = 0;
+	}
+	if (pcb->ids) {
+	        if (thread_is_64bit(thread))
+		        zfree(ids_zone64, pcb->ids);
+		else
+		        zfree(ids_zone32, pcb->ids);
+	}
 	thread->machine.pcb = (pcb_t)0;
+
+}
+
+void
+machine_thread_switch_addrmode(thread_t thread, int oldmode_is64bit)
+{
+	register pcb_t	pcb = thread->machine.pcb;
+
+        assert(pcb);
+	
+	if (pcb->sf != 0) {
+	        if (oldmode_is64bit)
+		        zfree(iss_zone64, pcb->sf);
+		else
+		        zfree(iss_zone32, pcb->sf);
+	}
+	machine_thread_create(thread, thread->task);
+
+	/* If we're switching ourselves, reset the pcb addresses etc. */
+	if (thread == current_thread())
+		act_machine_switch_pcb(thread);
 }
 
+
+
 /*
  * This is used to set the current thr_act/thread
  * when starting up a new processor
@@ -947,17 +1737,23 @@ machine_thread_destroy(
 void
 machine_set_current_thread( thread_t thread )
 {
-	mp_disable_preemption();
-
-        current_cpu_datap()->cpu_active_thread = thread;
-	current_cpu_datap()->cpu_active_kloaded = THREAD_NULL;
-
-	mp_enable_preemption();
+	current_cpu_datap()->cpu_active_thread = thread;
 }
 
+/*
+ * This is called when a task is termianted.
+ */
 void
 machine_thread_terminate_self(void)
 {
+	task_t self_task = current_task();
+	if (self_task) {
+	    user_ldt_t user_ldt = self_task->i386_ldt;
+	    if (user_ldt != 0) {
+		self_task->i386_ldt = 0;
+		user_ldt_free(user_ldt);
+	    }
+	}
 }
 
 void
@@ -988,6 +1784,37 @@ act_machine_return(int code)
 void
 machine_thread_init(void)
 {
+        if (cpu_mode_is64bit()) {
+	        iss_zone64 = zinit(sizeof(x86_sframe64_t),
+				   THREAD_MAX * sizeof(x86_sframe64_t),
+				   THREAD_CHUNK * sizeof(x86_sframe64_t),
+				   "x86_64 saved state");
+
+		assert(sizeof(x86_sframe_compat32_t) % 16 == 0);
+	        iss_zone32 = zinit(sizeof(x86_sframe_compat32_t),
+				   THREAD_MAX * sizeof(x86_sframe_compat32_t),
+				   THREAD_CHUNK * sizeof(x86_sframe_compat32_t),
+				   "x86_32 saved state");
+
+	        ids_zone32 = zinit(sizeof(x86_debug_state32_t),
+				   THREAD_MAX * (sizeof(x86_debug_state32_t)),
+				   THREAD_CHUNK * (sizeof(x86_debug_state32_t)),
+				   "x86_32 debug state");
+	        ids_zone64 = zinit(sizeof(x86_debug_state64_t),
+				   THREAD_MAX * sizeof(x86_debug_state64_t),
+				   THREAD_CHUNK * sizeof(x86_debug_state64_t),
+				   "x86_64 debug state");
+
+	} else {
+	        iss_zone32 = zinit(sizeof(x86_sframe32_t),
+				   THREAD_MAX * sizeof(x86_sframe32_t),
+				   THREAD_CHUNK * sizeof(x86_sframe32_t),
+				   "x86 saved state");
+	        ids_zone32 = zinit(sizeof(x86_debug_state32_t),
+				   THREAD_MAX * (sizeof(x86_debug_state32_t)),
+				   THREAD_CHUNK * (sizeof(x86_debug_state32_t)),
+				   "x86 debug state");
+	}
 	fpu_module_init();
 	iopb_init();
 }
@@ -1026,13 +1853,30 @@ dump_handlers(thread_t thr_act)
 void
 dump_regs(thread_t thr_act)
 {
-	if (thr_act->machine.pcb) {
-		register struct i386_saved_state *ssp = USER_REGS(thr_act);
-		/* Print out user register state */
+        if (thr_act->machine.pcb == NULL)
+	        return;
+
+        if (thread_is_64bit(thr_act)) {
+	        x86_saved_state64_t	*ssp;
+
+		ssp = USER_REGS64(thr_act);
+
+		panic("dump_regs: 64bit tasks not yet supported");
+
+	} else {
+	        x86_saved_state32_t	*ssp;
+
+		ssp = USER_REGS32(thr_act);
+
+		/*
+		 * Print out user register state
+		 */
 		printf("\tRegs:\tedi=%x esi=%x ebp=%x ebx=%x edx=%x\n",
 		    ssp->edi, ssp->esi, ssp->ebp, ssp->ebx, ssp->edx);
+
 		printf("\t\tecx=%x eax=%x eip=%x efl=%x uesp=%x\n",
 		    ssp->ecx, ssp->eax, ssp->eip, ssp->efl, ssp->uesp);
+
 		printf("\t\tcs=%x ss=%x\n", ssp->cs, ssp->ss);
 	}
 }
@@ -1068,14 +1912,24 @@ dump_act(thread_t thr_act)
 user_addr_t
 get_useraddr(void)
 {
-  
         thread_t thr_act = current_thread();
  
-	if (thr_act->machine.pcb) 
-         	return(thr_act->machine.pcb->iss.eip);
-	else 
-		return(0);
+	if (thr_act->machine.pcb == NULL) 
+	        return (0);
+
+        if (thread_is_64bit(thr_act)) {
+	        x86_saved_state64_t	*iss64;
+		
+		iss64 = USER_REGS64(thr_act);
+
+         	return(iss64->isf.rip);
+	} else {
+	        x86_saved_state32_t	*iss32;
 
+		iss32 = USER_REGS32(thr_act);
+
+         	return(iss32->eip);
+	}
 }
 
 /*
@@ -1085,16 +1939,17 @@ get_useraddr(void)
 vm_offset_t
 machine_stack_detach(thread_t thread)
 {
-  vm_offset_t stack;
+	vm_offset_t     stack;
 
-		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED,MACH_STACK_DETACH),
-			thread, thread->priority,
-			thread->sched_pri, 0,
-			0);
+	KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DETACH),
+		     thread, thread->priority,
+		     thread->sched_pri, 0,
+		     0);
 
-  stack = thread->kernel_stack;
-  thread->kernel_stack = 0;
-  return(stack);
+	stack = thread->kernel_stack;
+	thread->kernel_stack = 0;
+
+	return (stack);
 }
 
 /*
@@ -1106,23 +1961,21 @@ machine_stack_attach(
 	thread_t		thread,
 	vm_offset_t		stack)
 {
-  struct i386_kernel_state *statep;
-
-		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED,MACH_STACK_ATTACH),
-			thread, thread->priority,
-			thread->sched_pri, 0, 0);
+	struct x86_kernel_state32 *statep;
 
-  assert(stack);
-  statep = STACK_IKS(stack);
-  thread->kernel_stack = stack;
+	KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_ATTACH),
+		     thread, thread->priority,
+		     thread->sched_pri, 0, 0);
 
-  statep->k_eip = (unsigned long) Thread_continue;
-  statep->k_ebx = (unsigned long) thread_continue;
-  statep->k_esp = (unsigned long) STACK_IEL(stack);
+	assert(stack);
+	thread->kernel_stack = stack;
 
-  STACK_IEL(stack)->saved_state = &thread->machine.pcb->iss;
+	statep = STACK_IKS(stack);
+	statep->k_eip = (unsigned long) Thread_continue;
+	statep->k_ebx = (unsigned long) thread_continue;
+	statep->k_esp = (unsigned long) STACK_IEL(stack);
 
-  return;
+	return;
 }
 
 /*
@@ -1133,97 +1986,180 @@ void
 machine_stack_handoff(thread_t old,
 	      thread_t new)
 {
-  vm_offset_t stack;
+	vm_offset_t     stack;
 
-		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED,MACH_STACK_HANDOFF),
-			thread, thread->priority,
-			thread->sched_pri, 0, 0);
+	assert(new);
+	assert(old);
 
-  assert(new);
-  assert(old);
+	stack = old->kernel_stack;
+	if (stack == old->reserved_stack) {
+		assert(new->reserved_stack);
+		old->reserved_stack = new->reserved_stack;
+		new->reserved_stack = stack;
+	}
+	old->kernel_stack = 0;
+	/*
+	 * A full call to machine_stack_attach() is unnecessry
+	 * because old stack is already initialized.
+	 */
+	new->kernel_stack = stack;
 
-  stack = machine_stack_detach(old);
-  machine_stack_attach(new, stack);
+	fpu_save_context(old);
 
-  PMAP_SWITCH_CONTEXT(old->task, new->task, cpu_number());
+	old->machine.specFlags &= ~OnProc;
+	new->machine.specFlags |= OnProc;
 
-  KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_STACK_HANDOFF) | DBG_FUNC_NONE,
-		     (int)old, (int)new, old->sched_pri, new->sched_pri, 0);
+	PMAP_SWITCH_CONTEXT(old, new, cpu_number());
+	act_machine_switch_pcb(new);
 
-  machine_set_current_thread(new);
+	KERNEL_DEBUG_CONSTANT(
+		MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_HANDOFF)|DBG_FUNC_NONE,
+		old->reason, (int)new, old->sched_pri, new->sched_pri, 0);
 
-  current_cpu_datap()->cpu_active_stack = new->kernel_stack;
+	machine_set_current_thread(new);
 
-  return;
+	return;
 }
 
-struct i386_act_context {
-	struct i386_saved_state ss;
-	struct i386_float_state fs;
+
+
+
+struct x86_act_context32 {
+	x86_saved_state32_t ss;
+	x86_float_state32_t fs;
+	x86_debug_state32_t ds;
+};
+
+struct x86_act_context64 {
+	x86_saved_state64_t ss;
+	x86_float_state64_t fs;
+	x86_debug_state64_t ds;
 };
 
+
+
 void *
 act_thread_csave(void)
 {
-struct i386_act_context *ic;
-kern_return_t kret;
-int val;
+        kern_return_t kret;
+	mach_msg_type_number_t val;
+        thread_t thr_act = current_thread();
+
+        if (thread_is_64bit(thr_act)) {
+	        struct x86_act_context64 *ic64;
 
-		ic = (struct i386_act_context *)kalloc(sizeof(struct i386_act_context));
+	        ic64 = (struct x86_act_context64 *)kalloc(sizeof(struct x86_act_context64));
+
+		if (ic64 == (struct x86_act_context64 *)NULL)
+		        return((void *)0);
+
+		val = x86_SAVED_STATE64_COUNT; 
+		kret = machine_thread_get_state(thr_act, x86_SAVED_STATE64,
+						(thread_state_t) &ic64->ss, &val);
+		if (kret != KERN_SUCCESS) {
+		        kfree(ic64, sizeof(struct x86_act_context64));
+			return((void *)0);
+		}
+		val = x86_FLOAT_STATE64_COUNT; 
+		kret = machine_thread_get_state(thr_act, x86_FLOAT_STATE64,
+						(thread_state_t) &ic64->fs, &val);
 
-		if (ic == (struct i386_act_context *)NULL)
-				return((void *)0);
+		if (kret != KERN_SUCCESS) {
+		        kfree(ic64, sizeof(struct x86_act_context64));
+			return((void *)0);
+		}
 
-		val = i386_SAVED_STATE_COUNT; 
-		kret = machine_thread_get_state(current_thread(),
-						i386_SAVED_STATE,
-						(thread_state_t) &ic->ss,
+		val = x86_DEBUG_STATE64_COUNT;
+		kret = machine_thread_get_state(thr_act,
+						x86_DEBUG_STATE64,
+						(thread_state_t)&ic64->ds,
 						&val);
 		if (kret != KERN_SUCCESS) {
-				kfree(ic,sizeof(struct i386_act_context));
-				return((void *)0);
+		        kfree(ic64, sizeof(struct x86_act_context64));
+			return((void *)0);
+		}
+		return(ic64);
+
+	} else {
+	        struct x86_act_context32 *ic32;
+
+	        ic32 = (struct x86_act_context32 *)kalloc(sizeof(struct x86_act_context32));
+
+		if (ic32 == (struct x86_act_context32 *)NULL)
+		        return((void *)0);
+
+		val = x86_SAVED_STATE32_COUNT; 
+		kret = machine_thread_get_state(thr_act, x86_SAVED_STATE32,
+						(thread_state_t) &ic32->ss, &val);
+		if (kret != KERN_SUCCESS) {
+		        kfree(ic32, sizeof(struct x86_act_context32));
+			return((void *)0);
 		}
-		val = i386_FLOAT_STATE_COUNT; 
-		kret = machine_thread_get_state(current_thread(),
-						i386_FLOAT_STATE,
-						(thread_state_t) &ic->fs,
+		val = x86_FLOAT_STATE32_COUNT; 
+		kret = machine_thread_get_state(thr_act, x86_FLOAT_STATE32,
+						(thread_state_t) &ic32->fs, &val);
+		if (kret != KERN_SUCCESS) {
+		        kfree(ic32, sizeof(struct x86_act_context32));
+			return((void *)0);
+		}
+
+		val = x86_DEBUG_STATE32_COUNT;
+		kret = machine_thread_get_state(thr_act,
+						x86_DEBUG_STATE32,
+						(thread_state_t)&ic32->ds,
 						&val);
 		if (kret != KERN_SUCCESS) {
-				kfree(ic,sizeof(struct i386_act_context));
-				return((void *)0);
+		        kfree(ic32, sizeof(struct x86_act_context32));
+			return((void *)0);
 		}
-		return(ic);
+		return(ic32);
+	}
 }
+
+
 void 
 act_thread_catt(void *ctx)
 {
-struct i386_act_context *ic;
-kern_return_t kret;
-
-		ic = (struct i386_act_context *)ctx;
-
-		if (ic == (struct i386_act_context *)NULL)
-				return;
-
-		kret = machine_thread_set_state(current_thread(),
-						i386_SAVED_STATE,
-						(thread_state_t) &ic->ss,
-						i386_SAVED_STATE_COUNT);
-		if (kret != KERN_SUCCESS) 
-				goto out;
-
-		kret = machine_thread_set_state(current_thread(),
-						i386_FLOAT_STATE,
-						(thread_state_t) &ic->fs,
-						i386_FLOAT_STATE_COUNT);
-		if (kret != KERN_SUCCESS)
-				goto out;
-out:
-	kfree(ic,sizeof(struct i386_act_context));		
+        thread_t thr_act = current_thread();
+	kern_return_t kret;
+
+	if (ctx == (void *)NULL)
+	        return;
+
+        if (thread_is_64bit(thr_act)) {
+	        struct x86_act_context64 *ic64;
+
+	        ic64 = (struct x86_act_context64 *)ctx;
+
+		kret = machine_thread_set_state(thr_act, x86_SAVED_STATE64,
+						(thread_state_t) &ic64->ss, x86_SAVED_STATE64_COUNT);
+		if (kret == KERN_SUCCESS) {
+		        machine_thread_set_state(thr_act, x86_FLOAT_STATE64,
+						 (thread_state_t) &ic64->fs, x86_FLOAT_STATE64_COUNT);
+		}
+		kfree(ic64, sizeof(struct x86_act_context64));
+	} else {
+	        struct x86_act_context32 *ic32;
+
+	        ic32 = (struct x86_act_context32 *)ctx;
+
+		kret = machine_thread_set_state(thr_act, x86_SAVED_STATE32,
+						(thread_state_t) &ic32->ss, x86_SAVED_STATE32_COUNT);
+		if (kret == KERN_SUCCESS) {
+		        kret = machine_thread_set_state(thr_act, x86_FLOAT_STATE32,
+						 (thread_state_t) &ic32->fs, x86_FLOAT_STATE32_COUNT);
+			if (kret == KERN_SUCCESS && thr_act->machine.pcb->ids)
+				machine_thread_set_state(thr_act,
+							 x86_DEBUG_STATE32,
+							 (thread_state_t)&ic32->ds,
+							 x86_DEBUG_STATE32_COUNT);
+		}
+		kfree(ic32, sizeof(struct x86_act_context32));
+	}
 }
 
-void act_thread_cfree(void *ctx)
+
+void act_thread_cfree(__unused void *ctx)
 {
-	kfree(ctx,sizeof(struct i386_act_context));		
+	/* XXX - Unused */
 }
-
diff --git a/osfmk/i386/perfmon.c b/osfmk/i386/perfmon.c
index f2cb2b850..109ab1e7a 100644
--- a/osfmk/i386/perfmon.c
+++ b/osfmk/i386/perfmon.c
@@ -36,13 +36,24 @@
 #include <i386/cpuid.h>
 #include <i386/lock.h>
 #include <vm/vm_kern.h>
+#include <kern/task.h>
 
-#ifdef DEBUG
+#if DEBUG
 #define	DBG(x...)	kprintf(x)
 #else
 #define	DBG(x...)
 #endif
 
+decl_simple_lock_data(,pmc_lock)
+static task_t	pmc_owner = TASK_NULL;
+static int	pmc_thread_count = 0;
+
+/* PMC Facility Owner:
+ * TASK_NULL - no one owns it
+ * kernel_task - owned by pmc
+ * other task - owned by another task
+ */
+
 /*
  * Table of ESCRs and addresses associated with performance counters/CCCRs.
  * See Intel SDM Vol 3, Table 15-4 (section 15.9):
@@ -280,24 +291,18 @@ pmc_p6_intr(void *state)
 			(*pmc_table->ovf_func[id])(id, state);
 }
 
-int
-pmc_init(void)
+void *
+pmc_alloc(void)
 {
 	int		ret;
-	cpu_core_t	*my_core;
 	pmc_table_t	*pmc_table;
 	pmc_machine_t	pmc_type;
 
-	my_core = cpu_core();
-	assert(my_core);
-
 	pmc_type = _pmc_machine_type();
 	if (pmc_type == pmc_none) {
-		return KERN_FAILURE;
+		return NULL;
 	}
 	
-	pmc_table = (pmc_table_t *) my_core->pmc;
-	if (pmc_table == NULL) {
 		ret = kmem_alloc(kernel_map,
 			(void *) &pmc_table, sizeof(pmc_table_t));
 		if (ret != KERN_SUCCESS)
@@ -321,18 +326,10 @@ pmc_init(void)
 		default:
 			break;
 		}
-		if (!atomic_cmpxchg((uint32_t *) &my_core->pmc,
-				    0, (uint32_t) pmc_table)) {
-			kmem_free(kernel_map,
-				  (vm_offset_t) pmc_table, sizeof(pmc_table_t));
-		}
-	}
-	DBG("pmc_init() done for cpu %d my_core->pmc=0x%x type=%d\n",
-		cpu_number(), my_core->pmc, pmc_type);
-
-	return KERN_SUCCESS;
+	return (void *) pmc_table;
 }
 
+
 static inline pmc_table_t *
 pmc_table_valid(pmc_id_t id)
 {
@@ -556,3 +553,81 @@ pmc_set_ovf_func(pmc_id_t id, pmc_ovf_func_t func)
 
 	return KERN_SUCCESS;
 }
+
+int
+pmc_acquire(task_t task)
+{
+	kern_return_t retval = KERN_SUCCESS;
+  
+	simple_lock(&pmc_lock);
+  
+	if(pmc_owner == task) {
+		DBG("pmc_acquire - "
+		    "ACQUIRED: already owner\n");
+		retval = KERN_SUCCESS;
+		/* already own it */
+	} else if(pmc_owner == TASK_NULL) { /* no one owns it */
+		pmc_owner = task;
+		pmc_thread_count = 0;
+		DBG("pmc_acquire - "
+		    "ACQUIRED: no current owner - made new owner\n");
+		retval = KERN_SUCCESS;
+	} else { /* someone already owns it */
+		if(pmc_owner == kernel_task) {
+			if(pmc_thread_count == 0) {
+				/* kernel owns it but no threads using it */
+				pmc_owner = task;
+				pmc_thread_count = 0;
+				DBG("pmc_acquire - "
+				    "ACQUIRED: owned by kernel, no threads\n");
+				retval = KERN_SUCCESS;
+			} else {
+				DBG("pmc_acquire - "
+				    "DENIED: owned by kernel, in use\n");
+				retval = KERN_RESOURCE_SHORTAGE;
+			}
+		} else { /* non-kernel owner */
+			DBG("pmc_acquire - "	
+			    "DENIED: owned by another task\n");
+			retval = KERN_RESOURCE_SHORTAGE;
+		}
+	}
+  
+	simple_unlock(&pmc_lock);
+	return retval;
+}
+
+int
+pmc_release(task_t task)
+{
+	kern_return_t retval = KERN_SUCCESS;
+	task_t old_pmc_owner = pmc_owner;
+  
+	simple_lock(&pmc_lock);
+  
+	if(task != pmc_owner) {
+		retval = KERN_NO_ACCESS;
+	} else {
+		if(old_pmc_owner == kernel_task) {
+			if(pmc_thread_count>0) {
+				DBG("pmc_release - "
+				    "NOT RELEASED: owned by kernel, in use\n");
+				retval = KERN_NO_ACCESS;
+			} else {
+				DBG("pmc_release - "
+				    "RELEASED: was owned by kernel\n");
+				pmc_owner = TASK_NULL;
+				retval = KERN_SUCCESS;
+			}
+		} else {
+			DBG("pmc_release - "
+			    "RELEASED: was owned by user\n");
+			pmc_owner = TASK_NULL;
+			retval = KERN_SUCCESS;
+		}
+	}
+
+	simple_unlock(&pmc_lock);
+	return retval;
+}
+
diff --git a/osfmk/i386/perfmon.h b/osfmk/i386/perfmon.h
index 0b0be1c3b..27baf5b7b 100644
--- a/osfmk/i386/perfmon.h
+++ b/osfmk/i386/perfmon.h
@@ -289,7 +289,7 @@ typedef void (pmc_ovf_func_t)(pmc_id_t id, void *state);
  * In-kernel PMC access primitives:
  */
 /* Generic: */
-extern int pmc_init(void);
+extern void *pmc_alloc(void);
 extern int pmc_machine_type(pmc_machine_t *type);
 extern boolean_t pmc_is_reserved(pmc_id_t id);
 extern int pmc_reserve(pmc_id_t id);
@@ -308,4 +308,7 @@ extern int pmc_escr_read(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr);
 extern int pmc_escr_write(pmc_id_t id, pmc_escr_id_t esid, pmc_escr_t *escr);
 extern int pmc_set_ovf_func(pmc_id_t id, pmc_ovf_func_t *func);
 
+extern int pmc_acquire(task_t);
+extern int pmc_release(task_t);
+
 #endif /* _I386_PERFMON_H_ */
diff --git a/osfmk/i386/phys.c b/osfmk/i386/phys.c
index 13ebbd267..5c3449325 100644
--- a/osfmk/i386/phys.c
+++ b/osfmk/i386/phys.c
@@ -126,13 +126,17 @@ pmap_copy_part_page(
 	vm_offset_t	dst_offset,
 	vm_size_t	len)
 {
-        vm_offset_t  src, dst;
+        pmap_paddr_t src, dst;
+
 	assert(psrc != vm_page_fictitious_addr);
 	assert(pdst != vm_page_fictitious_addr);
-	src = (vm_offset_t)i386_ptob(psrc);
-	dst = (vm_offset_t)i386_ptob(pdst);
-	assert(((dst & PAGE_MASK) + dst_offset + len) <= PAGE_SIZE);
-	assert(((src & PAGE_MASK) + src_offset + len) <= PAGE_SIZE);
+
+	src = i386_ptob(psrc);
+	dst = i386_ptob(pdst);
+
+	assert((((uint32_t)dst & PAGE_MASK) + dst_offset + len) <= PAGE_SIZE);
+	assert((((uint32_t)src & PAGE_MASK) + src_offset + len) <= PAGE_SIZE);
+
 	bcopy_phys((addr64_t)src + (src_offset & INTEL_OFFMASK),
 		   (addr64_t)dst + (dst_offset & INTEL_OFFMASK),
 		   len);
@@ -149,21 +153,24 @@ pmap_copy_part_lpage(
 	vm_offset_t	dst_offset,
 	vm_size_t	len)
 {
-	pt_entry_t *ptep;
-	thread_t	thr_act = current_thread();
+        mapwindow_t *map;
 
 	assert(pdst != vm_page_fictitious_addr);
-	ptep = pmap_pte(thr_act->map->pmap, i386_ptob(pdst));
-	if (0 == ptep)
-		panic("pmap_copy_part_lpage ptep");
-	assert(((pdst & PAGE_MASK) + dst_offset + len) <= PAGE_SIZE);
-	if (*(pt_entry_t *) CM2)
-	  panic("pmap_copy_part_lpage");
-	*(int *) CM2 = INTEL_PTE_VALID | INTEL_PTE_RW | (*ptep & PG_FRAME) | 
-	  INTEL_PTE_REF | INTEL_PTE_MOD;
-	invlpg((unsigned int) CA2);
-	memcpy((void *) (CA2 + (dst_offset & INTEL_OFFMASK)), (void *) src, len);
-	*(pt_entry_t *) CM2 = 0;
+	assert((dst_offset + len) <= PAGE_SIZE);
+
+        mp_disable_preemption();
+
+        map = pmap_get_mapwindow(INTEL_PTE_VALID | INTEL_PTE_RW | (i386_ptob(pdst) & PG_FRAME) | 
+                                 INTEL_PTE_REF | INTEL_PTE_MOD);
+	if (map == 0) {
+	        panic("pmap_copy_part_lpage");
+        }
+	invlpg((uintptr_t)map->prv_CADDR);
+
+	memcpy((void *) (map->prv_CADDR + (dst_offset & INTEL_OFFMASK)), (void *) src, len);
+	*map->prv_CMAP = 0;
+
+	mp_enable_preemption();
 }
 
 /*
@@ -177,21 +184,24 @@ pmap_copy_part_rpage(
 	vm_offset_t	dst,
 	vm_size_t	len)
 {
-  pt_entry_t *ptep;
-  thread_t thr_act = current_thread();
+        mapwindow_t *map;
 
 	assert(psrc != vm_page_fictitious_addr);
-	ptep = pmap_pte(thr_act->map->pmap, i386_ptob(psrc));
-	if (0 == ptep)
-		panic("pmap_copy_part_rpage ptep");
-	assert(((psrc & PAGE_MASK) + src_offset + len) <= PAGE_SIZE);
-	if (*(pt_entry_t *) CM2)
-	  panic("pmap_copy_part_rpage");
-	*(pt_entry_t *) CM2 = INTEL_PTE_VALID | INTEL_PTE_RW | (*ptep & PG_FRAME) | 
-	  INTEL_PTE_REF;
-	invlpg((unsigned int) CA2);
-	memcpy((void *) dst, (void *) (CA2 + (src_offset & INTEL_OFFMASK)), len);
-	*(pt_entry_t *) CM2 = 0;
+	assert((src_offset + len) <= PAGE_SIZE);
+
+        mp_disable_preemption();
+
+        map = pmap_get_mapwindow(INTEL_PTE_VALID | INTEL_PTE_RW | (i386_ptob(psrc) & PG_FRAME) | 
+                                 INTEL_PTE_REF);
+	if (map == 0) {
+	        panic("pmap_copy_part_rpage");
+        }
+	invlpg((uintptr_t) map->prv_CADDR);
+
+	memcpy((void *) dst, (void *) (map->prv_CADDR + (src_offset & INTEL_OFFMASK)), len);
+	*map->prv_CMAP = 0;
+
+	mp_enable_preemption();
 }
 
 /*
@@ -199,19 +209,20 @@ pmap_copy_part_rpage(
  *
  *	Convert a kernel virtual address to a physical address
  */
-vm_offset_t
+addr64_t
 kvtophys(
 	vm_offset_t addr)
 {
         pt_entry_t *ptep;
 	pmap_paddr_t pa;
-
-	if ((ptep = pmap_pte(kernel_pmap, addr)) == PT_ENTRY_NULL) {
-	  pa = 0;
+	
+	mp_disable_preemption();
+	if ((ptep = pmap_pte(kernel_pmap, (vm_map_offset_t)addr)) == PT_ENTRY_NULL) {
+	        pa = 0;
 	} else {
-	  pa =  pte_to_pa(*ptep) | (addr & INTEL_OFFMASK);
+	        pa =  pte_to_pa(*ptep) | (addr & INTEL_OFFMASK);
 	}
-	if (0 == pa)
-		kprintf("kvtophys ret 0!\n");
-	return (pa);
+	mp_enable_preemption_no_check();
+	return ((addr64_t)pa);
 }
+
diff --git a/osfmk/i386/pio.h b/osfmk/i386/pio.h
index bd2f2f6b9..a5de5df74 100644
--- a/osfmk/i386/pio.h
+++ b/osfmk/i386/pio.h
@@ -59,6 +59,9 @@
 #define I386_PIO_H
 #include <mach_assert.h>
 
+#if !MACH_ASSERT
+#include <architecture/i386/pio.h>
+#else
 typedef unsigned short i386_ioport_t;
 
 /* read a longword */
@@ -115,51 +118,6 @@ extern void		loutb(
 				i386_ioport_t	port,
 				char		* data,
 				int		count);
+#endif /* !MACH_ASSERT */
 
-#if defined(__GNUC__) && (!MACH_ASSERT)
-extern __inline__ unsigned long	inl(
-				i386_ioport_t port)
-{
-	unsigned long datum;
-	__asm__ volatile("inl %1, %0" : "=a" (datum) : "d" (port));
-	return(datum);
-}
-
-extern __inline__ unsigned short inw(
-				i386_ioport_t port)
-{
-	unsigned short datum;
-	__asm__ volatile(".byte 0x66; inl %1, %0" : "=a" (datum) : "d" (port));
-	return(datum);
-}
-
-extern __inline__ unsigned char inb(
-				i386_ioport_t port)
-{
-	unsigned char datum;
-	__asm__ volatile("inb %1, %0" : "=a" (datum) : "d" (port));
-	return(datum);
-}
-
-extern __inline__ void outl(
-				i386_ioport_t port,
-				unsigned long datum)
-{
-	__asm__ volatile("outl %0, %1" : : "a" (datum), "d" (port));
-}
-
-extern __inline__ void outw(
-				i386_ioport_t port,
-				unsigned short datum)
-{
-	__asm__ volatile(".byte 0x66; outl %0, %1" : : "a" (datum), "d" (port));
-}
-
-extern __inline__ void outb(
-				i386_ioport_t port,
-				unsigned char datum)
-{
-	__asm__ volatile("outb %0, %1" : : "a" (datum), "d" (port));
-}
-#endif /* defined(__GNUC__) && (!MACH_ASSERT) */
 #endif /* I386_PIO_H */
diff --git a/osfmk/i386/pmCPU.c b/osfmk/i386/pmCPU.c
new file mode 100644
index 000000000..63108e188
--- /dev/null
+++ b/osfmk/i386/pmCPU.c
@@ -0,0 +1,475 @@
+/*
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * CPU-specific power management support.
+ *
+ * Implements the "wrappers" to the KEXT.
+ */
+#include <i386/machine_routines.h>
+#include <i386/machine_cpu.h>
+#include <i386/misc_protos.h>
+#include <i386/pmap.h>
+#include <i386/asm.h>
+#include <i386/mp.h>
+#include <i386/proc_reg.h>
+#include <kern/pms.h>
+#include <kern/processor.h>
+#include <i386/pmCPU.h>
+#include <i386/cpuid.h>
+#include <i386/rtclock.h>
+#if MACH_KDB
+#include <i386/db_machdep.h>
+#include <ddb/db_aout.h>
+#include <ddb/db_access.h>
+#include <ddb/db_sym.h>
+#include <ddb/db_variables.h>
+#include <ddb/db_command.h>
+#include <ddb/db_output.h>
+#include <ddb/db_expr.h>
+#endif
+
+extern int disableConsoleOutput;
+
+decl_simple_lock_data(,pm_init_lock);
+
+/*
+ * The following is set when the KEXT loads and initializes.
+ */
+pmDispatch_t	*pmDispatch	= NULL;
+
+/*
+ * Current power management states (for use until KEXT is loaded).
+ */
+static pmInitState_t	pmInitState;
+
+/*
+ * Nap control variables:
+ */
+uint32_t napCtl = 0;			/* Defaults to neither napping
+					   nor halting */
+uint32_t forcenap = 0;			/* Force nap (fn) boot-arg controls */
+uint32_t maxBusDelay = 0xFFFFFFFF;	/* Maximum memory bus delay that
+					   I/O devices can tolerate
+					   before errors (nanoseconds) */
+uint32_t C4C2SnoopDelay = 0;		/* C4 to C2 transition time -
+					   time before a C4 system
+					   can snoop (nanoseconds) */
+
+/*
+ * We are being asked to set PState (sel).
+ */
+void
+pmsCPUSet(uint32_t sel)
+{
+    if (pmDispatch != NULL && pmDispatch->pmsCPUSet != NULL)
+	(*pmDispatch->pmsCPUSet)(sel);
+    else
+	pmInitState.PState = sel;
+}
+
+/*
+ * This code configures the initial step tables.  It should be called after
+ * the timebase frequency is initialized.
+ *
+ * Note that this is not used in normal operation.  It is strictly for
+ * debugging/testing purposes.
+ */
+void
+pmsCPUConf(void)
+{
+
+    if (pmDispatch != NULL && pmDispatch->pmsCPUConf != NULL)
+	(*pmDispatch->pmsCPUConf)();
+}
+
+/*
+ * Machine-dependent initialization.
+ */
+void
+pmsCPUMachineInit(void)
+{
+    /*
+     * Initialize some of the initial state to "uninitialized" until
+     * it gets set with something more useful.  This allows the KEXT
+     * to determine if the initial value was actually set to something.
+     */
+    pmInitState.PState = -1;
+    pmInitState.PLimit = -1;
+
+    if (pmDispatch != NULL && pmDispatch->pmsCPUMachineInit != NULL)
+	(*pmDispatch->pmsCPUMachineInit)();
+}
+
+/*
+ * This function should be called once for each processor to force the
+ * processor to the correct initial voltage and frequency.
+ */
+void
+pmsCPUInit(void)
+{
+    pmsCPUMachineInit();
+    if (pmDispatch != NULL && pmDispatch->pmsCPUInit != NULL)
+	(*pmDispatch->pmsCPUInit)();
+}
+
+/*
+ * Broadcast a change to all processing including ourselves.
+ */
+void
+pmsCPURun(uint32_t nstep)
+{
+    if (pmDispatch != NULL && pmDispatch->pmsCPURun != NULL)
+	(*pmDispatch->pmsCPURun)(nstep);
+}
+
+/*
+ * Return the current state of a core.
+ */
+uint32_t
+pmsCPUQuery(void)
+{
+    if (pmDispatch != NULL && pmDispatch->pmsCPUQuery != NULL)
+	return((*pmDispatch->pmsCPUQuery)());
+
+    /*
+     * Return a non-sense value.
+     */
+    return((~0) << 16);
+}
+
+/*
+ * Return the current state of the package.
+ */
+uint32_t
+pmsCPUPackageQuery(void)
+{
+    if (pmDispatch != NULL && pmDispatch->pmsCPUPackageQuery != NULL)
+	return((*pmDispatch->pmsCPUPackageQuery)());
+
+    /*
+     * Return a non-sense value.
+     */
+    return((~0) << 16);
+}
+
+/*
+ * Force the CPU package to the lowest power level.  This is a low-level
+ * interface meant to be called from the panic or debugger code to bring
+ * the CPU to a safe power level for unmanaged operation.
+ *
+ * Note that while this will bring an entire package to a safe level, it
+ * cannot affect other packages.  As a general rule, this should be run on
+ * every code as part of entering the debugger or on the panic path.
+ */
+void
+pmsCPUYellowFlag(void)
+{
+    if (pmDispatch != NULL && pmDispatch->pmsCPUYellowFlag != NULL)
+	(*pmDispatch->pmsCPUYellowFlag)();
+}
+
+/*
+ * Restore the CPU to the power state it was in before a yellow flag.
+ */
+void
+pmsCPUGreenFlag(void)
+{
+    if (pmDispatch != NULL && pmDispatch->pmsCPUGreenFlag != NULL)
+	(*pmDispatch->pmsCPUGreenFlag)();
+}
+
+/*
+ * Load a new ratio/VID table.
+ *
+ * Note that this interface is specific to the Intel SpeedStep implementation.
+ * It is expected that this will only be called once to override the default
+ * ratio/VID table when the platform starts.
+ *
+ * Normally, the table will need to be replaced at the same time that the
+ * stepper program proper is replaced, as the PState indices from an old
+ * program may no longer be valid.  When replacing the default program this
+ * should not be a problem as any new table will have at least two PState
+ * entries and the default program only references P0 and P1.
+ */
+kern_return_t
+pmsCPULoadVIDTable(uint16_t *tablep, int nstates)
+{
+    if (pmDispatch != NULL && pmDispatch->pmsCPULoadVIDTable != NULL)
+	return((*pmDispatch->pmsCPULoadVIDTable)(tablep, nstates));
+    else {
+	int	i;
+
+	if (nstates > MAX_PSTATES)
+	    return(KERN_FAILURE);
+
+	for (i = 0; i < nstates; i += 1)
+	    pmInitState.VIDTable[i] = tablep[i];
+    }
+    return(KERN_SUCCESS);
+}
+
+/*
+ * Set the (global) PState limit.  CPUs will not be permitted to run at
+ * a lower (more performant) PState than this.
+ */
+kern_return_t
+pmsCPUSetPStateLimit(uint32_t limit)
+{
+    if (pmDispatch != NULL && pmDispatch->pmsCPUSetPStateLimit != NULL)
+	return((*pmDispatch->pmsCPUSetPStateLimit)(limit));
+
+    pmInitState.PLimit = limit;
+    return(KERN_SUCCESS);
+}
+
+/*
+ * Initialize the Cstate change code.
+ */
+void
+power_management_init(void)
+{
+    uint32_t	cpuModel;
+    uint32_t	cpuFamily;
+    uint32_t	xcpuid[4];
+
+    /*
+     * Initialize the lock for the KEXT initialization.
+     */
+    simple_lock_init(&pm_init_lock, 0);
+
+    /*
+     * XXX
+     *
+     * The following is a hack to disable power management on some systems
+     * until the KEXT is done.  This is strictly temporary!!!
+     */
+    do_cpuid(1, xcpuid);
+    cpuFamily = (xcpuid[eax] >> 8) & 0xf;
+    cpuModel  = (xcpuid[eax] >> 4) & 0xf;
+
+    if (cpuFamily != 0x6 || cpuModel < 0xe)
+	pmDispatch = NULL;
+
+    if (pmDispatch != NULL && pmDispatch->cstateInit != NULL)
+	(*pmDispatch->cstateInit)();
+}
+
+/*
+ * This function will update the system nap policy.  It should be called
+ * whenever conditions change: when the system is ready to being napping
+ * and if something changes the rules (e.g. a sysctl altering the policy
+ * for debugging).
+ */
+void
+machine_nap_policy(void)
+{
+    if (pmDispatch != NULL && pmDispatch->cstateNapPolicy != NULL)
+	napCtl = (*pmDispatch->cstateNapPolicy)(forcenap, napCtl);
+}
+
+/*
+ * ACPI calls the following routine to set/update mwait hints.  A table
+ * (possibly null) specifies the available Cstates and their hints, all
+ * other states are assumed to be invalid.  ACPI may update available
+ * states to change the nap policy (for example, while AC power is
+ * available).
+ */
+kern_return_t
+Cstate_table_set(Cstate_hint_t *tablep, unsigned int nstates)
+{
+    if (forcenap)
+	return(KERN_SUCCESS);
+
+    if (pmDispatch != NULL && pmDispatch->cstateTableSet != NULL)
+	return((*pmDispatch->cstateTableSet)(tablep, nstates));
+    else {
+	unsigned int	i;
+
+	for (i = 0; i < nstates; i += 1) {
+	    pmInitState.CStates[i].number = tablep[i].number;
+	    pmInitState.CStates[i].hint   = tablep[i].hint;
+	}
+
+	pmInitState.CStatesCount = nstates;
+    }
+    return(KERN_SUCCESS);
+}
+
+static inline void
+sti(void) {
+	__asm__ volatile ( "sti" : : : "memory");
+}
+
+/*
+ * Called when the CPU is idle.  It will choose the best C state to
+ * be in.
+ */
+void
+machine_idle_cstate(void)
+{
+    if (pmDispatch != NULL && pmDispatch->cstateMachineIdle != NULL)
+	(*pmDispatch->cstateMachineIdle)(napCtl);
+    else {
+	sti();
+    }
+}
+
+static pmStats_t *
+pmsCPUStats(void)
+{
+    cpu_data_t	*pp;
+
+    pp = current_cpu_datap();
+    return(&pp->cpu_pmStats);
+}
+
+static pmsd *
+pmsCPUStepperData(void)
+{
+    cpu_data_t	*pp;
+
+    pp = current_cpu_datap();
+    return(&pp->pms);
+}
+
+static uint64_t *
+CPUHPETAddr(void)
+{
+    cpu_data_t	*pp;
+    pp = current_cpu_datap();
+    return(pp->cpu_pmHpet);
+}
+
+/*
+ * Called by the power management kext to register itself and to get the
+ * callbacks it might need into other power management functions.
+ */
+void
+pmRegister(pmDispatch_t *cpuFuncs, pmCallBacks_t *callbacks)
+{
+    if (callbacks != NULL) {
+	callbacks->Park        = pmsPark;
+	callbacks->Run         = pmsRun;
+	callbacks->RunLocal    = pmsRunLocal;
+	callbacks->SetStep     = pmsSetStep;
+	callbacks->NapPolicy   = machine_nap_policy;
+	callbacks->Build       = pmsBuild;
+	callbacks->Stats       = pmsCPUStats;
+	callbacks->StepperData = pmsCPUStepperData;
+	callbacks->HPETAddr    = CPUHPETAddr;
+	callbacks->InitState   = &pmInitState;
+	callbacks->resetPop    = resetPop;
+    }
+
+    if (cpuFuncs != NULL)
+	pmDispatch = cpuFuncs;
+}
+
+/*
+ * Unregisters the power management functions from the kext.
+ */
+void
+pmUnRegister(pmDispatch_t *cpuFuncs)
+{
+    if (cpuFuncs != NULL && pmDispatch == cpuFuncs)
+	pmDispatch = NULL;
+}
+
+#if MACH_KDB
+/*
+ * XXX stubs for now
+ */
+void
+db_cfg(__unused db_expr_t addr,
+       __unused int have_addr,
+       __unused db_expr_t count,
+       __unused char *modif)
+{
+    return;
+}
+
+void
+db_display_iokit(__unused db_expr_t addr,
+		 __unused int have_addr,
+		 __unused db_expr_t count,
+		 __unused char *modif)
+{
+    return;
+}
+
+void
+db_dtimers(__unused db_expr_t addr,
+	   __unused int have_addr,
+	   __unused db_expr_t count,
+	   __unused char *modif)
+{
+    return;
+}
+
+void
+db_intcnt(__unused db_expr_t addr,
+	  __unused int have_addr,
+	  __unused db_expr_t count,
+	  __unused char *modif)
+{
+    return;
+}
+
+void
+db_nap(__unused db_expr_t addr,
+       __unused int have_addr,
+       __unused db_expr_t count,
+       __unused char *modif)
+{
+    return;
+}
+
+void
+db_pmgr(__unused db_expr_t addr,
+	__unused int have_addr,
+	__unused db_expr_t count,
+	__unused char *modif)
+{
+    return;
+}
+
+void
+db_test(__unused db_expr_t addr,
+	__unused int have_addr,
+	__unused db_expr_t count,
+	__unused char *modif)
+{
+    return;
+}
+
+void
+db_getpmgr(__unused pmData_t *pmj)
+{
+}
+#endif
diff --git a/osfmk/i386/pmCPU.h b/osfmk/i386/pmCPU.h
new file mode 100644
index 000000000..6fa8c3016
--- /dev/null
+++ b/osfmk/i386/pmCPU.h
@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifdef KERNEL_PRIVATE
+#ifndef _I386_PMCPU_H_
+#define _I386_PMCPU_H_
+
+#include <kern/pms.h>
+
+#ifndef ASSEMBLER
+
+typedef enum { C1, C2, C3, C4, Hlt, C3Res, All, Cnum } pm_Cstate_t;
+typedef struct pmStats {
+	uint64_t	pmNapCnt[Cnum];		/* Total nap calls for states */
+	uint64_t	pmNapTime[Cnum];	/* Total nap time for states */
+	uint64_t	pmNapC2HPET;		/* Total nap time for C2 using HPET for stats */
+	uint64_t	pmNapC4HPET;		/* Total nap time for C4 using HPET for stats */
+	uint64_t	pmNapHPETPops;		/* Number of times we detect HPET popping */
+	uint64_t	pmHPETRupt;		/* Number of HPET interruptions */
+	uint32_t	pmCurC3Res;		/* Current value of the C3 residency timer */
+	uint32_t	pmLastApic;		/* Last value of apic timer */
+	uint32_t	pmNewApic;		/* New value of apic timer */
+	uint64_t	pmHpetTim;		/* Time to next interrupt in HPET ticks */
+	uint64_t	pmHpetCmp;		/* HPET comparator */
+	uint64_t	pmHpetCfg;		/* HPET configuration */
+	uint64_t	pmLSNs;			/* (TEST) Last set nanotime */
+	uint64_t	pmLLHpet;		/* (TEST) Last loaded HPET */
+} pmStats_t;
+
+#define MAX_PSTATES	32			/* architectural limit */
+
+typedef enum { Cn1, Cn2, Cn3, Cn4, Cnmax } Cstate_number_t;
+typedef struct {
+	Cstate_number_t	number;
+	uint32_t	hint;
+} Cstate_hint_t;
+
+
+struct pmData {
+	uint8_t pad[93];
+};
+typedef struct 	pmData pmData_t;
+
+#define pmNapHalt	0x00000010
+#define pmNapC1		0x00000008
+#define pmNapC2		0x00000004
+#define pmNapC3		0x00000002
+#define pmNapC4		0x00000001
+#define pmNapMask	0x000000FF
+
+#define cfgAdr 		0xCF8
+#define cfgDat 		0xCFC
+#define lpcCfg 		(0x80000000 | (0 << 16) | (31 << 11) | (0 << 8))
+
+/*
+ * Dispatch table for functions that get installed when the power
+ * management KEXT loads.
+ */
+typedef struct
+{
+    /*
+     * The following are the stepper table interfaces.
+     */
+    void	(*pmsCPUMachineInit)(void);
+    void	(*pmsCPUInit)(void);
+    void	(*pmsCPUSet)(uint32_t sel);
+    void	(*pmsCPUConf)(void);
+    void	(*pmsCPURun)(uint32_t nstep);
+    uint32_t	(*pmsCPUQuery)(void);
+    uint32_t	(*pmsCPUPackageQuery)(void);
+    void	(*pmsCPUYellowFlag)(void);
+    void	(*pmsCPUGreenFlag)(void);
+    kern_return_t	(*pmsCPULoadVIDTable)(uint16_t *tablep, int nstates);
+    kern_return_t	(*pmsCPUSetPStateLimit)(uint32_t limit);
+
+    /*
+     * The following are the 'C' State interfaces.
+     */
+    void	(*cstateInit)(void);
+    void	(*cstateMachineIdle)(uint32_t napCtl);
+    kern_return_t	(*cstateTableSet)(Cstate_hint_t *tablep, unsigned int nstates);
+    uint32_t	(*cstateNapPolicy)(uint32_t forcenap, uint32_t napCtl);
+} pmDispatch_t;
+
+typedef struct {
+    uint32_t		PState;
+    uint32_t		PLimit;
+    uint16_t		VIDTable[MAX_PSTATES];
+    uint32_t		VIDTableCount;
+    Cstate_hint_t	CStates[Cnmax];
+    uint32_t		CStatesCount;
+} pmInitState_t;
+
+typedef struct {
+    void	(*Park)(void);
+    void	(*Run)(uint32_t nstep);
+    void	(*RunLocal)(uint32_t nstep);
+    void	(*SetStep)(uint32_t nstep, int dir);
+    void	(*NapPolicy)(void);
+    kern_return_t	(*Build)(pmsDef *pd, uint32_t pdsize, pmsSetFunc_t *functab, uint32_t platformData, pmsQueryFunc_t queryFunc);
+    pmStats_t	*(*Stats)(void);
+    pmsd	*(*StepperData)(void);
+    uint64_t	*(*HPETAddr)(void);
+    pmInitState_t	*InitState;
+    void	(*resetPop)(void);
+} pmCallBacks_t;
+
+extern pmDispatch_t	*pmDispatch;
+
+extern uint32_t		maxBusDelay;
+extern uint32_t		C4C2SnoopDelay;
+extern uint32_t		forcenap;
+
+void power_management_init(void);
+void machine_nap_policy(void);
+kern_return_t Cstate_table_set(Cstate_hint_t *tablep, unsigned int nstates);
+void machine_idle_cstate(void);
+void pmRegister(pmDispatch_t *cpuFuncs, pmCallBacks_t *callbacks);
+void pmUnRegister(pmDispatch_t *cpuFuncs);
+
+#endif /* ASSEMBLER */
+#endif /* _I386_PMCPU_H_ */
+#endif /* KERNEL_PRIVATE */
diff --git a/osfmk/i386/pmap.c b/osfmk/i386/pmap.c
index cd845a8d0..a54c1ef33 100644
--- a/osfmk/i386/pmap.c
+++ b/osfmk/i386/pmap.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -122,6 +122,8 @@
 #include <i386/cpu_number.h>
 #include <i386/machine_cpu.h>
 #include <i386/mp_slave_boot.h>
+#include <i386/seg.h>
+#include <i386/cpu_capabilities.h>
 
 #if	MACH_KDB
 #include <ddb/db_command.h>
@@ -135,17 +137,35 @@
 #include <vm/vm_protos.h>
 
 #include <i386/mp.h>
+#include <i386/mp_desc.h>
+
+#include <sys/kdebug.h>
+
+#ifdef IWANTTODEBUG
+#undef	DEBUG
+#define DEBUG 1
+#define POSTCODE_DELAY 1
+#include <i386/postcode.h>
+#endif /* IWANTTODEBUG */
 
 /*
  * Forward declarations for internal functions.
  */
+void		pmap_expand_pml4(
+			pmap_t		map,
+			vm_map_offset_t	v);
+
+void		pmap_expand_pdpt(
+			pmap_t		map,
+			vm_map_offset_t	v);
+
 void		pmap_expand(
 			pmap_t		map,
-			vm_offset_t	v);
+			vm_map_offset_t	v);
 
-extern void	pmap_remove_range(
+static void	pmap_remove_range(
 			pmap_t		pmap,
-			vm_offset_t	va,
+			vm_map_offset_t	va,
 			pt_entry_t	*spte,
 			pt_entry_t	*epte);
 
@@ -161,9 +181,6 @@ void		phys_attribute_set(
 			ppnum_t	phys,
 			int		bits);
 
-void		pmap_growkernel(
-			vm_offset_t addr);
-
 void		pmap_set_reference(
 			ppnum_t pn);
 
@@ -172,24 +189,21 @@ void		pmap_movepage(
 			unsigned long	to,
 			vm_size_t	size);
 
-pt_entry_t *	pmap_mapgetpte(
-			vm_map_t	map,
-			vm_offset_t	v);
-
 boolean_t	phys_page_exists(
 			ppnum_t pn);
 
-#ifndef	set_dirbase
-void		set_dirbase(vm_offset_t	dirbase);
-#endif	/* set_dirbase */
+#ifdef PMAP_DEBUG
+void dump_pmap(pmap_t);
+void dump_4GB_pdpt(pmap_t p);
+void dump_4GB_pdpt_thread(thread_t tp);
+#endif
 
 #define	iswired(pte)	((pte) & INTEL_PTE_WIRED)
 
-#define	WRITE_PTE(pte_p, pte_entry)		*(pte_p) = (pte_entry);
-#define	WRITE_PTE_FAST(pte_p, pte_entry)	*(pte_p) = (pte_entry);
+int nx_enabled = 1;			/* enable no-execute protection */
+
+int cpu_64bit  = 0;
 
-#define value_64bit(value)  ((value) & 0xFFFFFFFF00000000LL)
-#define low32(x) ((unsigned int)((x) & 0x00000000ffffffffLL))
 
 /*
  *	Private data structures.
@@ -204,7 +218,7 @@ void		set_dirbase(vm_offset_t	dirbase);
 typedef struct pv_entry {
 	struct pv_entry	*next;		/* next pv_entry */
 	pmap_t		pmap;		/* pmap where mapping lies */
-	vm_offset_t	va;		/* virtual address for mapping */
+	vm_map_offset_t	va;		/* virtual address for mapping */
 } *pv_entry_t;
 
 #define PV_ENTRY_NULL	((pv_entry_t) 0)
@@ -231,7 +245,7 @@ int mappingrecurse = 0;
 	    pv_free_list = pv_e->next; \
             pv_free_count--; \
             if (pv_free_count < PV_LOW_WATER_MARK) \
-              if (hw_compare_and_store(0,1,&mappingrecurse)) \
+              if (hw_compare_and_store(0,1,(u_int *)&mappingrecurse)) \
                 thread_call_enter(mapping_adjust_call); \
 	} \
 	simple_unlock(&pv_free_list_lock); \
@@ -247,10 +261,7 @@ int mappingrecurse = 0;
 
 zone_t		pv_list_zone;		/* zone of pv_entry structures */
 
-#ifdef PAE
 static zone_t pdpt_zone;
-#endif
-
 
 /*
  *	Each entry in the pv_head_table is locked by a bit in the
@@ -270,14 +281,8 @@ pmap_paddr_t	vm_first_phys = (pmap_paddr_t) 0;
 pmap_paddr_t	vm_last_phys  = (pmap_paddr_t) 0;
 boolean_t	pmap_initialized = FALSE;/* Has pmap_init completed? */
 
-pmap_paddr_t    kernel_vm_end = (pmap_paddr_t)0;
-
-#define GROW_KERNEL_FUNCTION_IMPLEMENTED 1
-#if GROW_KERNEL_FUNCTION_IMPLEMENTED  /* not needed until growing kernel pmap */
 static struct vm_object kptobj_object_store;
 static vm_object_t kptobj;
-#endif
-
 
 /*
  *	Index into pv_head table, its lock bits, and the modify/reference
@@ -308,6 +313,7 @@ char	*pmap_phys_attributes;
  *	page-directory entry.
  */
 #define	PDE_MAPPED_SIZE		(pdetova(1))
+uint64_t pde_mapped_size;
 
 /*
  *	Locking and TLB invalidation
@@ -348,23 +354,21 @@ char	*pmap_phys_attributes;
 
 /*
  *	We raise the interrupt level to splvm, to block interprocessor
- *	interrupts during pmap operations.  We must take the CPU out of
- *	the cpus_active set while interrupts are blocked.
+ *	interrupts during pmap operations.  We mark the cpu's cr3 inactive
+ *	while interrupts are blocked.
  */
-#define SPLVM(spl)	{ \
-	spl = splhigh(); \
-	mp_disable_preemption(); \
-	i_bit_clear(cpu_number(), &cpus_active); \
-	mp_enable_preemption(); \
+#define SPLVM(spl)	{						\
+	spl = splhigh();						\
+	CPU_CR3_MARK_INACTIVE();					\
 }
 
-#define SPLX(spl)	{ \
-	mp_disable_preemption(); \
-	i_bit_set(cpu_number(), &cpus_active); \
-	mp_enable_preemption(); \
-	splx(spl); \
+#define SPLX(spl)	{						\
+	if (current_cpu_datap()->cpu_tlb_invalid)			\
+	    process_pmap_updates();					\
+	CPU_CR3_MARK_ACTIVE();						\
+	splx(spl);							\
 }
-
+	    
 /*
  *	Lock on pmap system
  */
@@ -411,8 +415,8 @@ extern int	disableSerialOuput;
 #define LOOP_CHECK(msg, pmap)						\
 	if (--loop_count == 0) {					\
 		mp_disable_preemption();				\
-	    	kprintf("%s: cpu %d pmap %x, cpus_active 0x%x\n",	\
-			  msg, cpu_number(), pmap, cpus_active);	\
+	    	kprintf("%s: cpu %d pmap %x\n",				\
+			  msg, cpu_number(), pmap);			\
             	Debugger("deadlock detection");				\
 		mp_enable_preemption();					\
 		loop_count = max_lock_loops;				\
@@ -422,76 +426,15 @@ extern int	disableSerialOuput;
 #define LOOP_CHECK(msg, pmap)
 #endif	/* USLOCK_DEBUG */
 
-#define PMAP_UPDATE_TLBS(pmap, s, e)					\
-{									\
-	cpu_set	cpu_mask;						\
-	cpu_set	users;							\
-									\
-	mp_disable_preemption();					\
-	cpu_mask = 1 << cpu_number();					\
-									\
-	/* Since the pmap is locked, other updates are locked */ 	\
-	/* out, and any pmap_activate has finished. */ 			\
- 									\
-	/* find other cpus using the pmap */ 				\
-	users = (pmap)->cpus_using & ~cpu_mask;        			\
-	if (users) { 							\
-            LOOP_VAR;							\
-	    /* signal them, and wait for them to finish */ 		\
-	    /* using the pmap */ 					\
-	    signal_cpus(users, (pmap), (s), (e));      			\
-	    while (((pmap)->cpus_using & cpus_active & ~cpu_mask)) {	\
-		LOOP_CHECK("PMAP_UPDATE_TLBS", pmap);			\
-		cpu_pause(); 						\
-	    }								\
-	} 								\
-	/* invalidate our own TLB if pmap is in use */ 			\
- 									\
-	if ((pmap)->cpus_using & cpu_mask) {   				\
-	    INVALIDATE_TLB((pmap), (s), (e));				\
-	} 								\
-									\
-	mp_enable_preemption();						\
-}
-
-#define MAX_TBIS_SIZE	32		/* > this -> TBIA */ /* XXX */
-
-#define INVALIDATE_TLB(m, s, e) {	\
-	flush_tlb(); 			\
-}
-
-/*
- *	Structures to keep track of pending TLB invalidations
- */
-cpu_set			cpus_active;
-cpu_set			cpus_idle;
 
-#define UPDATE_LIST_SIZE	4
+static void pmap_flush_tlbs(pmap_t pmap);
 
-struct pmap_update_item {
-	pmap_t		pmap;		/* pmap to invalidate */
-	vm_offset_t	start;		/* start address to invalidate */
-	vm_offset_t	end;		/* end address to invalidate */
-};
+#define PMAP_UPDATE_TLBS(pmap, s, e)					\
+	pmap_flush_tlbs(pmap)
 
-typedef	struct pmap_update_item	*pmap_update_item_t;
 
-/*
- *	List of pmap updates.  If the list overflows,
- *	the last entry is changed to invalidate all.
- */
-struct pmap_update_list {
-	decl_simple_lock_data(,lock)
-	int			count;
-	struct pmap_update_item	item[UPDATE_LIST_SIZE];
-} ;
-typedef	struct pmap_update_list	*pmap_update_list_t;
+#define MAX_TBIS_SIZE	32		/* > this -> TBIA */ /* XXX */
 
-extern void signal_cpus(
-			cpu_set		use_list,
-			pmap_t		pmap,
-			vm_offset_t	start,
-			vm_offset_t	end);
 
 pmap_memory_region_t pmap_memory_regions[PMAP_MEMORY_REGIONS_SIZE];
 
@@ -499,14 +442,12 @@ pmap_memory_region_t pmap_memory_regions[PMAP_MEMORY_REGIONS_SIZE];
  *	Other useful macros.
  */
 #define current_pmap()		(vm_map_pmap(current_thread()->map))
-#define pmap_in_use(pmap, cpu)	(((pmap)->cpus_using & (1 << (cpu))) != 0)
 
 struct pmap	kernel_pmap_store;
 pmap_t		kernel_pmap;
 
-#ifdef PMAP_QUEUE
-decl_simple_lock_data(,free_pmap_lock)
-#endif
+pd_entry_t    high_shared_pde;
+pd_entry_t    commpage64_pde;
 
 struct zone	*pmap_zone;		/* zone of pmap structures */
 
@@ -514,6 +455,9 @@ int		pmap_debug = 0;		/* flag for debugging prints */
 
 unsigned int	inuse_ptepages_count = 0;	/* debugging */
 
+addr64_t	kernel64_cr3;
+boolean_t	no_shared_cr3 = FALSE;	/* -no_shared_cr3 boot arg */
+
 /*
  *	Pmap cache.  Cache is threaded through ref_count field of pmap.
  *	Max will eventually be constant -- variable for experimentation.
@@ -524,11 +468,11 @@ pmap_t		pmap_cache_list;
 int		pmap_cache_count;
 decl_simple_lock_data(,pmap_cache_lock)
 
-extern	vm_offset_t	hole_start, hole_end;
-
 extern char end;
 
 static int nkpt;
+extern uint32_t lowGlo;
+extern void *version;
 
 pt_entry_t     *DMAP1, *DMAP2;
 caddr_t         DADDR1;
@@ -539,7 +483,7 @@ caddr_t         DADDR2;
 struct pmap_alias {
         vm_offset_t rpc;
         pmap_t pmap;
-        vm_offset_t va;
+        vm_map_offset_t va;
         int cookie;
 #define PMAP_ALIAS_COOKIE 0xdeadbeef
 } pmap_aliasbuf[PMAP_ALIAS_MAX];
@@ -548,73 +492,183 @@ extern vm_offset_t get_rpc();
 
 #endif  /* DEBUG_ALIAS */
 
-#define	pmap_pde(m, v)	(&((m)->dirbase[(vm_offset_t)(v) >> PDESHIFT]))
-#define pdir_pde(d, v) (d[(vm_offset_t)(v) >> PDESHIFT])
-
-static __inline int
-pmap_is_current(pmap_t pmap)
+/*
+ * for legacy, returns the address of the pde entry.
+ * for 64 bit, causes the pdpt page containing the pde entry to be mapped,
+ * then returns the mapped address of the pde entry in that page
+ */
+pd_entry_t *
+pmap_pde(pmap_t m, vm_map_offset_t v)
 {
-  return (pmap == kernel_pmap ||
-	  (pmap->dirbase[PTDPTDI] & PG_FRAME) == (PTDpde[0] & PG_FRAME));
+  pd_entry_t *pde;
+	if (!cpu_64bit || (m == kernel_pmap)) {
+	  pde = (&((m)->dirbase[(vm_offset_t)(v) >> PDESHIFT]));
+	} else {
+	  assert(m);
+	  assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
+	  pde = pmap64_pde(m, v);
+	}
+	return pde;
 }
 
 
 /*
- * return address of mapped pte for vaddr va in pmap pmap.
+ * the single pml4 page per pmap is allocated at pmap create time and exists
+ * for the duration of the pmap. we allocate this page in kernel vm (to save us one
+ * level of page table dynamic mapping.
+ * this returns the address of the requested pml4 entry in the top level page.
  */
-pt_entry_t     *
-pmap_pte(pmap_t pmap, vm_offset_t va)
-{
-  pd_entry_t     *pde;
-  pd_entry_t     newpf;
-
-  pde = pmap_pde(pmap, va);
-  if (*pde != 0) {
-    if (pmap_is_current(pmap))
-      return( vtopte(va));
-    newpf = *pde & PG_FRAME;
-    if (((*CM4) & PG_FRAME) != newpf) {
-      *CM4 = newpf | INTEL_PTE_RW | INTEL_PTE_VALID;
-      invlpg((u_int)CA4);
-    }
-    return (pt_entry_t *)CA4 + (i386_btop(va) & (NPTEPG-1));
+static inline
+pml4_entry_t *
+pmap64_pml4(pmap_t pmap, vm_map_offset_t vaddr)
+{
+  return ((pml4_entry_t *)pmap->pm_hold + ((vm_offset_t)((vaddr>>PML4SHIFT)&(NPML4PG-1))));
+}
+
+/*
+ * maps in the pml4 page, if any, containing the pdpt entry requested
+ * and returns the address of the pdpt entry in that mapped page
+ */
+pdpt_entry_t *
+pmap64_pdpt(pmap_t pmap, vm_map_offset_t vaddr)
+{
+  pml4_entry_t newpf;
+  pml4_entry_t *pml4;
+  int i;
+
+  assert(pmap);
+  assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
+  if ((vaddr > 0x00007FFFFFFFFFFFULL) && (vaddr < 0xFFFF800000000000ULL)) {
+    return(0);
   }
-  return(0);
+
+  pml4 = pmap64_pml4(pmap, vaddr);
+
+	if (pml4 && ((*pml4 & INTEL_PTE_VALID))) {
+
+		newpf = *pml4 & PG_FRAME;
+
+
+		for (i=PMAP_PDPT_FIRST_WINDOW; i < PMAP_PDPT_FIRST_WINDOW+PMAP_PDPT_NWINDOWS; i++) {
+		  if (((*(current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CMAP)) & PG_FRAME) == newpf) {
+		  return((pdpt_entry_t *)(current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CADDR) + 
+			 ((vm_offset_t)((vaddr>>PDPTSHIFT)&(NPDPTPG-1))));
+		  }
+		}
+
+		  current_cpu_datap()->cpu_pmap->pdpt_window_index++;
+		  if (current_cpu_datap()->cpu_pmap->pdpt_window_index > (PMAP_PDPT_FIRST_WINDOW+PMAP_PDPT_NWINDOWS-1))
+		    current_cpu_datap()->cpu_pmap->pdpt_window_index = PMAP_PDPT_FIRST_WINDOW;
+		  pmap_store_pte(
+				 (current_cpu_datap()->cpu_pmap->mapwindow[current_cpu_datap()->cpu_pmap->pdpt_window_index].prv_CMAP),
+				 newpf | INTEL_PTE_RW | INTEL_PTE_VALID);
+		  invlpg((u_int)(current_cpu_datap()->cpu_pmap->mapwindow[current_cpu_datap()->cpu_pmap->pdpt_window_index].prv_CADDR));
+		  return ((pdpt_entry_t *)(current_cpu_datap()->cpu_pmap->mapwindow[current_cpu_datap()->cpu_pmap->pdpt_window_index].prv_CADDR) +
+			  ((vm_offset_t)((vaddr>>PDPTSHIFT)&(NPDPTPG-1))));
+	}
+
+	return (0);
 }
-	
-#define DEBUG_PTE_PAGE	0
 
-#if	DEBUG_PTE_PAGE
-void
-ptep_check(
-	ptep_t	ptep)
+/*
+ * maps in the pdpt page, if any, containing the pde entry requested
+ * and returns the address of the pde entry in that mapped page
+ */
+pd_entry_t *
+pmap64_pde(pmap_t pmap, vm_map_offset_t vaddr)
 {
-	register pt_entry_t	*pte, *epte;
-	int			ctu, ctw;
+  pdpt_entry_t newpf;
+  pdpt_entry_t *pdpt;
+  int i;
 
-	/* check the use and wired counts */
-	if (ptep == PTE_PAGE_NULL)
-		return;
-	pte = pmap_pte(ptep->pmap, ptep->va);
-	epte = pte + INTEL_PGBYTES/sizeof(pt_entry_t);
-	ctu = 0;
-	ctw = 0;
-	while (pte < epte) {
-		if (pte->pfn != 0) {
-			ctu++;
-			if (pte->wired)
-				ctw++;
+  assert(pmap);
+  assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
+  if ((vaddr > 0x00007FFFFFFFFFFFULL) && (vaddr < 0xFFFF800000000000ULL)) {
+    return(0);
+  }
+
+  /*  if (vaddr & (1ULL << 63)) panic("neg addr");*/
+  pdpt = pmap64_pdpt(pmap, vaddr);
+
+	  if (pdpt && ((*pdpt & INTEL_PTE_VALID))) {
+
+		newpf = *pdpt & PG_FRAME;
+
+		for (i=PMAP_PDE_FIRST_WINDOW; i < PMAP_PDE_FIRST_WINDOW+PMAP_PDE_NWINDOWS; i++) {
+		  if (((*(current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CMAP)) & PG_FRAME) == newpf) {
+		  return((pd_entry_t *)(current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CADDR) + 
+			 ((vm_offset_t)((vaddr>>PDSHIFT)&(NPDPG-1))));
+		  }
 		}
-		pte++;
+
+		  current_cpu_datap()->cpu_pmap->pde_window_index++;
+		  if (current_cpu_datap()->cpu_pmap->pde_window_index > (PMAP_PDE_FIRST_WINDOW+PMAP_PDE_NWINDOWS-1))
+		    current_cpu_datap()->cpu_pmap->pde_window_index = PMAP_PDE_FIRST_WINDOW;
+		  pmap_store_pte(
+				 (current_cpu_datap()->cpu_pmap->mapwindow[current_cpu_datap()->cpu_pmap->pde_window_index].prv_CMAP),
+				 newpf | INTEL_PTE_RW | INTEL_PTE_VALID);
+		  invlpg((u_int)(current_cpu_datap()->cpu_pmap->mapwindow[current_cpu_datap()->cpu_pmap->pde_window_index].prv_CADDR));
+		  return ((pd_entry_t *)(current_cpu_datap()->cpu_pmap->mapwindow[current_cpu_datap()->cpu_pmap->pde_window_index].prv_CADDR) +
+			  ((vm_offset_t)((vaddr>>PDSHIFT)&(NPDPG-1))));
 	}
 
-	if (ctu != ptep->use_count || ctw != ptep->wired_count) {
-		printf("use %d wired %d - actual use %d wired %d\n",
-		    	ptep->use_count, ptep->wired_count, ctu, ctw);
-		panic("pte count");
+	return (0);
+}
+
+
+
+/*
+ * return address of mapped pte for vaddr va in pmap pmap.
+ * must be called with pre-emption or interrupts disabled
+ * if targeted pmap is not the kernel pmap
+ * since we may be passing back a virtual address that is
+ * associated with this cpu... pre-emption or interrupts
+ * must remain disabled until the caller is done using
+ * the pointer that was passed back .
+ *
+ * maps the pde page, if any, containing the pte in and returns
+ * the address of the pte in that mapped page
+ */
+pt_entry_t     *
+pmap_pte(pmap_t pmap, vm_map_offset_t vaddr)
+{
+        pd_entry_t     *pde;
+	pd_entry_t     newpf;
+	int i;
+
+	assert(pmap);
+	pde = pmap_pde(pmap,vaddr);
+
+	if (pde && ((*pde & INTEL_PTE_VALID))) {
+	  if (pmap == kernel_pmap) {
+	    return (vtopte(vaddr)); /* compat kernel still has pte's mapped */
+	  }
+
+	        assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
+
+		newpf = *pde & PG_FRAME;
+
+		for (i=PMAP_PTE_FIRST_WINDOW; i < PMAP_PTE_FIRST_WINDOW+PMAP_PTE_NWINDOWS; i++) {
+		  if (((*(current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CMAP)) & PG_FRAME) == newpf) {
+		  return((pt_entry_t *)(current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CADDR) + 
+			 ((vm_offset_t)i386_btop(vaddr) & (NPTEPG-1)));
+		  }
+		}
+
+		  current_cpu_datap()->cpu_pmap->pte_window_index++;
+		  if (current_cpu_datap()->cpu_pmap->pte_window_index > (PMAP_PTE_FIRST_WINDOW+PMAP_PTE_NWINDOWS-1))
+		    current_cpu_datap()->cpu_pmap->pte_window_index = PMAP_PTE_FIRST_WINDOW;
+		  pmap_store_pte(
+				 (current_cpu_datap()->cpu_pmap->mapwindow[current_cpu_datap()->cpu_pmap->pte_window_index].prv_CMAP),
+				 newpf | INTEL_PTE_RW | INTEL_PTE_VALID);
+		  invlpg((u_int)(current_cpu_datap()->cpu_pmap->mapwindow[current_cpu_datap()->cpu_pmap->pte_window_index].prv_CADDR));
+		  return ((pt_entry_t *)(current_cpu_datap()->cpu_pmap->mapwindow[current_cpu_datap()->cpu_pmap->pte_window_index].prv_CADDR) +
+			  ((vm_offset_t)i386_btop(vaddr) & (NPTEPG-1)));
 	}
+
+	return(0);
 }
-#endif	/* DEBUG_PTE_PAGE */
+
 
 /*
  *	Map memory at initialization.  The physical addresses being
@@ -625,17 +679,18 @@ ptep_check(
  */
 vm_offset_t
 pmap_map(
-	register vm_offset_t	virt,
-	register vm_offset_t	start_addr,
-	register vm_offset_t	end_addr,
-	register vm_prot_t	prot)
+	vm_offset_t	virt,
+	vm_map_offset_t	start_addr,
+	vm_map_offset_t	end_addr,
+	vm_prot_t	prot,
+	unsigned int	flags)
 {
-	register int		ps;
+	int		ps;
 
 	ps = PAGE_SIZE;
 	while (start_addr < end_addr) {
-		pmap_enter(kernel_pmap,
-			virt, (ppnum_t) i386_btop(start_addr), prot, 0, FALSE);
+		pmap_enter(kernel_pmap, (vm_map_offset_t)virt,
+			   (ppnum_t) i386_btop(start_addr), prot, flags, FALSE);
 		virt += ps;
 		start_addr += ps;
 	}
@@ -651,30 +706,36 @@ pmap_map(
  */
 vm_offset_t
 pmap_map_bd(
-	register vm_offset_t	virt,
-	register vm_offset_t	start_addr,
-	register vm_offset_t	end_addr,
-	vm_prot_t		prot)
+	vm_offset_t	virt,
+	vm_map_offset_t	start_addr,
+	vm_map_offset_t	end_addr,
+	vm_prot_t	prot,
+	unsigned int	flags)
 {
-	register pt_entry_t	template;
-	register pt_entry_t	*pte;
+	pt_entry_t	template;
+	pt_entry_t	*pte;
 
 	template = pa_to_pte(start_addr)
-		| INTEL_PTE_NCACHE
 		| INTEL_PTE_REF
 		| INTEL_PTE_MOD
 		| INTEL_PTE_WIRED
 		| INTEL_PTE_VALID;
+
+	if(flags & (VM_MEM_NOT_CACHEABLE | VM_WIMG_USE_DEFAULT)) {
+	    template |= INTEL_PTE_NCACHE;
+	    if(!(flags & (VM_MEM_GUARDED | VM_WIMG_USE_DEFAULT)))
+		    template |= INTEL_PTE_PTA;
+	}
+
 	if (prot & VM_PROT_WRITE)
 	    template |= INTEL_PTE_WRITE;
 
-	/* XXX move pmap_pte out of loop, once one pte mapped, all are */
 	while (start_addr < end_addr) {
-		pte = pmap_pte(kernel_pmap, virt);
+		pte = pmap_pte(kernel_pmap, (vm_map_offset_t)virt);
 		if (pte == PT_ENTRY_NULL) {
 			panic("pmap_map_bd: Invalid kernel address\n");
 		}
-		WRITE_PTE_FAST(pte, template)
+		pmap_store_pte(pte, template);
 		pte_increment_pa(template);
 		virt += PAGE_SIZE;
 		start_addr += PAGE_SIZE;
@@ -691,6 +752,142 @@ extern  vm_offset_t     etext;
 extern  void            *sectHIBB;
 extern  int             sectSizeHIB;
 
+
+vm_offset_t
+pmap_high_shared_remap(enum high_fixed_addresses e, vm_offset_t va, int sz)
+{
+  vm_offset_t ve = pmap_index_to_virt(e);
+  pt_entry_t *ptep;
+  pmap_paddr_t pa;
+  int i;
+
+  assert(0 == (va & PAGE_MASK));  /* expecting page aligned */
+  ptep = pmap_pte(kernel_pmap, (vm_map_offset_t)ve);
+
+  for (i=0; i< sz; i++) {
+    pa = (pmap_paddr_t) kvtophys(va);
+    pmap_store_pte(ptep, (pa & PG_FRAME)
+				| INTEL_PTE_VALID
+		                | INTEL_PTE_GLOBAL
+				| INTEL_PTE_RW
+				| INTEL_PTE_REF
+				| INTEL_PTE_MOD);
+    va+= PAGE_SIZE;
+    ptep++;
+  }
+  return ve;
+}
+
+vm_offset_t
+pmap_cpu_high_shared_remap(int cpu, enum high_cpu_types e, vm_offset_t va, int sz)
+{ 
+  enum high_fixed_addresses	a = e + HIGH_CPU_END * cpu;
+  return pmap_high_shared_remap(HIGH_FIXED_CPUS_BEGIN + a, va, sz);
+}
+
+void pmap_init_high_shared(void);
+
+extern vm_offset_t gdtptr, idtptr;
+
+extern uint32_t low_intstack;
+
+extern struct fake_descriptor ldt_desc_pattern;
+extern struct fake_descriptor tss_desc_pattern;
+
+extern char hi_remap_text, hi_remap_etext;
+extern char t_zero_div;
+
+pt_entry_t *pte_unique_base;
+
+void
+pmap_init_high_shared(void)
+{
+
+	vm_offset_t haddr;
+        struct __gdt_desc_struct gdt_desc = {0,0,0};
+	struct __idt_desc_struct idt_desc = {0,0,0};
+#if MACH_KDB
+	struct i386_tss *ttss;
+#endif
+
+	kprintf("HIGH_MEM_BASE 0x%x fixed per-cpu begin 0x%x\n", 
+		HIGH_MEM_BASE,pmap_index_to_virt(HIGH_FIXED_CPUS_BEGIN));
+	pte_unique_base = pmap_pte(kernel_pmap, (vm_map_offset_t)pmap_index_to_virt(HIGH_FIXED_CPUS_BEGIN));
+
+	if (i386_btop(&hi_remap_etext - &hi_remap_text + 1) >
+				HIGH_FIXED_TRAMPS_END - HIGH_FIXED_TRAMPS + 1)
+		panic("tramps too large");
+	haddr = pmap_high_shared_remap(HIGH_FIXED_TRAMPS,
+					(vm_offset_t) &hi_remap_text, 3);
+	kprintf("tramp: 0x%x, ",haddr);
+	printf("hi mem tramps at 0x%x\n",haddr);
+	/* map gdt up high and update ptr for reload */
+	haddr = pmap_high_shared_remap(HIGH_FIXED_GDT,
+					(vm_offset_t) master_gdt, 1);
+	__asm__ __volatile__("sgdt %0": "=m" (gdt_desc): :"memory");
+	gdt_desc.address = haddr;
+	kprintf("GDT: 0x%x, ",haddr);
+	/* map ldt up high */
+	haddr = pmap_high_shared_remap(HIGH_FIXED_LDT_BEGIN,
+					(vm_offset_t) master_ldt,
+					HIGH_FIXED_LDT_END - HIGH_FIXED_LDT_BEGIN + 1);
+	kprintf("LDT: 0x%x, ",haddr);
+	/* put new ldt addr into gdt */
+	master_gdt[sel_idx(KERNEL_LDT)] = ldt_desc_pattern;
+	master_gdt[sel_idx(KERNEL_LDT)].offset = (vm_offset_t) haddr;
+	fix_desc(&master_gdt[sel_idx(KERNEL_LDT)], 1);
+	master_gdt[sel_idx(USER_LDT)] = ldt_desc_pattern;
+	master_gdt[sel_idx(USER_LDT)].offset = (vm_offset_t) haddr;
+	fix_desc(&master_gdt[sel_idx(USER_LDT)], 1);
+
+	/* map idt up high */
+	haddr = pmap_high_shared_remap(HIGH_FIXED_IDT,
+					(vm_offset_t) master_idt, 1);
+	__asm__ __volatile__("sidt %0" : "=m" (idt_desc));
+	idt_desc.address = haddr;
+	kprintf("IDT: 0x%x, ", haddr);
+	/* remap ktss up high and put new high addr into gdt */
+	haddr = pmap_high_shared_remap(HIGH_FIXED_KTSS,
+					(vm_offset_t) &master_ktss, 1);
+	master_gdt[sel_idx(KERNEL_TSS)] = tss_desc_pattern;
+	master_gdt[sel_idx(KERNEL_TSS)].offset = (vm_offset_t) haddr;
+	fix_desc(&master_gdt[sel_idx(KERNEL_TSS)], 1);
+	kprintf("KTSS: 0x%x, ",haddr);
+#if MACH_KDB
+	/* remap dbtss up high and put new high addr into gdt */
+	haddr = pmap_high_shared_remap(HIGH_FIXED_DBTSS,
+					(vm_offset_t) &master_dbtss, 1);
+	master_gdt[sel_idx(DEBUG_TSS)] = tss_desc_pattern;
+	master_gdt[sel_idx(DEBUG_TSS)].offset = (vm_offset_t) haddr;
+	fix_desc(&master_gdt[sel_idx(DEBUG_TSS)], 1);
+	ttss = (struct i386_tss *)haddr;
+	kprintf("DBTSS: 0x%x, ",haddr);
+#endif	/* MACH_KDB */
+
+	/* remap dftss up high and put new high addr into gdt */
+	haddr = pmap_high_shared_remap(HIGH_FIXED_DFTSS,
+					(vm_offset_t) &master_dftss, 1);
+	master_gdt[sel_idx(DF_TSS)] = tss_desc_pattern;
+	master_gdt[sel_idx(DF_TSS)].offset = (vm_offset_t) haddr;
+	fix_desc(&master_gdt[sel_idx(DF_TSS)], 1);
+	kprintf("DFTSS: 0x%x\n",haddr);
+
+	/* remap mctss up high and put new high addr into gdt */
+	haddr = pmap_high_shared_remap(HIGH_FIXED_DFTSS,
+					(vm_offset_t) &master_mctss, 1);
+	master_gdt[sel_idx(MC_TSS)] = tss_desc_pattern;
+	master_gdt[sel_idx(MC_TSS)].offset = (vm_offset_t) haddr;
+	fix_desc(&master_gdt[sel_idx(MC_TSS)], 1);
+	kprintf("MCTSS: 0x%x\n",haddr);
+
+	__asm__ __volatile__("lgdt %0": "=m" (gdt_desc));
+	__asm__ __volatile__("lidt %0": "=m" (idt_desc));
+	kprintf("gdt/idt reloaded, ");
+	set_tr(KERNEL_TSS);
+	kprintf("tr reset to KERNEL_TSS\n");
+}
+
+
 /*
  *	Bootstrap the system enough to run with virtual memory.
  *	Map the kernel's code and data, and allocate the system page table.
@@ -712,50 +909,60 @@ extern  int             sectSizeHIB;
 
 void
 pmap_bootstrap(
-	__unused vm_offset_t	load_start)
+	__unused vm_offset_t	load_start,
+	boolean_t		IA32e)
 {
 	vm_offset_t	va;
 	pt_entry_t	*pte;
 	int i;
 	int wpkernel, boot_arg;
+	pdpt_entry_t *pdpt;
 
 	vm_last_addr = VM_MAX_KERNEL_ADDRESS;	/* Set the highest address
 						 * known to VM */
-
 	/*
 	 *	The kernel's pmap is statically allocated so we don't
 	 *	have to use pmap_create, which is unlikely to work
 	 *	correctly at this part of the boot sequence.
 	 */
 
+
 	kernel_pmap = &kernel_pmap_store;
-#ifdef PMAP_QUEUE
-	kernel_pmap->pmap_link.next = (queue_t)kernel_pmap;		/* Set up anchor forward */
-	kernel_pmap->pmap_link.prev = (queue_t)kernel_pmap;		/* Set up anchor reverse */
-#endif
 	kernel_pmap->ref_count = 1;
+	kernel_pmap->nx_enabled = FALSE;
+	kernel_pmap->pm_64bit = 0;
 	kernel_pmap->pm_obj = (vm_object_t) NULL;
 	kernel_pmap->dirbase = (pd_entry_t *)((unsigned int)IdlePTD | KERNBASE);
-	kernel_pmap->pdirbase = (pd_entry_t *)IdlePTD;
-#ifdef PAE
-	kernel_pmap->pm_pdpt = (pd_entry_t *)((unsigned int)IdlePDPT | KERNBASE );
-	kernel_pmap->pm_ppdpt = (vm_offset_t)IdlePDPT;
-#endif
+	kernel_pmap->pdirbase = (pmap_paddr_t)((int)IdlePTD);
+	pdpt = (pd_entry_t *)((unsigned int)IdlePDPT | KERNBASE );
+	kernel_pmap->pm_pdpt = pdpt;
+	kernel_pmap->pm_cr3 = (pmap_paddr_t)((int)IdlePDPT);
 
 	va = (vm_offset_t)kernel_pmap->dirbase;
 	/* setup self referential mapping(s) */
-	for (i = 0; i< NPGPTD; i++ ) {
+	for (i = 0; i< NPGPTD; i++, pdpt++) {
 	  pmap_paddr_t pa;
 	  pa = (pmap_paddr_t) kvtophys(va + i386_ptob(i));
-	  * (pd_entry_t *) (kernel_pmap->dirbase + PTDPTDI + i) = 
+	  pmap_store_pte(
+	    (pd_entry_t *) (kernel_pmap->dirbase + PTDPTDI + i),
 	    (pa & PG_FRAME) | INTEL_PTE_VALID | INTEL_PTE_RW | INTEL_PTE_REF |
-	    INTEL_PTE_MOD | INTEL_PTE_WIRED ;
-#ifdef PAE
-	  kernel_pmap->pm_pdpt[i] = pa | INTEL_PTE_VALID;
-#endif
+	      INTEL_PTE_MOD | INTEL_PTE_WIRED) ;
+	  pmap_store_pte(pdpt, pa | INTEL_PTE_VALID);
 	}
 
+	cpu_64bit = IA32e;
+	
+	lo_kernel_cr3 = kernel_pmap->pm_cr3;
+	current_cpu_datap()->cpu_kernel_cr3 = (addr64_t) kernel_pmap->pm_cr3;
+
+	/* save the value we stuff into created pmaps to share the gdts etc */
+	high_shared_pde = *pmap_pde(kernel_pmap, HIGH_MEM_BASE);
+	/* make sure G bit is on for high shared pde entry */
+	high_shared_pde |= INTEL_PTE_GLOBAL;
+	pmap_store_pte(pmap_pde(kernel_pmap, HIGH_MEM_BASE), high_shared_pde);
+
 	nkpt = NKPT;
+	inuse_ptepages_count += NKPT;
 
 	virtual_avail = (vm_offset_t)VADDR(KPTDI,0) + (vm_offset_t)first_avail;
 	virtual_end = (vm_offset_t)(VM_MAX_KERNEL_ADDRESS);
@@ -765,25 +972,18 @@ pmap_bootstrap(
 	 * mapping of pages.
 	 */
 #define	SYSMAP(c, p, v, n)	\
-	v = (c)va; va += ((n)*INTEL_PGBYTES); p = pte; pte += (n);
+	v = (c)va; va += ((n)*INTEL_PGBYTES); p = pte; pte += (n)
 
 	va = virtual_avail;
-	pte = (pt_entry_t *) pmap_pte(kernel_pmap, va);
-
-	/*
-	 * CMAP1/CMAP2 are used for zeroing and copying pages.
-         * CMAP3 is used for ml_phys_read/write.
-	 */
-	SYSMAP(caddr_t, CM1, CA1, 1)
-	* (pt_entry_t *) CM1 = 0;
-	SYSMAP(caddr_t, CM2, CA2, 1)
-	* (pt_entry_t *) CM2 = 0;
-	SYSMAP(caddr_t, CM3, CA3, 1)
-	* (pt_entry_t *) CM3 = 0;
+	pte = vtopte(va);
 
-	/* used by pmap_pte */
-	SYSMAP(caddr_t, CM4, CA4, 1)
-	  * (pt_entry_t *) CM4 = 0;
+        for (i=0; i<PMAP_NWINDOWS; i++) {
+            SYSMAP(caddr_t,
+		   (current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CMAP),
+                   (current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CADDR),
+		   1);
+            *current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CMAP = 0;
+        }
 
 	/* DMAP user for debugger */
 	SYSMAP(caddr_t, DMAP1, DADDR1, 1);
@@ -797,45 +997,109 @@ pmap_bootstrap(
 	virtual_avail = va;
 
 	wpkernel = 1;
-	if (PE_parse_boot_arg("debug", &boot_arg)) {
-	  if (boot_arg & DB_PRT) wpkernel = 0;
-	  if (boot_arg & DB_NMI) wpkernel = 0;
+	if (PE_parse_boot_arg("wpkernel", &boot_arg)) {
+		if (boot_arg == 0)
+			wpkernel = 0;
 	}
 
-	/* remap kernel text readonly if not debugging or kprintfing */
+	/* Remap kernel text readonly unless the "wpkernel" boot-arg is present
+	 * and set to 0.
+	 */
 	if (wpkernel)
 	{
 		vm_offset_t     myva;
 		pt_entry_t     *ptep;
 
-		for (myva = i386_round_page(VM_MIN_KERNEL_ADDRESS + MP_BOOT + MP_BOOTSTACK); myva < etext; myva += PAGE_SIZE) {
+		for (myva = i386_round_page(MP_BOOT + MP_BOOTSTACK); myva < etext; myva += PAGE_SIZE) {
                         if (myva >= (vm_offset_t)sectHIBB && myva < ((vm_offset_t)sectHIBB + sectSizeHIB))
                                 continue;
-			ptep = pmap_pte(kernel_pmap, myva);
+			ptep = pmap_pte(kernel_pmap, (vm_map_offset_t)myva);
 			if (ptep)
-				*ptep &= ~INTEL_PTE_RW;
+				pmap_store_pte(ptep, *ptep & ~INTEL_PTE_RW);
 		}
-		flush_tlb();
 	}
 
+	/* no matter what,  kernel page zero is not accessible */
+	pte = pmap_pte(kernel_pmap, 0);
+	pmap_store_pte(pte, INTEL_PTE_INVALID);
+
+	/* map lowmem global page into fixed addr 0x2000 */
+	if (0 == (pte = pmap_pte(kernel_pmap,0x2000))) panic("lowmem pte");
+
+	pmap_store_pte(pte, kvtophys((vm_offset_t)&lowGlo)|INTEL_PTE_VALID|INTEL_PTE_REF|INTEL_PTE_MOD|INTEL_PTE_WIRED|INTEL_PTE_RW);
+	flush_tlb();
+
 	simple_lock_init(&kernel_pmap->lock, 0);
 	simple_lock_init(&pv_free_list_lock, 0);
 
-	/* invalidate user virtual addresses */
-	memset((char *)kernel_pmap->dirbase,
-	       0,
-	       (KPTDI) * sizeof(pd_entry_t));
+        pmap_init_high_shared();
+
+	pde_mapped_size = PDE_MAPPED_SIZE;
+
+	if (cpu_64bit) {
+	  pdpt_entry_t *ppdpt   = (pdpt_entry_t *)IdlePDPT;
+	  pdpt_entry_t *ppdpt64 = (pdpt_entry_t *)IdlePDPT64;
+	  pdpt_entry_t *ppml4   = (pdpt_entry_t *)IdlePML4;
+	  int istate = ml_set_interrupts_enabled(FALSE);
+
+	  /*
+	   * Clone a new 64-bit 3rd-level page table directory, IdlePML4,
+	   * with page bits set for the correct IA-32e operation and so that
+	   * the legacy-mode IdlePDPT is retained for slave processor start-up.
+	   * This is necessary due to the incompatible use of page bits between
+	   * 64-bit and legacy modes.
+	   */
+	  kernel_pmap->pm_cr3 = (pmap_paddr_t)((int)IdlePML4); /* setup in start.s for us */
+	  kernel_pmap->pm_pml4 = IdlePML4;
+	  kernel_pmap->pm_pdpt = (pd_entry_t *)
+					((unsigned int)IdlePDPT64 | KERNBASE );
+#define PAGE_BITS INTEL_PTE_VALID|INTEL_PTE_RW|INTEL_PTE_USER|INTEL_PTE_REF
+	  pmap_store_pte(kernel_pmap->pm_pml4,
+		 	 (uint32_t)IdlePDPT64 | PAGE_BITS);
+	  pmap_store_pte((ppdpt64+0), *(ppdpt+0) | PAGE_BITS);
+	  pmap_store_pte((ppdpt64+1), *(ppdpt+1) | PAGE_BITS);
+	  pmap_store_pte((ppdpt64+2), *(ppdpt+2) | PAGE_BITS);
+	  pmap_store_pte((ppdpt64+3), *(ppdpt+3) | PAGE_BITS);
+
+	  /*
+	   * The kernel is also mapped in the uber-sapce at the 4GB starting
+	   * 0xFFFFFF80:00000000. This is the highest entry in the 4th-level.
+	   */
+	  pmap_store_pte((ppml4+KERNEL_UBER_PML4_INDEX), *(ppml4+0));
+
+	  kernel64_cr3 = (addr64_t) kernel_pmap->pm_cr3;
+	  cpu_IA32e_enable(current_cpu_datap());
+	  current_cpu_datap()->cpu_is64bit = TRUE;
+	  /* welcome to a 64 bit world */
+
+	  /* Re-initialize and load descriptors */
+	  cpu_desc_init64(&cpu_data_master, TRUE);
+	  cpu_desc_load64(&cpu_data_master);
+	  fast_syscall_init64();
+
+	  pde_mapped_size = 512*4096 ; 
+
+	  ml_set_interrupts_enabled(istate);
+
+	}
+	kernel_pmap->pm_hold = (vm_offset_t)kernel_pmap->pm_pml4;
 
 	kprintf("Kernel virtual space from 0x%x to 0x%x.\n",
 			VADDR(KPTDI,0), virtual_end);
-#ifdef PAE
-	kprintf("Available physical space from 0x%llx to 0x%llx\n",
-			avail_start, avail_end);
 	printf("PAE enabled\n");
-#else
-	kprintf("Available physical space from 0x%x to 0x%x\n",
+	if (cpu_64bit){
+	  printf("64 bit mode enabled\n");kprintf("64 bit mode enabled\n"); }
+
+	kprintf("Available physical space from 0x%llx to 0x%llx\n",
 			avail_start, avail_end);
-#endif
+
+	/*
+	 * By default for 64-bit users loaded at 4GB, share kernel mapping.
+	 * But this may be overridden by the -no_shared_cr3 boot-arg.
+	 */
+	if (PE_parse_boot_arg("-no_shared_cr3", &no_shared_cr3)) {
+		kprintf("Shared kernel address space disabled\n");
+	}
 }
 
 void
@@ -858,8 +1122,8 @@ pmap_init(void)
 	register long		npages;
 	vm_offset_t		addr;
 	register vm_size_t	s;
-	vm_offset_t vaddr;
-	ppnum_t ppn;
+	vm_map_offset_t		vaddr;
+	ppnum_t			ppn;
 
 	/*
 	 *	Allocate memory for the pv_head_table and its lock bits,
@@ -898,11 +1162,8 @@ pmap_init(void)
 	pmap_zone = zinit(s, 400*s, 4096, "pmap"); /* XXX */
 	s = (vm_size_t) sizeof(struct pv_entry);
 	pv_list_zone = zinit(s, 10000*s, 4096, "pv_list"); /* XXX */
-#ifdef PAE
-	//	s = (vm_size_t) (sizeof(pdpt_entry_t) * NPGPTD);
 	s = 63;
 	pdpt_zone = zinit(s, 400*s, 4096, "pdpt"); /* XXX */
-#endif
 
 	/*
 	 *	Only now, when all of the data structures are allocated,
@@ -915,23 +1176,22 @@ pmap_init(void)
 	vm_first_phys = 0;
 	vm_last_phys = avail_end;
 
-#if GROW_KERNEL_FUNCTION_IMPLEMENTED
 	kptobj = &kptobj_object_store;
 	_vm_object_allocate((vm_object_size_t)NKPDE, kptobj);
 	kernel_pmap->pm_obj = kptobj;
-#endif
 
 	/* create pv entries for kernel pages mapped by low level
 	   startup code.  these have to exist so we can pmap_remove()
 	   e.g. kext pages from the middle of our addr space */
 
-	vaddr = (vm_offset_t)VM_MIN_KERNEL_ADDRESS;
+	vaddr = (vm_map_offset_t)0;
 	for (ppn = 0; ppn < i386_btop(avail_start) ; ppn++ ) {
 	  pv_entry_t	pv_e;
 
 	  pv_e = pai_to_pvh(ppn);
 	  pv_e->va = vaddr;
 	  vaddr += PAGE_SIZE;
+	  kernel_pmap->stats.resident_count++;
 	  pv_e->pmap = kernel_pmap;
 	  pv_e->next = PV_ENTRY_NULL;
 	}
@@ -944,10 +1204,6 @@ pmap_init(void)
 	pmap_cache_list = PMAP_NULL;
 	pmap_cache_count = 0;
 	simple_lock_init(&pmap_cache_lock, 0);
-#ifdef PMAP_QUEUE
-	simple_lock_init(&free_pmap_lock, 0);
-#endif
-
 }
 
 void
@@ -958,7 +1214,7 @@ x86_lowmem_free(void)
 	   the actual pages that are released are determined by which
 	   pages the memory sizing code puts into the region table */
 
-	ml_static_mfree((vm_offset_t) i386_ptob(pmap_memory_regions[0].base)|VM_MIN_KERNEL_ADDRESS,
+	ml_static_mfree((vm_offset_t) i386_ptob(pmap_memory_regions[0].base),
 			(vm_size_t) i386_ptob(pmap_memory_regions[0].end - pmap_memory_regions[0].base));
 }
 
@@ -1008,15 +1264,20 @@ pmap_verify_free(
  */
 pmap_t
 pmap_create(
-	vm_size_t	size)
+	    vm_map_size_t	sz,
+	    boolean_t  		is_64bit)
 {
-  register pmap_t			p;
-#ifdef PMAP_QUEUE
-  register pmap_t pro;
-  spl_t s;
-#endif
-	register int i;
-	register vm_offset_t va;
+        register pmap_t			p;
+	int		i;
+	vm_offset_t	va;
+	vm_size_t	size;
+	pdpt_entry_t    *pdpt;
+	pml4_entry_t    *pml4p;
+	int template;
+	pd_entry_t      *pdp;
+	spl_t s;
+
+	size = (vm_size_t) sz;
 
 	/*
 	 *	A software use-only map doesn't even need a map.
@@ -1029,61 +1290,168 @@ pmap_create(
 	p = (pmap_t) zalloc(pmap_zone);
 	if (PMAP_NULL == p)
 	  panic("pmap_create zalloc");
-	if (KERN_SUCCESS != kmem_alloc_wired(kernel_map, (vm_offset_t *)(&p->dirbase), NBPTD))
-	  panic("pmap_create kmem_alloc_wired");
-#ifdef PAE
-	p->pm_hold = (vm_offset_t)zalloc(pdpt_zone);
-	if ((vm_offset_t)NULL == p->pm_hold) {
-	  panic("pdpt zalloc");
-	}
-	p->pm_pdpt = (pdpt_entry_t *) (( p->pm_hold + 31) & ~31);
-	p->pm_ppdpt = kvtophys((vm_offset_t)p->pm_pdpt);  /* XXX */
-#endif
-	if (NULL == (p->pm_obj = vm_object_allocate((vm_object_size_t)(NPGPTD*NPDEPG))))
-	  panic("pmap_create vm_object_allocate");
-	memcpy(p->dirbase, 
-	       (void *)((unsigned int)IdlePTD | KERNBASE),
-	       NBPTD);
-	va = (vm_offset_t)p->dirbase;
-	p->pdirbase = (pd_entry_t *)(kvtophys(va));
-	simple_lock_init(&p->lock, 0);
-
-	/* setup self referential mapping(s) */
-	for (i = 0; i< NPGPTD; i++ ) {
-	  pmap_paddr_t pa;
-	  pa = (pmap_paddr_t) kvtophys(va + i386_ptob(i));
-	  * (pd_entry_t *) (p->dirbase + PTDPTDI + i) = 
-	    (pa & PG_FRAME) | INTEL_PTE_VALID | INTEL_PTE_RW | INTEL_PTE_REF |
-	    INTEL_PTE_MOD | INTEL_PTE_WIRED ;
-#ifdef PAE
-	  p->pm_pdpt[i] = pa | INTEL_PTE_VALID;
-#endif
-	}
 
-	p->cpus_using = 0;
+	/* init counts now since we'll be bumping some */
+	simple_lock_init(&p->lock, 0);
 	p->stats.resident_count = 0;
 	p->stats.wired_count = 0;
 	p->ref_count = 1;
+	p->nx_enabled = 1;
+	p->pm_64bit = is_64bit;
+	p->pm_kernel_cr3 = FALSE;
+	p->pm_shared = FALSE;
+
+	if (!cpu_64bit) {
+	  /* legacy 32 bit setup */
+	  /* in the legacy case the pdpt layer is hardwired to 4 entries and each
+	   * entry covers 1GB of addr space */
+	  if (KERN_SUCCESS != kmem_alloc_wired(kernel_map, (vm_offset_t *)(&p->dirbase), NBPTD))
+	    panic("pmap_create kmem_alloc_wired");
+	  p->pm_hold = (vm_offset_t)zalloc(pdpt_zone);
+	  if ((vm_offset_t)NULL == p->pm_hold) {
+	    panic("pdpt zalloc");
+	  }
+	  pdpt = (pdpt_entry_t *) (( p->pm_hold + 31) & ~31);
+	  p->pm_cr3 = (pmap_paddr_t)kvtophys((vm_offset_t)pdpt);
+	  if (NULL == (p->pm_obj = vm_object_allocate((vm_object_size_t)(NPGPTD*NPTDPG))))
+	    panic("pmap_create vm_object_allocate");
+
+	  memset((char *)p->dirbase, 0, NBPTD);
+
+	  va = (vm_offset_t)p->dirbase;
+	  p->pdirbase = kvtophys(va);
+
+	  template = cpu_64bit ? INTEL_PTE_VALID|INTEL_PTE_RW|INTEL_PTE_USER|INTEL_PTE_REF : INTEL_PTE_VALID;
+	  for (i = 0; i< NPGPTD; i++, pdpt++) {
+	    pmap_paddr_t pa;
+	    pa = (pmap_paddr_t) kvtophys(va + i386_ptob(i));
+	    pmap_store_pte(pdpt, pa | template);
+	  }
+
+	  /* map the high shared pde */
+	  pmap_store_pte(pmap_pde(p, HIGH_MEM_BASE), high_shared_pde);
 
-#ifdef PMAP_QUEUE
-	/* insert new pmap at head of queue hanging off kernel_pmap */
-	SPLVM(s);
-	simple_lock(&free_pmap_lock);
-	p->pmap_link.next = (queue_t)kernel_pmap->pmap_link.next;
-	kernel_pmap->pmap_link.next = (queue_t)p;
+	} else {
 
-	pro = (pmap_t) p->pmap_link.next;
-	p->pmap_link.prev = (queue_t)pro->pmap_link.prev;
-	pro->pmap_link.prev = (queue_t)p;
+	  /* 64 bit setup  */
 
-	
-	simple_unlock(&free_pmap_lock);
-	SPLX(s);
-#endif
+	  /* alloc the pml4 page in kernel vm */
+	  if (KERN_SUCCESS != kmem_alloc_wired(kernel_map, (vm_offset_t *)(&p->pm_hold), PAGE_SIZE))
+	    panic("pmap_create kmem_alloc_wired pml4");
+
+	  memset((char *)p->pm_hold, 0, PAGE_SIZE);
+	  p->pm_cr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_hold);
+
+	  inuse_ptepages_count++;
+	  p->stats.resident_count++;
+	  p->stats.wired_count++;
+
+	/* allocate the vm_objs to hold the pdpt, pde and pte pages */
+
+	if (NULL == (p->pm_obj_pml4 = vm_object_allocate((vm_object_size_t)(NPML4PGS))))
+	  panic("pmap_create pdpt obj");
+
+	if (NULL == (p->pm_obj_pdpt = vm_object_allocate((vm_object_size_t)(NPDPTPGS))))
+	  panic("pmap_create pdpt obj");
+
+	if (NULL == (p->pm_obj = vm_object_allocate((vm_object_size_t)(NPDEPGS))))
+	  panic("pmap_create pte obj");
+
+	/* uber space points to uber mapped kernel */
+	s = splhigh();
+	pml4p = pmap64_pml4(p, 0ULL);
+	pmap_store_pte((pml4p+KERNEL_UBER_PML4_INDEX),*kernel_pmap->pm_pml4);
+	if (!is_64bit) {
+	  while ((pdp = pmap64_pde(p, (uint64_t)HIGH_MEM_BASE)) == PD_ENTRY_NULL) {
+	    splx(s);
+	    pmap_expand_pdpt(p, (uint64_t)HIGH_MEM_BASE); /* need room for another pde entry */
+	    s = splhigh();
+	  }
+	  pmap_store_pte(pdp, high_shared_pde);
+	}
+
+	splx(s);
+	}
 
 	return(p);
 }
 
+void
+pmap_set_4GB_pagezero(pmap_t p)
+{
+	int		spl;
+	pdpt_entry_t	*user_pdptp;
+	pdpt_entry_t	*kern_pdptp;
+
+	assert(p->pm_64bit);
+
+	/* Kernel-shared cr3 may be disabled by boot arg. */
+	if (no_shared_cr3)
+		return;
+
+	/*
+	 * Set the bottom 4 3rd-level pte's to be the kernel's.
+	 */
+	spl = splhigh();
+	while ((user_pdptp = pmap64_pdpt(p, 0x0)) == PDPT_ENTRY_NULL) {
+		splx(spl);
+		pmap_expand_pml4(p, 0x0);
+		spl = splhigh();
+	}
+	kern_pdptp = kernel_pmap->pm_pdpt;
+	pmap_store_pte(user_pdptp+0, *(kern_pdptp+0));
+	pmap_store_pte(user_pdptp+1, *(kern_pdptp+1));
+	pmap_store_pte(user_pdptp+2, *(kern_pdptp+2));
+	pmap_store_pte(user_pdptp+3, *(kern_pdptp+3));
+
+	p->pm_kernel_cr3 = TRUE;
+
+	splx(spl);
+
+}
+
+void
+pmap_load_kernel_cr3(void)
+{
+	uint32_t	kernel_cr3;
+
+	assert(!ml_get_interrupts_enabled());
+
+	/*
+	 * Reload cr3 with the true kernel cr3.
+	 * Note: kernel's pml4 resides below 4GB physical.
+	 */
+	kernel_cr3 = current_cpu_datap()->cpu_kernel_cr3;
+	set_cr3(kernel_cr3);
+	current_cpu_datap()->cpu_active_cr3 = kernel_cr3;
+	current_cpu_datap()->cpu_task_map = TASK_MAP_32BIT;
+	current_cpu_datap()->cpu_tlb_invalid = FALSE;
+	__asm__ volatile("mfence");
+}
+
+void
+pmap_clear_4GB_pagezero(pmap_t p)
+{
+	int		spl;
+	pdpt_entry_t	*user_pdptp;
+
+	if (!p->pm_kernel_cr3)
+		return;
+
+	spl = splhigh();
+	user_pdptp = pmap64_pdpt(p, 0x0);
+	pmap_store_pte(user_pdptp+0, 0);
+	pmap_store_pte(user_pdptp+1, 0);
+	pmap_store_pte(user_pdptp+2, 0);
+	pmap_store_pte(user_pdptp+3, 0);
+
+	p->pm_kernel_cr3 = FALSE;
+
+	pmap_load_kernel_cr3();
+
+	splx(spl);
+}
+
 /*
  *	Retire the given physical map from service.
  *	Should only be called if the map contains
@@ -1094,47 +1462,29 @@ void
 pmap_destroy(
 	register pmap_t	p)
 {
-	register pt_entry_t	*pdep;
 	register int		c;
 	spl_t                   s;
+#if 0
+	register pt_entry_t	*pdep;
 	register vm_page_t	m;
-#ifdef PMAP_QUEUE
-	register pmap_t        pre,pro;
 #endif
 
 	if (p == PMAP_NULL)
 		return;
-
 	SPLVM(s);
 	simple_lock(&p->lock);
 	c = --p->ref_count;
 	if (c == 0) {
-		register int    my_cpu;
-
-		mp_disable_preemption();
-		my_cpu = cpu_number();
-
 		/* 
 		 * If some cpu is not using the physical pmap pointer that it
 		 * is supposed to be (see set_dirbase), we might be using the
 		 * pmap that is being destroyed! Make sure we are
 		 * physically on the right pmap:
 		 */
-		/* force pmap/cr3 update */
 		PMAP_UPDATE_TLBS(p,
 				 VM_MIN_ADDRESS,
 				 VM_MAX_KERNEL_ADDRESS);
 
-		if (PMAP_REAL(my_cpu) == p) {
-			PMAP_CPU_CLR(p, my_cpu);
-			PMAP_REAL(my_cpu) = kernel_pmap;
-#ifdef PAE
-			set_cr3((unsigned int)kernel_pmap->pm_ppdpt);
-#else
-			set_cr3((unsigned int)kernel_pmap->pdirbase);
-#endif
-		}
-		mp_enable_preemption();
 	}
 	simple_unlock(&p->lock);
 	SPLX(s);
@@ -1143,31 +1493,21 @@ pmap_destroy(
 	    return;	/* still in use */
 	}
 
-#ifdef PMAP_QUEUE
-	/* remove from pmap queue */
-	SPLVM(s);
-	simple_lock(&free_pmap_lock);
-
-	pre = (pmap_t)p->pmap_link.prev;
-	pre->pmap_link.next = (queue_t)p->pmap_link.next;
-	pro = (pmap_t)p->pmap_link.next;
-	pro->pmap_link.prev = (queue_t)p->pmap_link.prev;
-
-	simple_unlock(&free_pmap_lock);
-	SPLX(s);
-#endif
-
 	/*
 	 *	Free the memory maps, then the
 	 *	pmap structure.
 	 */
 
+	if (!cpu_64bit) {
+#if 0
 	pdep = (pt_entry_t *)p->dirbase;
 
 	while (pdep < (pt_entry_t *)&p->dirbase[(UMAXPTDI+1)]) {
-	  int ind;
+	    int ind;
+
 	    if (*pdep & INTEL_PTE_VALID) {
-	      ind = pdep - (pt_entry_t *)&p->dirbase[0];
+	        ind = pdep - (pt_entry_t *)&p->dirbase[0];
+
 		vm_object_lock(p->pm_obj);
 		m = vm_page_lookup(p->pm_obj, (vm_object_offset_t)ind);
 		if (m == VM_PAGE_NULL) {
@@ -1176,25 +1516,49 @@ pmap_destroy(
 		vm_page_lock_queues();
 		vm_page_free(m);
 		inuse_ptepages_count--;
+
 		vm_object_unlock(p->pm_obj);
 		vm_page_unlock_queues();
 
 		/*
 		 *	Clear pdes, this might be headed for the cache.
 		 */
-		*pdep++ = 0;
+		pmap_store_pte(pdep, 0);
+		pdep++;
 	    }
 	    else {
-	      *pdep++ = 0;
+	      pmap_store_pte(pdep, 0);
+	      pdep++;
 	    }
 	
 	}
-
-	vm_object_deallocate(p->pm_obj);
-	kmem_free(kernel_map, (vm_offset_t)p->dirbase, NBPTD);
-#ifdef PAE
-	zfree(pdpt_zone, (void *)p->pm_hold);
+#else
+	inuse_ptepages_count -= p->pm_obj->resident_page_count;
 #endif
+	vm_object_deallocate(p->pm_obj);
+	  kmem_free(kernel_map, (vm_offset_t)p->dirbase, NBPTD);
+	  zfree(pdpt_zone, (void *)p->pm_hold);
+	} else {
+
+	  /* 64 bit */
+
+	  pmap_unmap_sharedpage(p);
+
+	  /* free 64 bit mode structs */
+	  inuse_ptepages_count--;
+	  kmem_free(kernel_map, (vm_offset_t)p->pm_hold, PAGE_SIZE);
+
+	  inuse_ptepages_count -= p->pm_obj_pml4->resident_page_count;
+	  vm_object_deallocate(p->pm_obj_pml4);
+
+	  inuse_ptepages_count -= p->pm_obj_pdpt->resident_page_count;
+	  vm_object_deallocate(p->pm_obj_pdpt);
+
+	  inuse_ptepages_count -= p->pm_obj->resident_page_count;
+	  vm_object_deallocate(p->pm_obj);
+
+	}
+
 	zfree(pmap_zone, p);
 }
 
@@ -1229,11 +1593,10 @@ pmap_reference(
  *	Assumes that the pte-page exists.
  */
 
-/* static */
-void
+static void
 pmap_remove_range(
 	pmap_t			pmap,
-	vm_offset_t		va,
+	vm_map_offset_t		vaddr,
 	pt_entry_t		*spte,
 	pt_entry_t		*epte)
 {
@@ -1242,21 +1605,16 @@ pmap_remove_range(
 	int			pai;
 	pmap_paddr_t		pa;
 
-#if	DEBUG_PTE_PAGE
-	if (pmap != kernel_pmap)
-		ptep_check(get_pte_page(spte));
-#endif	/* DEBUG_PTE_PAGE */
 	num_removed = 0;
 	num_unwired = 0;
 
 	for (cpte = spte; cpte < epte;
-	     cpte++, va += PAGE_SIZE) {
+	     cpte++, vaddr += PAGE_SIZE) {
 
 	    pa = pte_to_pa(*cpte);
 	    if (pa == 0)
 		continue;
 
-	    num_removed++;
 	    if (iswired(*cpte))
 		num_unwired++;
 
@@ -1268,9 +1626,10 @@ pmap_remove_range(
 		 */
 		register pt_entry_t	*lpte = cpte;
 
-		*lpte = 0;
+		pmap_store_pte(lpte, 0);
 		continue;
 	    }
+	    num_removed++;
 
 	    pai = pa_index(pa);
 	    LOCK_PVH(pai);
@@ -1282,9 +1641,9 @@ pmap_remove_range(
 		register pt_entry_t	*lpte;
 
 		lpte = cpte;
-		    pmap_phys_attributes[pai] |=
+		pmap_phys_attributes[pai] |=
 			*lpte & (PHYS_MODIFIED|PHYS_REFERENCED);
-		    *lpte = 0;
+		pmap_store_pte(lpte, 0);
 
 	    }
 
@@ -1299,7 +1658,7 @@ pmap_remove_range(
 		if (pv_h->pmap == PMAP_NULL) {
 		    panic("pmap_remove: null pv_list!");
 		}
-		if (pv_h->va == va && pv_h->pmap == pmap) {
+		if (pv_h->va == vaddr && pv_h->pmap == pmap) {
 		    /*
 		     * Header is the pv_entry.  Copy the next one
 		     * to header and free the next one (we cannot
@@ -1321,7 +1680,7 @@ pmap_remove_range(
 			if ((cur = prev->next) == PV_ENTRY_NULL) {
 			  panic("pmap-remove: mapping not in pv_list!");
 			}
-		    } while (cur->va != va || cur->pmap != pmap);
+		    } while (cur->va != vaddr || cur->pmap != pmap);
 		    prev->next = cur->next;
 		    PV_FREE(cur);
 		}
@@ -1370,39 +1729,31 @@ pmap_remove(
 	spl_t			spl;
 	register pt_entry_t	*pde;
 	register pt_entry_t	*spte, *epte;
-	vm_offset_t		l;
-	vm_offset_t    s, e;
-	vm_offset_t    orig_s;
+	addr64_t		l64;
+	addr64_t    		orig_s64;
 
-	if (map == PMAP_NULL)
+	if (map == PMAP_NULL || s64 == e64)
 		return;
 
 	PMAP_READ_LOCK(map, spl);
 
-	if (value_64bit(s64) || value_64bit(e64)) {
-	  panic("pmap_remove addr overflow");
-	}
-
-	orig_s = s = (vm_offset_t)low32(s64);
-	e = (vm_offset_t)low32(e64);
-
-	pde = pmap_pde(map, s);
-
-	while (s < e) {
-	    l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
-	    if (l > e)
-		l = e;
-	    if (*pde & INTEL_PTE_VALID) {
-	      spte = (pt_entry_t *)pmap_pte(map, (s & ~(PDE_MAPPED_SIZE-1)));
-		spte = &spte[ptenum(s)];
-		epte = &spte[intel_btop(l-s)];
-		pmap_remove_range(map, s, spte, epte);
+	orig_s64 = s64;
+
+	while (s64 < e64) {
+	    l64 = (s64 + pde_mapped_size) & ~(pde_mapped_size-1);
+	    if (l64 > e64)
+		l64 = e64;
+	    pde = pmap_pde(map, s64);
+	    if (pde && (*pde & INTEL_PTE_VALID)) {
+	        spte = (pt_entry_t *)pmap_pte(map, (s64 & ~(pde_mapped_size-1)));
+		spte = &spte[ptenum(s64)];
+		epte = &spte[intel_btop(l64-s64)];
+		pmap_remove_range(map, s64, spte, epte);
 	    }
-	    s = l;
+	    s64 = l64;
 	    pde++;
 	}
-
-	PMAP_UPDATE_TLBS(map, orig_s, e);
+	PMAP_UPDATE_TLBS(map, orig_s64, e64);
 
 	PMAP_READ_UNLOCK(map, spl);
 }
@@ -1426,10 +1777,10 @@ pmap_page_protect(
 	register pmap_t		pmap;
 	spl_t			spl;
 	boolean_t		remove;
-	pmap_paddr_t             phys;
+	pmap_paddr_t            phys;
 
 	assert(pn != vm_page_fictitious_addr);
-	phys = (pmap_paddr_t)i386_ptob(pn);
+
 	if (!valid_page(pn)) {
 	    /*
 	     *	Not a managed page.
@@ -1451,17 +1802,17 @@ pmap_page_protect(
 		remove = TRUE;
 		break;
 	}
+	phys = (pmap_paddr_t)i386_ptob(pn);
+	pai = pa_index(phys);
+	pv_h = pai_to_pvh(pai);
+
 
 	/*
 	 *	Lock the pmap system first, since we will be changing
 	 *	several pmaps.
 	 */
-
 	PMAP_WRITE_LOCK(spl);
 
-	pai = pa_index(phys);
-	pv_h = pai_to_pvh(pai);
-
 	/*
 	 * Walk down PV list, changing or removing all mappings.
 	 * We do not have to lock the pv_list because we have
@@ -1469,93 +1820,95 @@ pmap_page_protect(
 	 */
 	if (pv_h->pmap != PMAP_NULL) {
 
-	    prev = pv_e = pv_h;
-	    do {
-	      register vm_offset_t va;
-		pmap = pv_e->pmap;
-		/*
-		 * Lock the pmap to block pmap_extract and similar routines.
-		 */
-		simple_lock(&pmap->lock);
-
-		{
-
-		    va = pv_e->va;
-		    pte = pmap_pte(pmap, va);
-
-		    /*
-		     * Consistency checks.
-		     */
-		    /* assert(*pte & INTEL_PTE_VALID); XXX */
-		    /* assert(pte_to_phys(*pte) == phys); */
-
-		}
+	        prev = pv_e = pv_h;
 
-		/*
-		 * Remove the mapping if new protection is NONE
-		 * or if write-protecting a kernel mapping.
-		 */
-		if (remove || pmap == kernel_pmap) {
-		    /*
-		     * Remove the mapping, collecting any modify bits.
-		     */
-		    {
-			    pmap_phys_attributes[pai] |=
-				*pte & (PHYS_MODIFIED|PHYS_REFERENCED);
-			    *pte++ = 0;
-			    PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
-		    }
+		do {
+		        register vm_map_offset_t vaddr;
 
-		    assert(pmap->stats.resident_count >= 1);
-		    pmap->stats.resident_count--;
+			pmap = pv_e->pmap;
+			/*
+			 * Lock the pmap to block pmap_extract and similar routines.
+			 */
+			simple_lock(&pmap->lock);
 
-		    /*
-		     * Remove the pv_entry.
-		     */
-		    if (pv_e == pv_h) {
+			vaddr = pv_e->va;
+			pte = pmap_pte(pmap, vaddr);
+			if(0 == pte) {
+			  kprintf("pmap_page_protect pmap 0x%x pn 0x%x vaddr 0x%llx\n",pmap, pn, vaddr);
+			  panic("pmap_page_protect");
+			}
 			/*
-			 * Fix up head later.
+			 * Consistency checks.
 			 */
-			pv_h->pmap = PMAP_NULL;
-		    }
-		    else {
+			/* assert(*pte & INTEL_PTE_VALID); XXX */
+			/* assert(pte_to_phys(*pte) == phys); */
+
+
 			/*
-			 * Delete this entry.
+			 * Remove the mapping if new protection is NONE
+			 * or if write-protecting a kernel mapping.
 			 */
-			prev->next = pv_e->next;
-			PV_FREE(pv_e);
-		    }
-		}
-		else {
-		    /*
-		     * Write-protect.
-		     */
+			if (remove || pmap == kernel_pmap) {
+			        /*
+				 * Remove the mapping, collecting any modify bits.
+				 */
+			        pmap_store_pte(pte, *pte & ~INTEL_PTE_VALID);
 
-			*pte &= ~INTEL_PTE_WRITE;
-			pte++;
-			PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
-		    /*
-		     * Advance prev.
-		     */
-		    prev = pv_e;
-		}
+				PMAP_UPDATE_TLBS(pmap, vaddr, vaddr + PAGE_SIZE);
 
-		simple_unlock(&pmap->lock);
+			        pmap_phys_attributes[pai] |= *pte & (PHYS_MODIFIED|PHYS_REFERENCED);
 
-	    } while ((pv_e = prev->next) != PV_ENTRY_NULL);
+				pmap_store_pte(pte, 0);
 
-	    /*
-	     * If pv_head mapping was removed, fix it up.
-	     */
-	    if (pv_h->pmap == PMAP_NULL) {
-		pv_e = pv_h->next;
-		if (pv_e != PV_ENTRY_NULL) {
-		    *pv_h = *pv_e;
-		    PV_FREE(pv_e);
+
+				//XXX breaks DEBUG build		    assert(pmap->stats.resident_count >= 1);
+				pmap->stats.resident_count--;
+
+				/*
+				 * Remove the pv_entry.
+				 */
+				if (pv_e == pv_h) {
+				        /*
+					 * Fix up head later.
+					 */
+				        pv_h->pmap = PMAP_NULL;
+				}
+				else {
+				        /*
+					 * Delete this entry.
+					 */
+				        prev->next = pv_e->next;
+					PV_FREE(pv_e);
+				}
+			} else {
+			        /*
+				 * Write-protect.
+				 */
+			        pmap_store_pte(pte, *pte & ~INTEL_PTE_WRITE);
+
+				PMAP_UPDATE_TLBS(pmap, vaddr, vaddr + PAGE_SIZE);
+				/*
+				 * Advance prev.
+				 */
+				prev = pv_e;
+			}
+
+			simple_unlock(&pmap->lock);
+
+		} while ((pv_e = prev->next) != PV_ENTRY_NULL);
+
+		/*
+		 * If pv_head mapping was removed, fix it up.
+		 */
+		if (pv_h->pmap == PMAP_NULL) {
+		        pv_e = pv_h->next;
+
+			if (pv_e != PV_ENTRY_NULL) {
+			        *pv_h = *pv_e;
+				PV_FREE(pv_e);
+			}
 		}
-	    }
 	}
-
 	PMAP_WRITE_UNLOCK(spl);
 }
 
@@ -1583,64 +1936,89 @@ unsigned int pmap_disconnect(
 void
 pmap_protect(
 	pmap_t		map,
-	vm_offset_t	s,
-	vm_offset_t	e,
+	vm_map_offset_t	sva,
+	vm_map_offset_t	eva,
 	vm_prot_t	prot)
 {
 	register pt_entry_t	*pde;
 	register pt_entry_t	*spte, *epte;
-	vm_offset_t		l;
+	vm_map_offset_t		lva;
+	vm_map_offset_t		orig_sva;
 	spl_t		spl;
-	vm_offset_t    orig_s = s;
-
+	boolean_t	set_NX;
 
 	if (map == PMAP_NULL)
 		return;
 
-	/*
-	 * Determine the new protection.
-	 */
-	switch (prot) {
-	    case VM_PROT_READ:
-	    case VM_PROT_READ|VM_PROT_EXECUTE:
-		break;
-	    case VM_PROT_READ|VM_PROT_WRITE:
-	    case VM_PROT_ALL:
-		return;	/* nothing to do */
-	    default:
-		pmap_remove(map, (addr64_t)s, (addr64_t)e);
+	if (prot == VM_PROT_NONE) {
+		pmap_remove(map, sva, eva);
 		return;
 	}
 
+	if ( (prot & VM_PROT_EXECUTE) || !nx_enabled || !map->nx_enabled )
+	        set_NX = FALSE;
+	else
+	        set_NX = TRUE;
+
 	SPLVM(spl);
 	simple_lock(&map->lock);
 
-	pde = pmap_pde(map, s);
-	while (s < e) {
-	    l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
-	    if (l > e)
-		l = e;
-	    if (*pde & INTEL_PTE_VALID) {
-	      spte = (pt_entry_t *)pmap_pte(map, (s & ~(PDE_MAPPED_SIZE-1)));
-		spte = &spte[ptenum(s)];
-		epte = &spte[intel_btop(l-s)];
+	orig_sva = sva;
+	while (sva < eva) {
+	    lva = (sva + pde_mapped_size) & ~(pde_mapped_size-1);
+	    if (lva > eva)
+		lva = eva;
+	    pde = pmap_pde(map, sva);
+	    if (pde && (*pde & INTEL_PTE_VALID)) {
+	        spte = (pt_entry_t *)pmap_pte(map, (sva & ~(pde_mapped_size-1)));
+		spte = &spte[ptenum(sva)];
+		epte = &spte[intel_btop(lva-sva)];
 
 		while (spte < epte) {
-		    if (*spte & INTEL_PTE_VALID)
-			*spte &= ~INTEL_PTE_WRITE;
+		    if (*spte & INTEL_PTE_VALID) {
+		      
+		        if (prot & VM_PROT_WRITE)
+			    pmap_store_pte(spte, *spte | INTEL_PTE_WRITE);
+			else
+			    pmap_store_pte(spte, *spte & ~INTEL_PTE_WRITE);
+
+			if (set_NX == TRUE)
+			    pmap_store_pte(spte, *spte | INTEL_PTE_NX);
+			else
+			    pmap_store_pte(spte, *spte & ~INTEL_PTE_NX);
+
+		    }
 		    spte++;
 		}
 	    }
-	    s = l;
+	    sva = lva;
 	    pde++;
 	}
-
-	PMAP_UPDATE_TLBS(map, orig_s, e);
+	PMAP_UPDATE_TLBS(map, orig_sva, eva);
 
 	simple_unlock(&map->lock);
 	SPLX(spl);
 }
 
+/* Map a (possibly) autogenned block */
+void
+pmap_map_block(
+	pmap_t		pmap, 
+	addr64_t	va,
+	ppnum_t 	pa,
+	uint32_t	size,
+	vm_prot_t	prot,
+	int		attr,
+	__unused unsigned int	flags)
+{
+        uint32_t page;
+
+	for (page = 0; page < size; page++) {
+	        pmap_enter(pmap, va, pa, prot, attr, TRUE);
+		va += PAGE_SIZE;
+		pa++;
+	}
+}
 
 
 /*
@@ -1658,7 +2036,7 @@ pmap_protect(
 void
 pmap_enter(
 	register pmap_t		pmap,
-	vm_offset_t		v,
+	vm_map_offset_t		vaddr,
 	ppnum_t                 pn,
 	vm_prot_t		prot,
 	unsigned int 		flags,
@@ -1671,19 +2049,26 @@ pmap_enter(
 	pt_entry_t		template;
 	spl_t			spl;
 	pmap_paddr_t		old_pa;
-	pmap_paddr_t             pa = (pmap_paddr_t)i386_ptob(pn);
+	pmap_paddr_t            pa = (pmap_paddr_t)i386_ptob(pn);
+	boolean_t		need_tlbflush = FALSE;
+	boolean_t		set_NX;
 
-	XPR(0x80000000, "%x/%x: pmap_enter %x/%x/%x\n",
+	XPR(0x80000000, "%x/%x: pmap_enter %x/%qx/%x\n",
 	    current_thread(),
 	    current_thread(), 
-	    pmap, v, pn);
+	    pmap, vaddr, pn);
 
 	assert(pn != vm_page_fictitious_addr);
 	if (pmap_debug)
-		printf("pmap(%x, %x)\n", v, pn);
+		printf("pmap(%qx, %x)\n", vaddr, pn);
 	if (pmap == PMAP_NULL)
 		return;
 
+	if ( (prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled )
+	        set_NX = FALSE;
+	else
+	        set_NX = TRUE;
+	
 	/*
 	 *	Must allocate a new pvlist entry while we're unlocked;
 	 *	zalloc may cause pageout (which will lock the pmap system).
@@ -1701,13 +2086,13 @@ pmap_enter(
 	 *	pages to map one VM page.
 	 */
 
-	while ((pte = pmap_pte(pmap, v)) == PT_ENTRY_NULL) {
+	while ((pte = pmap_pte(pmap, vaddr)) == PT_ENTRY_NULL) {
 		/*
 		 *	Must unlock to expand the pmap.
 		 */
 		PMAP_READ_UNLOCK(pmap, spl);
 
-		pmap_expand(pmap, v);
+		pmap_expand(pmap, vaddr); /* going to grow pde level page(s) */
 
 		PMAP_READ_LOCK(pmap, spl);
 	}
@@ -1723,7 +2108,7 @@ pmap_enter(
 	
 	    template = pa_to_pte(pa) | INTEL_PTE_VALID;
 
-	    if(flags & VM_MEM_NOT_CACHEABLE) {
+	    if(VM_MEM_NOT_CACHEABLE == (flags & (VM_MEM_NOT_CACHEABLE | VM_WIMG_USE_DEFAULT))) {
 		if(!(flags & VM_MEM_GUARDED))
 			template |= INTEL_PTE_PTA;
 		template |= INTEL_PTE_NCACHE;
@@ -1733,6 +2118,10 @@ pmap_enter(
 		template |= INTEL_PTE_USER;
 	    if (prot & VM_PROT_WRITE)
 		template |= INTEL_PTE_WRITE;
+
+	    if (set_NX == TRUE)
+		template |= INTEL_PTE_NX;
+
 	    if (wired) {
 		template |= INTEL_PTE_WIRED;
 		if (!iswired(*pte))
@@ -1747,9 +2136,11 @@ pmap_enter(
 
 		if (*pte & INTEL_PTE_MOD)
 		    template |= INTEL_PTE_MOD;
-		WRITE_PTE(pte, template)
-		  pte++;
 
+		pmap_store_pte(pte, template);
+		pte++;
+
+		need_tlbflush = TRUE;
 	    goto Done;
 	}
 
@@ -1760,6 +2151,7 @@ pmap_enter(
 	 *	   2) Add pvlist entry for new mapping
 	 *	   3) Enter new mapping.
 	 *
+	 *      SHARING FAULTS IS HORRIBLY BROKEN
 	 *	SHARING_FAULTS complicates this slightly in that it cannot
 	 *	replace the mapping, but must remove it (because adding the
 	 *	pvlist entry for the new mapping may remove others), and
@@ -1773,12 +2165,6 @@ pmap_enter(
 
 	if (old_pa != (pmap_paddr_t) 0) {
 
-
-#if	DEBUG_PTE_PAGE
-	    if (pmap != kernel_pmap)
-		ptep_check(get_pte_page(pte));
-#endif	/* DEBUG_PTE_PAGE */
-
 	    /*
 	     *	Don't do anything to pages outside valid memory here.
 	     *	Instead convince the code that enters a new mapping
@@ -1799,8 +2185,8 @@ pmap_enter(
 
 		    pmap_phys_attributes[pai] |=
 			*pte & (PHYS_MODIFIED|PHYS_REFERENCED);
-		    WRITE_PTE(pte, 0)
 
+		pmap_store_pte(pte, 0);
 		/*
 		 *	Remove the mapping from the pvlist for
 		 *	this physical page.
@@ -1812,7 +2198,8 @@ pmap_enter(
 		    if (pv_h->pmap == PMAP_NULL) {
 			panic("pmap_enter: null pv_list!");
 		    }
-		    if (pv_h->va == v && pv_h->pmap == pmap) {
+
+		    if (pv_h->va == vaddr && pv_h->pmap == pmap) {
 			/*
 			 * Header is the pv_entry.  Copy the next one
 			 * to header and free the next one (we cannot
@@ -1834,7 +2221,7 @@ pmap_enter(
 			    if ((cur = prev->next) == PV_ENTRY_NULL) {
 			        panic("pmap_enter: mapping not in pv_list!");
 			    }
-			} while (cur->va != v || cur->pmap != pmap);
+			} while (cur->va != vaddr || cur->pmap != pmap);
 			prev->next = cur->next;
 			pv_e = cur;
 		    }
@@ -1849,13 +2236,13 @@ pmap_enter(
 		 *	one).  Do removal part of accounting.
 		 */
 		old_pa = (pmap_paddr_t) 0;
-		assert(pmap->stats.resident_count >= 1);
-		pmap->stats.resident_count--;
+
 		if (iswired(*pte)) {
 		    assert(pmap->stats.wired_count >= 1);
 		    pmap->stats.wired_count--;
 		}
 	    }
+	    need_tlbflush = TRUE;
         
 	}
 
@@ -1869,7 +2256,7 @@ pmap_enter(
 	    pai = pa_index(pa);
 
 
-#if SHARING_FAULTS
+#if SHARING_FAULTS /* this is horribly broken , do not enable */
 RetryPvList:
 	    /*
 	     * We can return here from the sharing fault code below
@@ -1884,7 +2271,7 @@ RetryPvList:
 		/*
 		 *	No mappings yet
 		 */
-		pv_h->va = v;
+		pv_h->va = vaddr;
 		pv_h->pmap = pmap;
 		pv_h->next = PV_ENTRY_NULL;
 	    }
@@ -1897,13 +2284,13 @@ RetryPvList:
 		     */
 		    pv_entry_t	e = pv_h;
 		    while (e != PV_ENTRY_NULL) {
-			if (e->pmap == pmap && e->va == v)
+			if (e->pmap == pmap && e->va == vaddr)
                             panic("pmap_enter: already in pv_list");
 			e = e->next;
 		    }
 		}
 #endif	/* DEBUG */
-#if SHARING_FAULTS
+#if SHARING_FAULTS /* broken, do not enable */
                 {
                     /*
                      * do sharing faults.
@@ -1929,6 +2316,7 @@ RetryPvList:
 			     */
                              pmap_remove_range(pmap, e->va, opte,
                                                       opte + 1);
+
 			     PMAP_UPDATE_TLBS(pmap, e->va, e->va + PAGE_SIZE);
 
 			     /*
@@ -1982,7 +2370,7 @@ RetryPvList:
                             if (logit) {
                                 pma = &pmap_aliasbuf[pmap_alias_index];
                                 pma->pmap = pmap;
-                                pma->va = v;
+                                pma->va = vaddr;
                                 pma->rpc = rpc;
                                 pma->cookie = PMAP_ALIAS_COOKIE;
                                 if (++pmap_alias_index >= PMAP_ALIAS_MAX)
@@ -2002,7 +2390,7 @@ RetryPvList:
 		      panic("pmap no pv_e's");
 		    }
 		}
-		pv_e->va = v;
+		pv_e->va = vaddr;
 		pv_e->pmap = pmap;
 		pv_e->next = pv_h->next;
 		pv_h->next = pv_e;
@@ -2012,13 +2400,18 @@ RetryPvList:
 		pv_e = PV_ENTRY_NULL;
 	    }
 	    UNLOCK_PVH(pai);
+
+	    /*
+	     * only count the mapping
+	     * for 'managed memory'
+	     */
+	    pmap->stats.resident_count++;
 	}
 
 	/*
-	 * Step 3) Enter and count the mapping.
+	 * Step 3) Enter the mapping.
 	 */
 
-	pmap->stats.resident_count++;
 
 	/*
 	 *	Build a template to speed up entering -
@@ -2036,18 +2429,22 @@ RetryPvList:
 		template |= INTEL_PTE_USER;
 	if (prot & VM_PROT_WRITE)
 		template |= INTEL_PTE_WRITE;
+
+	if (set_NX == TRUE)
+		template |= INTEL_PTE_NX;
+
 	if (wired) {
 		template |= INTEL_PTE_WIRED;
 		pmap->stats.wired_count++;
 	}
-
-	 WRITE_PTE(pte, template)
+	pmap_store_pte(pte, template);
 
 Done:
-	 PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
+	if (need_tlbflush == TRUE)
+	        PMAP_UPDATE_TLBS(pmap, vaddr, vaddr + PAGE_SIZE);
 
 	if (pv_e != PV_ENTRY_NULL) {
-	    PV_FREE(pv_e);
+	        PV_FREE(pv_e);
 	}
 
 	PMAP_READ_UNLOCK(pmap, spl);
@@ -2063,7 +2460,7 @@ Done:
 void
 pmap_change_wiring(
 	register pmap_t	map,
-	vm_offset_t	v,
+	vm_map_offset_t	vaddr,
 	boolean_t	wired)
 {
 	register pt_entry_t	*pte;
@@ -2076,7 +2473,7 @@ pmap_change_wiring(
 	 */
 	PMAP_READ_LOCK(map, spl);
 
-	if ((pte = pmap_pte(map, v)) == PT_ENTRY_NULL)
+	if ((pte = pmap_pte(map, vaddr)) == PT_ENTRY_NULL)
 		panic("pmap_change_wiring: pte missing");
 
 	if (wired && !iswired(*pte)) {
@@ -2084,7 +2481,8 @@ pmap_change_wiring(
 	     *	wiring down mapping
 	     */
 	    map->stats.wired_count++;
-	    *pte++ |= INTEL_PTE_WIRED;
+	    pmap_store_pte(pte, *pte | INTEL_PTE_WIRED);
+	    pte++;
 	}
 	else if (!wired && iswired(*pte)) {
 	    /*
@@ -2092,7 +2490,8 @@ pmap_change_wiring(
 	     */
 	    assert(map->stats.wired_count >= 1);
 	    map->stats.wired_count--;
-	    *pte++ &= ~INTEL_PTE_WIRED;
+	    pmap_store_pte(pte, *pte & ~INTEL_PTE_WIRED);
+	    pte++;
 	}
 
 	PMAP_READ_UNLOCK(map, spl);
@@ -2107,18 +2506,18 @@ ppnum_t
 pmap_find_phys(pmap_t pmap, addr64_t va)
 {
 	pt_entry_t     *ptp;
-	vm_offset_t     a32;
 	ppnum_t         ppn;
 
-	if (value_64bit(va))
-		panic("pmap_find_phys 64 bit value");
-	a32 = (vm_offset_t) low32(va);
-	ptp = pmap_pte(pmap, a32);
+	mp_disable_preemption();
+
+	ptp = pmap_pte(pmap, va);
 	if (PT_ENTRY_NULL == ptp) {
 		ppn = 0;
 	} else {
 		ppn = (ppnum_t) i386_btop(pte_to_pa(*ptp));
 	}
+	mp_enable_preemption();
+
 	return ppn;
 }
 
@@ -2135,54 +2534,40 @@ pmap_find_phys(pmap_t pmap, addr64_t va)
 vm_offset_t
 pmap_extract(
 	register pmap_t	pmap,
-	vm_offset_t	va)
+	vm_map_offset_t	vaddr)
 {
-  ppnum_t ppn;
-  vm_offset_t vaddr;
+        ppnum_t ppn;
+	vm_offset_t paddr;
 
-  vaddr = (vm_offset_t)0;
-  ppn = pmap_find_phys(pmap, (addr64_t)va);
-  if (ppn) {
-    vaddr = ((vm_offset_t)i386_ptob(ppn)) | (va & INTEL_OFFMASK);
-  }
-  return (vaddr);
+	paddr = (vm_offset_t)0;
+	ppn = pmap_find_phys(pmap, vaddr);
+	if (ppn) {
+	        paddr = ((vm_offset_t)i386_ptob(ppn)) | (vaddr & INTEL_OFFMASK);
+	}
+	return (paddr);
 }
 
-
-/*
- *	Routine:	pmap_expand
- *
- *	Expands a pmap to be able to map the specified virtual address.
- *
- *	Allocates new virtual memory for the P0 or P1 portion of the
- *	pmap, then re-maps the physical pages that were in the old
- *	pmap to be in the new pmap.
- *
- *	Must be called with the pmap system and the pmap unlocked,
- *	since these must be unlocked to use vm_allocate or vm_deallocate.
- *	Thus it must be called in a loop that checks whether the map
- *	has been expanded enough.
- *	(We won't loop forever, since page tables aren't shrunk.)
- */
 void
-pmap_expand(
-	register pmap_t		map,
-	register vm_offset_t	v)
+pmap_expand_pml4(
+		 pmap_t map,
+		 vm_map_offset_t vaddr)
 {
-	pt_entry_t		*pdp;
 	register vm_page_t	m;
 	register pmap_paddr_t	pa;
-	register int		i;
+	uint64_t                i;
 	spl_t			spl;
 	ppnum_t                 pn;
+	pml4_entry_t            *pml4p;
 
-	if (map == kernel_pmap) {
-	  pmap_growkernel(v);
-	  return;
-	}
+	if (kernel_pmap == map) panic("expand kernel pml4");
+
+	spl = splhigh();
+	  pml4p = pmap64_pml4(map, vaddr);
+	  splx(spl);
+	  if (PML4_ENTRY_NULL == pml4p) panic("pmap_expand_pml4 no pml4p");
 
 	/*
-	 *	Allocate a VM page for the level 2 page table entries.
+	 *	Allocate a VM page for the pml4 page
 	 */
 	while ((m = vm_page_grab()) == VM_PAGE_NULL)
 		VM_PAGE_WAIT();
@@ -2193,14 +2578,25 @@ pmap_expand(
 	 */
 	pn = m->phys_page;
 	pa = i386_ptob(pn);
-	i = pdenum(map, v);
-	vm_object_lock(map->pm_obj);
-	vm_page_insert(m, map->pm_obj, (vm_object_offset_t)i);
+	i = pml4idx(map, vaddr);
+
+	vm_object_lock(map->pm_obj_pml4);
+#if 0 /* DEBUG */
+	if (0 != vm_page_lookup(map->pm_obj_pml4, (vm_object_offset_t)i)) {
+	  kprintf("pmap_expand_pml4: obj_pml4 not empty, pmap 0x%x pm_obj_pml4 0x%x vaddr 0x%llx i 0x%llx\n",
+		  map, map->pm_obj_pml4, vaddr, i);
+	}
+#endif
+	vm_page_insert(m, map->pm_obj_pml4, (vm_object_offset_t)i);
+
 	vm_page_lock_queues();
 	vm_page_wire(m);
-	inuse_ptepages_count++;
-	vm_object_unlock(map->pm_obj);
+
 	vm_page_unlock_queues();
+	vm_object_unlock(map->pm_obj_pml4);
+	inuse_ptepages_count++;
+	map->stats.resident_count++;
+	map->stats.wired_count++;
 
 	/*
 	 *	Zero the page.
@@ -2211,14 +2607,17 @@ pmap_expand(
 	/*
 	 *	See if someone else expanded us first
 	 */
-	if (pmap_pte(map, v) != PT_ENTRY_NULL) {
+	if (pmap64_pdpt(map, vaddr) != PDPT_ENTRY_NULL) {
 		PMAP_READ_UNLOCK(map, spl);
-		vm_object_lock(map->pm_obj);
+		vm_object_lock(map->pm_obj_pml4);
 		vm_page_lock_queues();
 		vm_page_free(m);
 		inuse_ptepages_count--;
+		map->stats.resident_count--;
+		map->stats.wired_count--;
+
 		vm_page_unlock_queues();
-		vm_object_unlock(map->pm_obj);
+		vm_object_unlock(map->pm_obj_pml4);
 		return;
 	}
 
@@ -2228,119 +2627,316 @@ pmap_expand(
 	 *	set several page directory entries.
 	 */
 
-	pdp = &map->dirbase[pdenum(map, v)];
-	    *pdp = pa_to_pte(pa)
-		| INTEL_PTE_VALID
-		| INTEL_PTE_USER
-		| INTEL_PTE_WRITE;
+	pml4p = pmap64_pml4(map, vaddr); /* refetch under lock */
 
-	PMAP_READ_UNLOCK(map, spl);
-	return;
-}
+	pmap_store_pte(pml4p, pa_to_pte(pa)
+				| INTEL_PTE_VALID
+				| INTEL_PTE_USER
+				| INTEL_PTE_WRITE);
 
-/*
- *	Copy the range specified by src_addr/len
- *	from the source map to the range dst_addr/len
- *	in the destination map.
- *
- *	This routine is only advisory and need not do anything.
- */
-#if	0
-void
-pmap_copy(
-	pmap_t		dst_pmap,
-	pmap_t		src_pmap,
-	vm_offset_t	dst_addr,
-	vm_size_t	len,
-	vm_offset_t	src_addr)
-{
-#ifdef	lint
-	dst_pmap++; src_pmap++; dst_addr++; len++; src_addr++;
-#endif	/* lint */
-}
-#endif/* 	0 */
+	PMAP_READ_UNLOCK(map, spl);
 
-/*
- * pmap_sync_page_data_phys(ppnum_t pa)
- * 
- * Invalidates all of the instruction cache on a physical page and
- * pushes any dirty data from the data cache for the same physical page
- * Not required in i386.
- */
-void
-pmap_sync_page_data_phys(__unused ppnum_t pa)
-{
 	return;
+
 }
 
-/*
- * pmap_sync_page_attributes_phys(ppnum_t pa)
- * 
- * Write back and invalidate all cachelines on a physical page.
- */
 void
-pmap_sync_page_attributes_phys(ppnum_t pa)
+pmap_expand_pdpt(
+		 pmap_t map,
+		 vm_map_offset_t vaddr)
 {
-	cache_flush_page_phys(pa);
-}
+	register vm_page_t	m;
+	register pmap_paddr_t	pa;
+	uint64_t                i;
+	spl_t			spl;
+	ppnum_t                 pn;
+	pdpt_entry_t            *pdptp;
 
-int	collect_ref;
-int	collect_unref;
+	if (kernel_pmap == map) panic("expand kernel pdpt");
 
-/*
- *	Routine:	pmap_collect
- *	Function:
- *		Garbage collects the physical map system for
- *		pages which are no longer used.
- *		Success need not be guaranteed -- that is, there
- *		may well be pages which are not referenced, but
- *		others may be collected.
- *	Usage:
- *		Called by the pageout daemon when pages are scarce.
- */
-void
-pmap_collect(
-	pmap_t 		p)
-{
-	register pt_entry_t	*pdp, *ptp;
-	pt_entry_t		*eptp;
-	int			wired;
-	spl_t                   spl;
+	spl = splhigh();
+	  while ((pdptp = pmap64_pdpt(map, vaddr)) == PDPT_ENTRY_NULL) {
+	    splx(spl);
+	    pmap_expand_pml4(map, vaddr); /* need room for another pdpt entry */
+	    spl = splhigh();
+	  }
+	  splx(spl);
 
-	if (p == PMAP_NULL)
-		return;
 
-	if (p == kernel_pmap)
-		return;
+	/*
+	 *	Allocate a VM page for the pdpt page
+	 */
+	while ((m = vm_page_grab()) == VM_PAGE_NULL)
+		VM_PAGE_WAIT();
 
 	/*
-	 *	Garbage collect map.
+	 *	put the page into the pmap's obj list so it
+	 *	can be found later.
 	 */
-	PMAP_READ_LOCK(p, spl);
+	pn = m->phys_page;
+	pa = i386_ptob(pn);
+	i = pdptidx(map, vaddr);
 
-	for (pdp = (pt_entry_t *)p->dirbase;
-	     pdp < (pt_entry_t *)&p->dirbase[(UMAXPTDI+1)];
-	     pdp++)
-	{
-	   if (*pdp & INTEL_PTE_VALID) {
-	      if(*pdp & INTEL_PTE_REF) {
-		*pdp &= ~INTEL_PTE_REF;
-		collect_ref++;
-	      } else {
-		collect_unref++;
-		ptp = pmap_pte(p, pdetova(pdp - (pt_entry_t *)p->dirbase));
-		eptp = ptp + NPTEPG;
+	vm_object_lock(map->pm_obj_pdpt);
+#if 0 /* DEBUG */
+	if (0 != vm_page_lookup(map->pm_obj_pdpt, (vm_object_offset_t)i)) {
+	  kprintf("pmap_expand_pdpt: obj_pdpt not empty, pmap 0x%x pm_obj_pdpt 0x%x vaddr 0x%llx i 0x%llx\n",
+		  map, map->pm_obj_pdpt, vaddr, i);
+	}
+#endif
+	vm_page_insert(m, map->pm_obj_pdpt, (vm_object_offset_t)i);
 
-		/*
-		 * If the pte page has any wired mappings, we cannot
-		 * free it.
-		 */
-		wired = 0;
-		{
-		    register pt_entry_t *ptep;
-		    for (ptep = ptp; ptep < eptp; ptep++) {
-			if (iswired(*ptep)) {
-			    wired = 1;
+	vm_page_lock_queues();
+	vm_page_wire(m);
+
+	vm_page_unlock_queues();
+	vm_object_unlock(map->pm_obj_pdpt);
+	inuse_ptepages_count++;
+	map->stats.resident_count++;
+	map->stats.wired_count++;
+
+	/*
+	 *	Zero the page.
+	 */
+	pmap_zero_page(pn);
+
+	PMAP_READ_LOCK(map, spl);
+	/*
+	 *	See if someone else expanded us first
+	 */
+	if (pmap64_pde(map, vaddr) != PD_ENTRY_NULL) {
+		PMAP_READ_UNLOCK(map, spl);
+		vm_object_lock(map->pm_obj_pdpt);
+		vm_page_lock_queues();
+		vm_page_free(m);
+		inuse_ptepages_count--;
+		map->stats.resident_count--;
+		map->stats.wired_count--;
+
+		vm_page_unlock_queues();
+		vm_object_unlock(map->pm_obj_pdpt);
+		return;
+	}
+
+	/*
+	 *	Set the page directory entry for this page table.
+	 *	If we have allocated more than one hardware page,
+	 *	set several page directory entries.
+	 */
+
+	pdptp = pmap64_pdpt(map, vaddr); /* refetch under lock */
+
+	pmap_store_pte(pdptp, pa_to_pte(pa)
+				| INTEL_PTE_VALID
+				| INTEL_PTE_USER
+				| INTEL_PTE_WRITE);
+
+	PMAP_READ_UNLOCK(map, spl);
+
+	return;
+
+}
+
+
+
+/*
+ *	Routine:	pmap_expand
+ *
+ *	Expands a pmap to be able to map the specified virtual address.
+ *
+ *	Allocates new virtual memory for the P0 or P1 portion of the
+ *	pmap, then re-maps the physical pages that were in the old
+ *	pmap to be in the new pmap.
+ *
+ *	Must be called with the pmap system and the pmap unlocked,
+ *	since these must be unlocked to use vm_allocate or vm_deallocate.
+ *	Thus it must be called in a loop that checks whether the map
+ *	has been expanded enough.
+ *	(We won't loop forever, since page tables aren't shrunk.)
+ */
+void
+pmap_expand(
+	pmap_t		map,
+	vm_map_offset_t	vaddr)
+{
+	pt_entry_t		*pdp;
+	register vm_page_t	m;
+	register pmap_paddr_t	pa;
+	uint64_t                 i;
+	spl_t			spl;
+	ppnum_t                 pn;
+
+	/*
+	 * if not the kernel map (while we are still compat kernel mode)
+	 * and we are 64 bit, propagate expand upwards
+	 */
+
+	if (cpu_64bit && (map != kernel_pmap)) {
+	  spl = splhigh();
+	  while ((pdp = pmap64_pde(map, vaddr)) == PD_ENTRY_NULL) {
+	    splx(spl);
+	    pmap_expand_pdpt(map, vaddr); /* need room for another pde entry */
+	    spl = splhigh();
+	  }
+	  splx(spl);
+	} else {
+	  pdp = pmap_pde(map, vaddr);
+	}
+
+
+	/*
+	 *	Allocate a VM page for the pde entries.
+	 */
+	while ((m = vm_page_grab()) == VM_PAGE_NULL)
+		VM_PAGE_WAIT();
+
+	/*
+	 *	put the page into the pmap's obj list so it
+	 *	can be found later.
+	 */
+	pn = m->phys_page;
+	pa = i386_ptob(pn);
+	i = pdeidx(map, vaddr);
+
+	vm_object_lock(map->pm_obj);
+#if 0 /* DEBUG */
+	if (0 != vm_page_lookup(map->pm_obj, (vm_object_offset_t)i)) {
+	  kprintf("pmap_expand: obj not empty, pmap 0x%x pm_obj 0x%x vaddr 0x%llx i 0x%llx\n",
+		  map, map->pm_obj, vaddr, i);
+	}
+#endif
+	vm_page_insert(m, map->pm_obj, (vm_object_offset_t)i);
+
+	vm_page_lock_queues();
+	vm_page_wire(m);
+	inuse_ptepages_count++;
+
+	vm_page_unlock_queues();
+	vm_object_unlock(map->pm_obj);
+
+	/*
+	 *	Zero the page.
+	 */
+	pmap_zero_page(pn);
+
+	PMAP_READ_LOCK(map, spl);
+	/*
+	 *	See if someone else expanded us first
+	 */
+	if (pmap_pte(map, vaddr) != PT_ENTRY_NULL) {
+		PMAP_READ_UNLOCK(map, spl);
+		vm_object_lock(map->pm_obj);
+
+		vm_page_lock_queues();
+		vm_page_free(m);
+		inuse_ptepages_count--;
+
+		vm_page_unlock_queues();
+		vm_object_unlock(map->pm_obj);
+		return;
+	}
+
+	pdp = pmap_pde(map, vaddr); /* refetch while locked */
+
+	/*
+	 *	Set the page directory entry for this page table.
+	 *	If we have allocated more than one hardware page,
+	 *	set several page directory entries.
+	 */
+
+	pmap_store_pte(pdp, pa_to_pte(pa)
+				| INTEL_PTE_VALID
+				| INTEL_PTE_USER
+				| INTEL_PTE_WRITE);
+	    
+
+	PMAP_READ_UNLOCK(map, spl);
+
+	return;
+}
+
+
+/*
+ * pmap_sync_page_data_phys(ppnum_t pa)
+ * 
+ * Invalidates all of the instruction cache on a physical page and
+ * pushes any dirty data from the data cache for the same physical page
+ * Not required in i386.
+ */
+void
+pmap_sync_page_data_phys(__unused ppnum_t pa)
+{
+	return;
+}
+
+/*
+ * pmap_sync_page_attributes_phys(ppnum_t pa)
+ * 
+ * Write back and invalidate all cachelines on a physical page.
+ */
+void
+pmap_sync_page_attributes_phys(ppnum_t pa)
+{
+	cache_flush_page_phys(pa);
+}
+
+int	collect_ref;
+int	collect_unref;
+
+/*
+ *	Routine:	pmap_collect
+ *	Function:
+ *		Garbage collects the physical map system for
+ *		pages which are no longer used.
+ *		Success need not be guaranteed -- that is, there
+ *		may well be pages which are not referenced, but
+ *		others may be collected.
+ *	Usage:
+ *		Called by the pageout daemon when pages are scarce.
+ */
+void
+pmap_collect(
+	pmap_t 		p)
+{
+	register pt_entry_t	*pdp, *ptp;
+	pt_entry_t		*eptp;
+	int			wired;
+	spl_t                   spl;
+
+	if (p == PMAP_NULL)
+		return;
+
+	if (p == kernel_pmap)
+		return;
+
+	/*
+	 *	Garbage collect map.
+	 */
+	PMAP_READ_LOCK(p, spl);
+
+	for (pdp = (pt_entry_t *)p->dirbase;
+	     pdp < (pt_entry_t *)&p->dirbase[(UMAXPTDI+1)];
+	     pdp++)
+	{
+	   if (*pdp & INTEL_PTE_VALID) {
+	      if(*pdp & INTEL_PTE_REF) {
+		pmap_store_pte(pdp, *pdp & ~INTEL_PTE_REF);
+		collect_ref++;
+	      } else {
+		collect_unref++;
+		ptp = pmap_pte(p, pdetova(pdp - (pt_entry_t *)p->dirbase));
+		eptp = ptp + NPTEPG;
+
+		/*
+		 * If the pte page has any wired mappings, we cannot
+		 * free it.
+		 */
+		wired = 0;
+		{
+		    register pt_entry_t *ptep;
+		    for (ptep = ptp; ptep < eptp; ptep++) {
+			if (iswired(*ptep)) {
+			    wired = 1;
 			    break;
 			}
 		    }
@@ -2357,7 +2953,7 @@ pmap_collect(
 		    /*
 		     * Invalidate the page directory pointer.
 		     */
-		    *pdp = 0x0;
+		    pmap_store_pte(pdp, 0x0);
 		 
 		    PMAP_READ_UNLOCK(p, spl);
 
@@ -2384,32 +2980,21 @@ pmap_collect(
 	   }
 	}
 	PMAP_UPDATE_TLBS(p, VM_MIN_ADDRESS, VM_MAX_ADDRESS);
+
 	PMAP_READ_UNLOCK(p, spl);
 	return;
 
 }
 
-/*
- *	Routine:	pmap_kernel
- *	Function:
- *		Returns the physical map handle for the kernel.
- */
-#if	0
-pmap_t
-pmap_kernel(void)
-{
-    	return (kernel_pmap);
-}
-#endif/* 	0 */
 
 void
 pmap_copy_page(src, dst)
 	ppnum_t src;
 	ppnum_t dst;
 {
-  bcopy_phys((addr64_t)i386_ptob(src),
-	     (addr64_t)i386_ptob(dst),
-	     PAGE_SIZE);
+        bcopy_phys((addr64_t)i386_ptob(src),
+		   (addr64_t)i386_ptob(dst),
+		   PAGE_SIZE);
 }
 
 
@@ -2430,8 +3015,8 @@ pmap_copy_page(src, dst)
 void
 pmap_pageable(
 	__unused pmap_t		pmap,
-	__unused vm_offset_t	start_addr,
-	__unused vm_offset_t	end_addr,
+	__unused vm_map_offset_t	start_addr,
+	__unused vm_map_offset_t	end_addr,
 	__unused boolean_t	pageable)
 {
 #ifdef	lint
@@ -2491,7 +3076,7 @@ phys_attribute_clear(
 		simple_lock(&pmap->lock);
 
 		{
-		    register vm_offset_t va;
+		    register vm_map_offset_t va;
 
 		    va = pv_e->va;
 		    pte = pmap_pte(pmap, va);
@@ -2508,7 +3093,8 @@ phys_attribute_clear(
 		 * Clear modify or reference bits.
 		 */
 
-			*pte++ &= ~bits;
+			pmap_store_pte(pte, *pte & ~bits);
+			pte++;
 			PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
 		}
 		simple_unlock(&pmap->lock);
@@ -2545,15 +3131,24 @@ phys_attribute_test(
 	    return (FALSE);
 	}
 
+	phys = i386_ptob(pn);
+	pai = pa_index(phys);
+	/*
+	 * super fast check...  if bits already collected
+	 * no need to take any locks...
+	 * if not set, we need to recheck after taking
+	 * the lock in case they got pulled in while
+	 * we were waiting for the lock
+	 */
+	if (pmap_phys_attributes[pai] & bits)
+	    return (TRUE);
+	pv_h = pai_to_pvh(pai);
+
 	/*
 	 *	Lock the pmap system first, since we will be checking
 	 *	several pmaps.
 	 */
-
 	PMAP_WRITE_LOCK(spl);
-	phys = i386_ptob(pn);
-	pai = pa_index(phys);
-	pv_h = pai_to_pvh(pai);
 
 	if (pmap_phys_attributes[pai] & bits) {
 	    PMAP_WRITE_UNLOCK(spl);
@@ -2578,7 +3173,7 @@ phys_attribute_test(
 		simple_lock(&pmap->lock);
 
 		{
-		    register vm_offset_t va;
+		    register vm_map_offset_t va;
 
 		    va = pv_e->va;
 		    pte = pmap_pte(pmap, va);
@@ -2744,45 +3339,49 @@ pmap_clear_refmod(ppnum_t pa, unsigned int mask)
 void
 pmap_modify_pages(
 	pmap_t		map,
-	vm_offset_t	s,
-	vm_offset_t	e)
+	vm_map_offset_t	sva,
+	vm_map_offset_t	eva)
 {
 	spl_t			spl;
 	register pt_entry_t	*pde;
 	register pt_entry_t	*spte, *epte;
-	vm_offset_t		l;
-	vm_offset_t             orig_s = s;
+	vm_map_offset_t		lva;
+	vm_map_offset_t		orig_sva;
 
 	if (map == PMAP_NULL)
 		return;
 
 	PMAP_READ_LOCK(map, spl);
 
-	pde = pmap_pde(map, s);
-	while (s && s < e) {
-	    l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
-	    if (l > e)
-		l = e;
-	    if (*pde & INTEL_PTE_VALID) {
-	      spte = (pt_entry_t *)pmap_pte(map, (s & ~(PDE_MAPPED_SIZE-1)));
-		if (l) {
-		   spte = &spte[ptenum(s)];
-		   epte = &spte[intel_btop(l-s)];
+	orig_sva = sva;
+	while (sva && sva < eva) {
+	    lva = (sva + pde_mapped_size) & ~(pde_mapped_size-1);
+	    if (lva > eva)
+		lva = eva;
+	    pde = pmap_pde(map, sva);
+	    if (pde && (*pde & INTEL_PTE_VALID)) {
+	      spte = (pt_entry_t *)pmap_pte(map, (sva & ~(pde_mapped_size-1)));
+		if (lva) {
+		   spte = &spte[ptenum(sva)];
+		   epte = &spte[intel_btop(lva-sva)];
 	        } else {
-		   epte = &spte[intel_btop(PDE_MAPPED_SIZE)];
-		   spte = &spte[ptenum(s)];
+		   epte = &spte[intel_btop(pde_mapped_size)];
+		   spte = &spte[ptenum(sva)];
 	        }
 		while (spte < epte) {
 		    if (*spte & INTEL_PTE_VALID) {
-			*spte |= (INTEL_PTE_MOD | INTEL_PTE_WRITE);
+			pmap_store_pte(spte, *spte
+						| INTEL_PTE_MOD
+						| INTEL_PTE_WRITE);
 		    }
 		    spte++;
 		}
 	    }
-	    s = l;
+	    sva = lva;
 	    pde++;
 	}
-	PMAP_UPDATE_TLBS(map, orig_s, e);
+	PMAP_UPDATE_TLBS(map, orig_sva, eva);
+
 	PMAP_READ_UNLOCK(map, spl);
 }
 
@@ -2802,225 +3401,21 @@ flush_dcache(__unused vm_offset_t	addr,
 	return;
 }
 
-/*
-*	    TLB Coherence Code (TLB "shootdown" code)
-* 
-* Threads that belong to the same task share the same address space and
-* hence share a pmap.  However, they  may run on distinct cpus and thus
-* have distinct TLBs that cache page table entries. In order to guarantee
-* the TLBs are consistent, whenever a pmap is changed, all threads that
-* are active in that pmap must have their TLB updated. To keep track of
-* this information, the set of cpus that are currently using a pmap is
-* maintained within each pmap structure (cpus_using). Pmap_activate() and
-* pmap_deactivate add and remove, respectively, a cpu from this set.
-* Since the TLBs are not addressable over the bus, each processor must
-* flush its own TLB; a processor that needs to invalidate another TLB
-* needs to interrupt the processor that owns that TLB to signal the
-* update.
-* 
-* Whenever a pmap is updated, the lock on that pmap is locked, and all
-* cpus using the pmap are signaled to invalidate. All threads that need
-* to activate a pmap must wait for the lock to clear to await any updates
-* in progress before using the pmap. They must ACQUIRE the lock to add
-* their cpu to the cpus_using set. An implicit assumption made
-* throughout the TLB code is that all kernel code that runs at or higher
-* than splvm blocks out update interrupts, and that such code does not
-* touch pageable pages.
-* 
-* A shootdown interrupt serves another function besides signaling a
-* processor to invalidate. The interrupt routine (pmap_update_interrupt)
-* waits for the both the pmap lock (and the kernel pmap lock) to clear,
-* preventing user code from making implicit pmap updates while the
-* sending processor is performing its update. (This could happen via a
-* user data write reference that turns on the modify bit in the page
-* table). It must wait for any kernel updates that may have started
-* concurrently with a user pmap update because the IPC code
-* changes mappings.
-* Spinning on the VALUES of the locks is sufficient (rather than
-* having to acquire the locks) because any updates that occur subsequent
-* to finding the lock unlocked will be signaled via another interrupt.
-* (This assumes the interrupt is cleared before the low level interrupt code 
-* calls pmap_update_interrupt()). 
-* 
-* The signaling processor must wait for any implicit updates in progress
-* to terminate before continuing with its update. Thus it must wait for an
-* acknowledgement of the interrupt from each processor for which such
-* references could be made. For maintaining this information, a set
-* cpus_active is used. A cpu is in this set if and only if it can 
-* use a pmap. When pmap_update_interrupt() is entered, a cpu is removed from
-* this set; when all such cpus are removed, it is safe to update.
-* 
-* Before attempting to acquire the update lock on a pmap, a cpu (A) must
-* be at least at the priority of the interprocessor interrupt
-* (splip<=splvm). Otherwise, A could grab a lock and be interrupted by a
-* kernel update; it would spin forever in pmap_update_interrupt() trying
-* to acquire the user pmap lock it had already acquired. Furthermore A
-* must remove itself from cpus_active.  Otherwise, another cpu holding
-* the lock (B) could be in the process of sending an update signal to A,
-* and thus be waiting for A to remove itself from cpus_active. If A is
-* spinning on the lock at priority this will never happen and a deadlock
-* will result.
-*/
-
-/*
- *	Signal another CPU that it must flush its TLB
- */
-void
-signal_cpus(
-	cpu_set		use_list,
-	pmap_t		pmap,
-	vm_offset_t	start_addr,
-	vm_offset_t	end_addr)
-{
-	register int		which_cpu, j;
-	register pmap_update_list_t	update_list_p;
-
-	while ((which_cpu = ffs((unsigned long)use_list)) != 0) {
-	    which_cpu -= 1;	/* convert to 0 origin */
+#if	MACH_KDB
 
-	    update_list_p = cpu_update_list(which_cpu);
-	    simple_lock(&update_list_p->lock);
+/* show phys page mappings and attributes */
 
-	    j = update_list_p->count;
-	    if (j >= UPDATE_LIST_SIZE) {
-		/*
-		 *	list overflowed.  Change last item to
-		 *	indicate overflow.
-		 */
-		update_list_p->item[UPDATE_LIST_SIZE-1].pmap  = kernel_pmap;
-		update_list_p->item[UPDATE_LIST_SIZE-1].start = VM_MIN_ADDRESS;
-		update_list_p->item[UPDATE_LIST_SIZE-1].end   = VM_MAX_KERNEL_ADDRESS;
-	    }
-	    else {
-		update_list_p->item[j].pmap  = pmap;
-		update_list_p->item[j].start = start_addr;
-		update_list_p->item[j].end   = end_addr;
-		update_list_p->count = j+1;
-	    }
-	    cpu_update_needed(which_cpu) = TRUE;
-	    simple_unlock(&update_list_p->lock);
-
-	    /* if its the kernel pmap, ignore cpus_idle */
-	    if (((cpus_idle & (1 << which_cpu)) == 0) ||
-		(pmap == kernel_pmap) || PMAP_REAL(which_cpu) == pmap)
-	      {
-		i386_signal_cpu(which_cpu, MP_TLB_FLUSH, ASYNC);
-	      }
-	    use_list &= ~(1 << which_cpu);
-	}
-}
+extern void	db_show_page(pmap_paddr_t pa);
 
 void
-process_pmap_updates(
-	register pmap_t		my_pmap)
+db_show_page(pmap_paddr_t pa)
 {
-	register int		my_cpu;
-	register pmap_update_list_t	update_list_p;
-	register int		j;
-	register pmap_t		pmap;
-
-	mp_disable_preemption();
-	my_cpu = cpu_number();
-	update_list_p = cpu_update_list(my_cpu);
-	simple_lock(&update_list_p->lock);
-
-	for (j = 0; j < update_list_p->count; j++) {
-	    pmap = update_list_p->item[j].pmap;
-	    if (pmap == my_pmap ||
-		pmap == kernel_pmap) {
-
-	      	if (pmap->ref_count <= 0) {
-			PMAP_CPU_CLR(pmap, my_cpu);
-			PMAP_REAL(my_cpu) = kernel_pmap;
-#ifdef PAE
-			set_cr3((unsigned int)kernel_pmap->pm_ppdpt);
-#else
-			set_cr3((unsigned int)kernel_pmap->pdirbase);
-#endif
-		} else
-			INVALIDATE_TLB(pmap,
-				       update_list_p->item[j].start,
-				       update_list_p->item[j].end);
-	    }
-	} 	
-	update_list_p->count = 0;
-	cpu_update_needed(my_cpu) = FALSE;
-	simple_unlock(&update_list_p->lock);
-	mp_enable_preemption();
-}
-
-/*
- *	Interrupt routine for TBIA requested from other processor.
- *	This routine can also be called at all interrupts time if
- *	the cpu was idle. Some driver interrupt routines might access
- *	newly allocated vm. (This is the case for hd)
- */
-void
-pmap_update_interrupt(void)
-{
-	register int		my_cpu;
-	spl_t			s;
-	register pmap_t		my_pmap;
-
-	mp_disable_preemption();
-	my_cpu = cpu_number();
-
-	/*
-	 *	Raise spl to splvm (above splip) to block out pmap_extract
-	 *	from IO code (which would put this cpu back in the active
-	 *	set).
-	 */
-	s = splhigh();
-	
-	my_pmap = PMAP_REAL(my_cpu);
-
-	if (!(my_pmap && pmap_in_use(my_pmap, my_cpu)))
-		my_pmap = kernel_pmap;
-
-	do {
-	    LOOP_VAR;
-
-	    /*
-	     *	Indicate that we're not using either user or kernel
-	     *	pmap.
-	     */
-	    i_bit_clear(my_cpu, &cpus_active);
-
-	    /*
-	     *	Wait for any pmap updates in progress, on either user
-	     *	or kernel pmap.
-	     */
-	    while (*(volatile int *)(&my_pmap->lock.interlock.lock_data) ||
-		   *(volatile int *)(&kernel_pmap->lock.interlock.lock_data)) {
-	    	LOOP_CHECK("pmap_update_interrupt", my_pmap);
-		cpu_pause();
-	    }
-
-	    process_pmap_updates(my_pmap);
-
-	    i_bit_set(my_cpu, &cpus_active);
-
-	} while (cpu_update_needed(my_cpu));
-	
-	splx(s);
-	mp_enable_preemption();
-}
-
-#if	MACH_KDB
-
-/* show phys page mappings and attributes */
-
-extern void	db_show_page(pmap_paddr_t pa);
-
-void
-db_show_page(pmap_paddr_t pa)
-{
-	pv_entry_t	pv_h;
-	int		pai;
-	char 		attr;
-	
-	pai = pa_index(pa);
-	pv_h = pai_to_pvh(pai);
+	pv_entry_t	pv_h;
+	int		pai;
+	char 		attr;
+	
+	pai = pa_index(pa);
+	pv_h = pai_to_pvh(pai);
 
 	attr = pmap_phys_attributes[pai];
 	printf("phys page %x ", pa);
@@ -3050,7 +3445,7 @@ void
 db_kvtophys(
 	vm_offset_t	vaddr)
 {
-	db_printf("0x%x", kvtophys(vaddr));
+	db_printf("0x%qx", kvtophys(vaddr));
 }
 
 /*
@@ -3061,7 +3456,7 @@ db_show_vaddrs(
 	pt_entry_t	*dirbase)
 {
 	pt_entry_t	*ptep, *pdep, tmp;
-	int		x, y, pdecnt, ptecnt;
+	unsigned int	x, y, pdecnt, ptecnt;
 
 	if (dirbase == 0) {
 		dirbase = kernel_pmap->dirbase;
@@ -3070,7 +3465,7 @@ db_show_vaddrs(
 		db_printf("need a dirbase...\n");
 		return;
 	}
-	dirbase = (pt_entry_t *) ((unsigned long) dirbase & ~INTEL_OFFMASK);
+	dirbase = (pt_entry_t *) (int) ((unsigned long) dirbase & ~INTEL_OFFMASK);
 
 	db_printf("dirbase: 0x%x\n", dirbase);
 
@@ -3115,153 +3510,24 @@ pmap_list_resident_pages(
 }
 #endif	/* MACH_VM_DEBUG */
 
-#ifdef MACH_BSD
-/*
- * pmap_pagemove
- *
- * BSD support routine to reassign virtual addresses.
- */
 
-void
-pmap_movepage(unsigned long from, unsigned long to, vm_size_t size)
-{
-	spl_t	spl;
-	pt_entry_t	*pte, saved_pte;
-
-	/* Lock the kernel map */
-	PMAP_READ_LOCK(kernel_pmap, spl);
-
-
-	while (size > 0) {
-		pte = pmap_pte(kernel_pmap, from);
-		if (pte == NULL)
-			panic("pmap_pagemove from pte NULL");
-		saved_pte = *pte;
-		PMAP_READ_UNLOCK(kernel_pmap, spl);
-
-		pmap_enter(kernel_pmap, to, (ppnum_t)i386_btop(i386_trunc_page(*pte)),
-			VM_PROT_READ|VM_PROT_WRITE, 0, *pte & INTEL_PTE_WIRED);
-
-		pmap_remove(kernel_pmap, (addr64_t)from, (addr64_t)(from+PAGE_SIZE));
-
-		PMAP_READ_LOCK(kernel_pmap, spl);
-		pte = pmap_pte(kernel_pmap, to);
-		if (pte == NULL)
-			panic("pmap_pagemove 'to' pte NULL");
-
-		*pte = saved_pte;
-
-		from += PAGE_SIZE;
-		to += PAGE_SIZE;
-		size -= PAGE_SIZE;
-	}
-
-	/* Get the processors to update the TLBs */
-	PMAP_UPDATE_TLBS(kernel_pmap, from, from+size);
-	PMAP_UPDATE_TLBS(kernel_pmap, to, to+size);
-
-	PMAP_READ_UNLOCK(kernel_pmap, spl);
-
-}
-#endif /* MACH_BSD */
 
 /* temporary workaround */
 boolean_t
-coredumpok(vm_map_t map, vm_offset_t va)
+coredumpok(__unused vm_map_t map, __unused vm_offset_t va)
 {
+#if 0
 	pt_entry_t     *ptep;
 
 	ptep = pmap_pte(map->pmap, va);
 	if (0 == ptep)
 		return FALSE;
 	return ((*ptep & (INTEL_PTE_NCACHE | INTEL_PTE_WIRED)) != (INTEL_PTE_NCACHE | INTEL_PTE_WIRED));
-}
-
-/*
- * grow the number of kernel page table entries, if needed
- */
-void
-pmap_growkernel(vm_offset_t addr)
-{
-#if GROW_KERNEL_FUNCTION_IMPLEMENTED
-	struct pmap *pmap;
-	int s;
-	vm_offset_t ptppaddr;
-	ppnum_t  ppn;
-	vm_page_t nkpg;
-	pd_entry_t newpdir = 0;
-
-	/*
-	 * Serialize.
-	 * Losers return to try again until the winner completes the work.
-	 */
-	if (kptobj == 0) panic("growkernel 0");
-	if (!vm_object_lock_try(kptobj)) {
-	    return;
-	}
-
-	vm_page_lock_queues();
-
-	s = splhigh();
-
-	/*
-	 * If this is the first time thru, locate the end of the
-	 * kernel page table entries and set nkpt to the current
-	 * number of kernel page table pages
-	 */
- 
-	if (kernel_vm_end == 0) {
-		kernel_vm_end = KERNBASE;
-		nkpt = 0;
-
-		while (pdir_pde(kernel_pmap->dirbase, kernel_vm_end)) {
-			kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
-			nkpt++;
-		}
-	}
-
-	/*
-	 * Now allocate and map the required number of page tables
-	 */
-	addr = (addr + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
-	while (kernel_vm_end < addr) {
-		if (pdir_pde(kernel_pmap->dirbase, kernel_vm_end)) {
-			kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
-			continue; /* someone already filled this one */
-		}
-
-		nkpg = vm_page_alloc(kptobj, nkpt);
-		if (!nkpg)
-			panic("pmap_growkernel: no memory to grow kernel");
-
-		nkpt++;
-		vm_page_wire(nkpg);
-		ppn  = nkpg->phys_page;
-		pmap_zero_page(ppn);
-		ptppaddr = i386_ptob(ppn);
-		newpdir = (pd_entry_t) (ptppaddr | INTEL_PTE_VALID | 
-					INTEL_PTE_RW | INTEL_PTE_REF | INTEL_PTE_MOD);
-		pdir_pde(kernel_pmap->dirbase, kernel_vm_end) = newpdir;
-
-		simple_lock(&free_pmap_lock);
-		for (pmap = (struct pmap *)kernel_pmap->pmap_link.next;
-		     pmap != kernel_pmap ;
-		     pmap = (struct pmap *)pmap->pmap_link.next ) {
-				*pmap_pde(pmap, kernel_vm_end) = newpdir;
-		}
-		simple_unlock(&free_pmap_lock);
-	}
-	splx(s);
-	vm_page_unlock_queues();
-	vm_object_unlock(kptobj);
+#else
+	return TRUE;
 #endif
 }
 
-pt_entry_t *
-pmap_mapgetpte(vm_map_t map, vm_offset_t v)
-{
-	return pmap_pte(map->pmap, v);
-}
 
 boolean_t
 phys_page_exists(
@@ -3318,27 +3584,88 @@ mapping_adjust()
 }
 
 void
-pmap_commpage_init(vm_offset_t kernel_commpage, vm_offset_t user_commpage, int cnt)
+pmap_commpage32_init(vm_offset_t kernel_commpage, vm_offset_t user_commpage, int cnt)
 {
   int i;
   pt_entry_t *opte, *npte;
   pt_entry_t pte;
 
+
   for (i = 0; i < cnt; i++) {
-    opte = pmap_pte(kernel_pmap, kernel_commpage);
+    opte = pmap_pte(kernel_pmap, (vm_map_offset_t)kernel_commpage);
     if (0 == opte) panic("kernel_commpage");
-    npte = pmap_pte(kernel_pmap, user_commpage);
-    if (0 == npte) panic("user_commpage");
     pte = *opte | INTEL_PTE_USER|INTEL_PTE_GLOBAL;
     pte &= ~INTEL_PTE_WRITE; // ensure read only
-    WRITE_PTE_FAST(npte, pte);
+    npte = pmap_pte(kernel_pmap, (vm_map_offset_t)user_commpage);
+    if (0 == npte) panic("user_commpage");
+    pmap_store_pte(npte, pte);
     kernel_commpage += INTEL_PGBYTES;
     user_commpage += INTEL_PGBYTES;
   }
 }
 
+#define PMAP_COMMPAGE64_CNT  (_COMM_PAGE64_AREA_USED/PAGE_SIZE)
+pt_entry_t pmap_commpage64_ptes[PMAP_COMMPAGE64_CNT];
+
+void
+pmap_commpage64_init(vm_offset_t kernel_commpage, __unused vm_map_offset_t user_commpage, int cnt)
+{
+  spl_t s;
+  int i;
+  pt_entry_t *kptep;
+
+  s = splhigh();
+  for (i = 0; i< cnt; i++) {
+    kptep = pmap_pte(kernel_pmap, (uint64_t)kernel_commpage + (i*PAGE_SIZE));
+    if ((0 == kptep) || (0 == (*kptep & INTEL_PTE_VALID))) panic("pmap_commpage64_init pte");
+    pmap_commpage64_ptes[i] = ((*kptep & ~INTEL_PTE_WRITE) | INTEL_PTE_USER);
+  }
+  splx(s);
+
+}
+
+void
+pmap_map_sharedpage(__unused task_t task, pmap_t p)
+{
+  pt_entry_t *ptep;
+  spl_t s;
+  int i;
+
+  if (!p->pm_64bit) return;
+  /* setup high 64 bit commpage */
+  s = splhigh();  
+  while ((ptep = pmap_pte(p, (uint64_t)_COMM_PAGE64_BASE_ADDRESS)) == PD_ENTRY_NULL) {
+    splx(s);
+    pmap_expand(p, (uint64_t)_COMM_PAGE64_BASE_ADDRESS);
+    s = splhigh();
+  }
+
+  for (i = 0; i< PMAP_COMMPAGE64_CNT; i++) {
+    ptep = pmap_pte(p, (uint64_t)_COMM_PAGE64_BASE_ADDRESS + (i*PAGE_SIZE));
+    if (0 == ptep) panic("pmap_map_sharedpage");
+    pmap_store_pte(ptep, pmap_commpage64_ptes[i]);
+  }
+  splx(s);
+
+}
+
+void
+pmap_unmap_sharedpage(pmap_t pmap)
+{
+  spl_t s;
+  pt_entry_t *ptep;
+  int i;
+
+  if (!pmap->pm_64bit) return;
+  s = splhigh();
+  for (i = 0; i< PMAP_COMMPAGE64_CNT; i++) {
+    ptep = pmap_pte(pmap, (uint64_t)_COMM_PAGE64_BASE_ADDRESS + (i*PAGE_SIZE));
+  if (ptep) pmap_store_pte(ptep, 0);
+  }
+  splx(s);
+}
+
 static cpu_pmap_t		cpu_pmap_master;
-static struct pmap_update_list	cpu_update_list_master;
 
 struct cpu_pmap *
 pmap_cpu_alloc(boolean_t is_boot_cpu)
@@ -3346,13 +3673,13 @@ pmap_cpu_alloc(boolean_t is_boot_cpu)
 	int			ret;
 	int			i;
 	cpu_pmap_t		*cp;
-	pmap_update_list_t	up;
 	vm_offset_t		address;
+	vm_map_address_t	mapaddr;
 	vm_map_entry_t		entry;
+	pt_entry_t		*pte;
 	
 	if (is_boot_cpu) {
 		cp = &cpu_pmap_master;
-		up = &cpu_update_list_master;
 	} else {
 		/*
 		 * The per-cpu pmap data structure itself.
@@ -3366,46 +3693,31 @@ pmap_cpu_alloc(boolean_t is_boot_cpu)
 		bzero((void *)cp, sizeof(cpu_pmap_t));
 
 		/*
-		 * The tlb flush update list.
+		 * The temporary windows used for copy/zero - see loose_ends.c
 		 */
-		ret = kmem_alloc(kernel_map,
-				 (vm_offset_t *) &up, sizeof(*up));
+		ret = vm_map_find_space(kernel_map,
+		    &mapaddr, PMAP_NWINDOWS*PAGE_SIZE, (vm_map_offset_t)0, 0, &entry);
 		if (ret != KERN_SUCCESS) {
-			printf("pmap_cpu_alloc() failed ret=%d\n", ret);
+			printf("pmap_cpu_alloc() "
+				"vm_map_find_space ret=%d\n", ret);
 			pmap_cpu_free(cp);
 			return NULL;
 		}
+		address = (vm_offset_t)mapaddr;
 
-		/*
-		 * The temporary windows used for copy/zero - see loose_ends.c
-		 */
-		for (i = 0; i < PMAP_NWINDOWS; i++) {
-			ret = vm_map_find_space(kernel_map,
-					&address, PAGE_SIZE, 0, &entry);
-			if (ret != KERN_SUCCESS) {
-				printf("pmap_cpu_alloc() "
-					"vm_map_find_space ret=%d\n", ret);
-				pmap_cpu_free(cp);
-				return NULL;
-			}
-			vm_map_unlock(kernel_map);
-
+		for (i = 0; i < PMAP_NWINDOWS; i++, address += PAGE_SIZE) {
+			while ((pte = pmap_pte(kernel_pmap, (vm_map_offset_t)address)) == 0)
+				pmap_expand(kernel_pmap, (vm_map_offset_t)address);
+			* (int *) pte = 0; 
 			cp->mapwindow[i].prv_CADDR = (caddr_t) address;
-			cp->mapwindow[i].prv_CMAP = vtopte(address);
-			* (int *) cp->mapwindow[i].prv_CMAP = 0; 
-
-			kprintf("pmap_cpu_alloc() "
-				"window=%d CADDR=0x%x CMAP=0x%x\n",
-				i, address, vtopte(address));
+			cp->mapwindow[i].prv_CMAP = pte;
 		}
+		vm_map_unlock(kernel_map);
 	}
 
-	/*
-	 *	Set up the pmap request list
-	 */
-	cp->update_list = up;
-	simple_lock_init(&up->lock, 0);
-	up->count = 0;
+	cp->pdpt_window_index = PMAP_PDPT_FIRST_WINDOW;
+	cp->pde_window_index = PMAP_PDE_FIRST_WINDOW;
+	cp->pte_window_index = PMAP_PTE_FIRST_WINDOW;
 
 	return cp;
 }
@@ -3414,9 +3726,461 @@ void
 pmap_cpu_free(struct cpu_pmap *cp)
 {
 	if (cp != NULL && cp != &cpu_pmap_master) {
-		if (cp->update_list != NULL)
-			kfree((void *) cp->update_list,
-				sizeof(*cp->update_list));
 		kfree((void *) cp, sizeof(cpu_pmap_t));
 	}
 }
+
+
+mapwindow_t *
+pmap_get_mapwindow(pt_entry_t pentry)
+{
+    mapwindow_t *mp;
+    int i;
+    boolean_t	istate;
+
+    /*
+     * can be called from hardware interrupt context
+     * so we need to protect the lookup process
+     */
+    istate = ml_set_interrupts_enabled(FALSE);
+
+    /*
+     * Note: 0th map reserved for pmap_pte()
+     */
+    for (i = PMAP_NWINDOWS_FIRSTFREE; i < PMAP_NWINDOWS; i++) {
+            mp = &current_cpu_datap()->cpu_pmap->mapwindow[i];
+
+	    if (*mp->prv_CMAP == 0) {
+	            *mp->prv_CMAP = pentry;
+		    break;
+	    }
+    }
+    if (i >= PMAP_NWINDOWS)
+            mp = NULL;
+    (void) ml_set_interrupts_enabled(istate);
+    
+    return (mp);
+}
+
+
+/*
+ *	kern_return_t pmap_nest(grand, subord, vstart, size)
+ *
+ *	grand  = the pmap that we will nest subord into
+ *	subord = the pmap that goes into the grand
+ *	vstart  = start of range in pmap to be inserted
+ *	nstart  = start of range in pmap nested pmap
+ *	size   = Size of nest area (up to 16TB)
+ *
+ *	Inserts a pmap into another.  This is used to implement shared segments.
+ *
+ *      on x86 this is very limited right now.  must be exactly 1 segment.
+ *
+ *	Note that we depend upon higher level VM locks to insure that things don't change while
+ *	we are doing this.  For example, VM should not be doing any pmap enters while it is nesting
+ *	or do 2 nests at once.
+ */
+
+
+kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t nstart, uint64_t size) {
+		
+        vm_map_offset_t	vaddr, nvaddr;
+	pd_entry_t	*pde,*npde;
+	unsigned int	i, need_flush;
+	unsigned int	num_pde;
+	spl_t		s;
+
+	// do validity tests
+
+	if(size & 0x0FFFFFFFULL) return KERN_INVALID_VALUE;	/* We can only do this for multiples of 256MB */
+	if((size >> 28) > 65536)  return KERN_INVALID_VALUE;	/* Max size we can nest is 16TB */
+	if(vstart & 0x0FFFFFFFULL) return KERN_INVALID_VALUE;	/* We can only do this aligned to 256MB */
+	if(nstart & 0x0FFFFFFFULL) return KERN_INVALID_VALUE;	/* We can only do this aligned to 256MB */
+	if(size == 0) {	   
+		panic("pmap_nest: size is invalid - %016llX\n", size);
+	}
+	if ((size >> 28) != 1) panic("pmap_nest: size 0x%llx must be 0x%x", size, NBPDE);
+
+	subord->pm_shared = TRUE;
+
+	// prepopulate subord pmap pde's if necessary
+
+	if (cpu_64bit) {
+	  s = splhigh();
+	  while (PD_ENTRY_NULL == (npde = pmap_pde(subord, nstart))) {
+	    splx(s);
+	    pmap_expand(subord, nstart);
+	    s = splhigh();
+	  }
+	  splx(s);
+	}
+
+	PMAP_READ_LOCK(subord,s);
+	nvaddr = (vm_map_offset_t)nstart;
+	need_flush = 0;
+	num_pde = size >> PDESHIFT;
+
+	for (i=0;i<num_pde;i++) {
+	  npde = pmap_pde(subord, nvaddr);
+	  if ((0 == npde) || (*npde++ & INTEL_PTE_VALID) == 0) {
+	    PMAP_READ_UNLOCK(subord,s);
+	    pmap_expand(subord, nvaddr); // pmap_expand handles races
+	    PMAP_READ_LOCK(subord,s);
+	    need_flush++;
+	  }
+	  nvaddr += NBPDE;
+	}
+
+	if (need_flush) {
+	  nvaddr = (vm_map_offset_t)nstart;
+	  PMAP_UPDATE_TLBS(subord, nvaddr, nvaddr + (1 << 28) -1 );
+	}
+	PMAP_READ_UNLOCK(subord,s);
+
+	// copy pde's from subord pmap into grand pmap
+
+	if (cpu_64bit) {
+	  s = splhigh();
+	  while (PD_ENTRY_NULL == (pde = pmap_pde(grand, vstart))) {
+	    splx(s);
+	    pmap_expand(grand, vstart);
+	    s = splhigh();
+	  }
+	  splx(s);
+	}
+
+	PMAP_READ_LOCK(grand,s);
+	vaddr = (vm_map_offset_t)vstart;
+	for (i=0;i<num_pde;i++,pde++) {
+	  pd_entry_t tpde;
+	  npde = pmap_pde(subord, nstart);
+	  if (npde == 0) panic("pmap_nest: no npde, subord 0x%x nstart 0x%llx", subord, nstart);
+	  tpde = *npde;
+	  nstart += NBPDE;
+	  pde = pmap_pde(grand, vaddr);
+	  if (pde == 0) panic("pmap_nest: no pde, grand  0x%x vaddr 0x%llx", grand, vaddr);
+	  vaddr += NBPDE;
+	  pmap_store_pte(pde, tpde);
+	}
+	PMAP_UPDATE_TLBS(grand, vaddr, vaddr + (1 << 28) -1 );
+
+	PMAP_READ_UNLOCK(grand,s);
+
+	return KERN_SUCCESS;
+}
+
+/*
+ *	kern_return_t pmap_unnest(grand, vaddr)
+ *
+ *	grand  = the pmap that we will nest subord into
+ *	vaddr  = start of range in pmap to be unnested
+ *
+ *	Removes a pmap from another.  This is used to implement shared segments.
+ *	On the current PPC processors, this is limited to segment (256MB) aligned
+ *	segment sized ranges.
+ */
+
+kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr) {
+			
+	spl_t s;
+	pd_entry_t *pde;
+	unsigned int i;
+	unsigned int num_pde;
+
+	PMAP_READ_LOCK(grand,s);
+
+	// invalidate all pdes for segment at vaddr in pmap grand
+
+	num_pde = (1<<28) >> PDESHIFT;
+
+	for (i=0;i<num_pde;i++,pde++) {
+	  pde = pmap_pde(grand, (vm_map_offset_t)vaddr);
+	  if (pde == 0) panic("pmap_unnest: no pde, grand 0x%x vaddr 0x%llx\n", grand, vaddr);
+	  pmap_store_pte(pde, (pd_entry_t)0);
+	  vaddr += NBPDE;
+	}
+	PMAP_UPDATE_TLBS(grand, vaddr, vaddr + (1<<28) -1 );
+
+	PMAP_READ_UNLOCK(grand,s);
+		
+	return KERN_SUCCESS;								/* Bye, bye, butterfly... */
+}
+
+void
+pmap_switch(pmap_t tpmap)
+{
+        spl_t	s;
+  	int	my_cpu;
+
+	s = splhigh();		/* Make sure interruptions are disabled */
+	my_cpu = cpu_number();
+
+	set_dirbase(tpmap, my_cpu);
+
+	splx(s);
+}
+
+
+/*
+ * disable no-execute capability on
+ * the specified pmap
+ */
+void pmap_disable_NX(pmap_t pmap) {
+  
+        pmap->nx_enabled = 0;
+}
+
+void
+pt_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size,
+		  vm_size_t *alloc_size, int *collectable, int *exhaustable)
+{
+        *count      = inuse_ptepages_count;
+	*cur_size   = PAGE_SIZE * inuse_ptepages_count;
+	*max_size   = PAGE_SIZE * (inuse_ptepages_count + vm_page_inactive_count + vm_page_active_count + vm_page_free_count);
+	*elem_size  = PAGE_SIZE;
+	*alloc_size = PAGE_SIZE;
+
+	*collectable = 1;
+	*exhaustable = 0;
+}
+
+vm_offset_t pmap_cpu_high_map_vaddr(int cpu, enum high_cpu_types e)
+{
+  enum high_fixed_addresses a;
+  a = e + HIGH_CPU_END * cpu;
+  return pmap_index_to_virt(HIGH_FIXED_CPUS_BEGIN + a);
+}
+
+vm_offset_t pmap_high_map_vaddr(enum high_cpu_types e)
+{
+  return pmap_cpu_high_map_vaddr(cpu_number(), e);
+}
+
+vm_offset_t pmap_high_map(pt_entry_t pte, enum high_cpu_types e)
+{
+  enum high_fixed_addresses a;
+  vm_offset_t vaddr;
+
+  a = e + HIGH_CPU_END * cpu_number();
+  vaddr = (vm_offset_t)pmap_index_to_virt(HIGH_FIXED_CPUS_BEGIN + a);
+  *(pte_unique_base + a) = pte;
+
+  /* TLB flush for this page for this  cpu */
+  invlpg((uintptr_t)vaddr);
+
+  return  vaddr;
+}
+
+
+/*
+ * Called with pmap locked, we:
+ *  - scan through per-cpu data to see which other cpus need to flush
+ *  - send an IPI to each non-idle cpu to be flushed
+ *  - wait for all to signal back that they are inactive or we see that
+ *    they are in an interrupt handler or at a safe point
+ *  - flush the local tlb is active for this pmap
+ *  - return ... the caller will unlock the pmap
+ */
+void
+pmap_flush_tlbs(pmap_t	pmap)
+{
+	unsigned int	cpu;
+	unsigned int	cpu_bit;
+	cpu_set		cpus_to_signal;
+	unsigned int	my_cpu = cpu_number();
+	pmap_paddr_t	pmap_cr3 = pmap->pm_cr3;
+	boolean_t	flush_self = FALSE;
+	uint64_t	deadline;
+
+	assert(!ml_get_interrupts_enabled());
+
+	/*
+	 * Scan other cpus for matching active or task CR3.
+	 * For idle cpus (with no active map) we mark them invalid but
+	 * don't signal -- they'll check as they go busy.
+	 * Note: for the kernel pmap we look for 64-bit shared address maps.
+	 */
+	cpus_to_signal = 0;
+	for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) {
+		if (!cpu_datap(cpu)->cpu_running)
+			continue;
+		if ((cpu_datap(cpu)->cpu_task_cr3   == pmap_cr3) ||
+		    (CPU_GET_ACTIVE_CR3(cpu) == pmap_cr3) ||
+		    (pmap->pm_shared) ||
+		    ((pmap == kernel_pmap) &&
+		     (!CPU_CR3_IS_ACTIVE(cpu) ||
+		      cpu_datap(cpu)->cpu_task_map == TASK_MAP_64BIT_SHARED))) {
+			if (cpu == my_cpu) {
+				flush_self = TRUE;
+				continue;
+			}
+			cpu_datap(cpu)->cpu_tlb_invalid = TRUE;
+			__asm__ volatile("mfence");
+
+			if (CPU_CR3_IS_ACTIVE(cpu)) {
+				cpus_to_signal |= cpu_bit;
+				i386_signal_cpu(cpu, MP_TLB_FLUSH, ASYNC);
+			}
+		}
+	}
+
+	if (cpus_to_signal) {
+	        KERNEL_DEBUG(0xef800024 | DBG_FUNC_START, cpus_to_signal, 0, 0, 0, 0);
+
+		deadline = mach_absolute_time() + LockTimeOut;
+		/*
+		 * Wait for those other cpus to acknowledge
+		 */
+		for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) {
+			while ((cpus_to_signal & cpu_bit) != 0) {
+			        if (!cpu_datap(cpu)->cpu_running ||
+				    cpu_datap(cpu)->cpu_tlb_invalid == FALSE ||
+				    !CPU_CR3_IS_ACTIVE(cpu)) {
+				        cpus_to_signal &= ~cpu_bit;
+					break;
+				}
+				if (mach_absolute_time() > deadline)
+				        panic("pmap_flush_tlbs() "
+					      "timeout pmap=%p cpus_to_signal=%p",
+					      pmap, cpus_to_signal);
+				cpu_pause();
+			}
+		        if (cpus_to_signal == 0)
+			        break;
+		}
+	        KERNEL_DEBUG(0xef800024 | DBG_FUNC_END, cpus_to_signal, 0, 0, 0, 0);
+	}
+
+	/*
+	 * Flush local tlb if required.
+	 * We need this flush even if the pmap being changed
+	 * is the user map... in case we do a copyin/out
+	 * before returning to user mode.
+	 */
+	if (flush_self)
+		flush_tlb();
+
+}
+
+void
+process_pmap_updates(void)
+{
+	flush_tlb();
+
+	current_cpu_datap()->cpu_tlb_invalid = FALSE;
+	__asm__ volatile("mfence");
+}
+
+void
+pmap_update_interrupt(void)
+{
+        KERNEL_DEBUG(0xef800028 | DBG_FUNC_START, 0, 0, 0, 0, 0);
+
+	assert(!ml_get_interrupts_enabled());
+
+	process_pmap_updates();
+
+        KERNEL_DEBUG(0xef800028 | DBG_FUNC_END, 0, 0, 0, 0, 0);
+}
+
+
+unsigned int pmap_cache_attributes(ppnum_t pn) {
+
+	if (!pmap_valid_page(pn))
+	        return (VM_WIMG_IO);
+
+	return (VM_WIMG_COPYBACK);
+}
+
+#ifdef PMAP_DEBUG
+void
+pmap_dump(pmap_t p)
+{
+  int i;
+
+  kprintf("pmap 0x%x\n",p);
+
+  kprintf("  pm_cr3 0x%llx\n",p->pm_cr3);
+  kprintf("  pm_pml4 0x%x\n",p->pm_pml4);
+  kprintf("  pm_pdpt 0x%x\n",p->pm_pdpt);
+
+  kprintf("    pml4[0] 0x%llx\n",*p->pm_pml4);
+  for (i=0;i<8;i++)
+    kprintf("    pdpt[%d] 0x%llx\n",i, p->pm_pdpt[i]);
+}
+
+void pmap_dump_wrap(void)
+{
+  pmap_dump(current_cpu_datap()->cpu_active_thread->task->map->pmap);
+}
+
+void
+dump_4GB_pdpt(pmap_t p)
+{
+	int		spl;
+	pdpt_entry_t	*user_pdptp;
+	pdpt_entry_t	*kern_pdptp;
+	pdpt_entry_t	*pml4p;
+
+	spl = splhigh();
+	while ((user_pdptp = pmap64_pdpt(p, 0x0)) == PDPT_ENTRY_NULL) {
+		splx(spl);
+		pmap_expand_pml4(p, 0x0);
+		spl = splhigh();
+	}
+	kern_pdptp = kernel_pmap->pm_pdpt;
+	if (kern_pdptp == NULL)
+		panic("kern_pdptp == NULL");
+	kprintf("dump_4GB_pdpt(%p)\n"
+		"kern_pdptp=%p (phys=0x%016llx)\n"
+		"\t 0x%08x: 0x%016llx\n"
+		"\t 0x%08x: 0x%016llx\n"
+		"\t 0x%08x: 0x%016llx\n"
+		"\t 0x%08x: 0x%016llx\n"
+		"\t 0x%08x: 0x%016llx\n"
+		"user_pdptp=%p (phys=0x%016llx)\n"
+		"\t 0x%08x: 0x%016llx\n"
+		"\t 0x%08x: 0x%016llx\n"
+		"\t 0x%08x: 0x%016llx\n"
+		"\t 0x%08x: 0x%016llx\n"
+		"\t 0x%08x: 0x%016llx\n",
+		p, kern_pdptp, kvtophys(kern_pdptp),
+		kern_pdptp+0, *(kern_pdptp+0),
+		kern_pdptp+1, *(kern_pdptp+1),
+		kern_pdptp+2, *(kern_pdptp+2),
+		kern_pdptp+3, *(kern_pdptp+3),
+		kern_pdptp+4, *(kern_pdptp+4),
+		user_pdptp, kvtophys(user_pdptp),
+		user_pdptp+0, *(user_pdptp+0),
+		user_pdptp+1, *(user_pdptp+1),
+		user_pdptp+2, *(user_pdptp+2),
+		user_pdptp+3, *(user_pdptp+3),
+		user_pdptp+4, *(user_pdptp+4));
+	kprintf("user pm_cr3=0x%016llx pm_hold=0x%08x pm_pml4=0x%08x\n",
+		p->pm_cr3, p->pm_hold, p->pm_pml4);
+	pml4p = (pdpt_entry_t *)p->pm_hold;
+	if (pml4p == NULL)
+		panic("user pml4p == NULL");
+	kprintf("\t 0x%08x: 0x%016llx\n"
+		"\t 0x%08x: 0x%016llx\n",
+		pml4p+0, *(pml4p),
+		pml4p+KERNEL_UBER_PML4_INDEX, *(pml4p+KERNEL_UBER_PML4_INDEX));
+	kprintf("kern pm_cr3=0x%016llx pm_hold=0x%08x pm_pml4=0x%08x\n",
+		kernel_pmap->pm_cr3, kernel_pmap->pm_hold, kernel_pmap->pm_pml4);
+	pml4p = (pdpt_entry_t *)kernel_pmap->pm_hold;
+	if (pml4p == NULL)
+		panic("kern pml4p == NULL");
+	kprintf("\t 0x%08x: 0x%016llx\n"
+		"\t 0x%08x: 0x%016llx\n",
+		pml4p+0, *(pml4p),
+		pml4p+511, *(pml4p+511));
+	splx(spl);
+}
+
+void dump_4GB_pdpt_thread(thread_t tp)
+{
+	dump_4GB_pdpt(tp->map->pmap);
+}
+
+
+#endif
diff --git a/osfmk/i386/pmap.h b/osfmk/i386/pmap.h
index 31c9a499d..8a50070d4 100644
--- a/osfmk/i386/pmap.h
+++ b/osfmk/i386/pmap.h
@@ -1,5 +1,6 @@
+
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -64,7 +65,7 @@
  *
  *	Machine-dependent structures for the physical map module.
  */
-
+#ifdef KERNEL_PRIVATE
 #ifndef	_PMAP_MACHINE_
 #define _PMAP_MACHINE_	1
 
@@ -80,10 +81,9 @@
 #include <kern/kern_types.h>
 #include <kern/thread.h>
 #include <kern/lock.h>
-#define PMAP_QUEUE 1
-#ifdef PMAP_QUEUE
-#include <kern/queue.h>
-#endif
+
+#include <i386/mp.h>
+#include <i386/proc_reg.h>
 
 /*
  *	Define the generic in terms of the specific
@@ -103,41 +103,22 @@
  *	i386/i486/i860 Page Table Entry
  */
 
-#ifdef PAE
-typedef uint64_t        pdpt_entry_t;
-typedef uint64_t 	pt_entry_t;
-typedef uint64_t        pd_entry_t;
-typedef uint64_t        pmap_paddr_t;
-#else
-typedef uint32_t 	pt_entry_t;
-typedef uint32_t        pd_entry_t;
-typedef uint32_t        pmap_paddr_t;
-#endif
-
-#define PT_ENTRY_NULL	((pt_entry_t *) 0)
-#define PD_ENTRY_NULL	((pt_entry_t *) 0)
-
 #endif	/* ASSEMBLER */
 
-#ifdef PAE
 #define NPGPTD          4
 #define PDESHIFT        21
 #define PTEMASK         0x1ff
 #define PTEINDX         3
-#else
-#define NPGPTD          1
-#define PDESHIFT        22
-#define PTEMASK         0x3ff
-#define PTEINDX         2
-#endif
+
 #define PTESHIFT        12
 
 #define PDESIZE		sizeof(pd_entry_t) /* for assembly files */
 #define PTESIZE		sizeof(pt_entry_t) /* for assembly files */
 
 #define INTEL_OFFMASK	(I386_PGBYTES - 1)
-#define PG_FRAME        (~((pmap_paddr_t)PAGE_MASK))
+#define PG_FRAME        0x000FFFFFFFFFF000ULL
 #define NPTEPG          (PAGE_SIZE/(sizeof (pt_entry_t)))
+#define NPTDPG          (PAGE_SIZE/(sizeof (pd_entry_t)))
 
 #define NBPTD           (NPGPTD << PAGE_SHIFT)
 #define NPDEPTD         (NBPTD / (sizeof (pd_entry_t)))
@@ -145,6 +126,82 @@ typedef uint32_t        pmap_paddr_t;
 #define NBPDE           (1 << PDESHIFT)
 #define PDEMASK         (NBPDE - 1)
 
+ /* cleanly define parameters for all the page table levels */
+typedef uint64_t        pml4_entry_t;
+#define NPML4PG         (PAGE_SIZE/(sizeof (pml4_entry_t)))
+#define PML4SHIFT       39
+#define PML4PGSHIFT     9
+#define NBPML4          (1ULL << PML4SHIFT)
+#define PML4MASK        (NBPML4-1)
+#define PML4_ENTRY_NULL ((pml4_entry_t *) 0)
+
+typedef uint64_t        pdpt_entry_t;
+#define NPDPTPG         (PAGE_SIZE/(sizeof (pdpt_entry_t)))
+#define PDPTSHIFT       30
+#define PDPTPGSHIFT     9
+#define NBPDPT          (1 << PDPTSHIFT)
+#define PDPTMASK        (NBPDPT-1)
+#define PDPT_ENTRY_NULL ((pdpt_entry_t *) 0)
+
+typedef uint64_t        pd_entry_t;
+#define NPDPG           (PAGE_SIZE/(sizeof (pd_entry_t)))
+#define PDSHIFT         21
+#define PDPGSHIFT       9
+#define NBPD            (1 << PDSHIFT)
+#define PDMASK          (NBPD-1)
+#define PD_ENTRY_NULL   ((pd_entry_t *) 0)
+
+typedef uint64_t        pt_entry_t;
+#define NPTPG           (PAGE_SIZE/(sizeof (pt_entry_t)))
+#define PTSHIFT         12
+#define PTPGSHIFT       9
+#define NBPT            (1 << PTSHIFT)
+#define PTMASK          (NBPT-1)
+#define PT_ENTRY_NULL	((pt_entry_t *) 0)
+
+typedef uint64_t  pmap_paddr_t;
+
+/*
+ * Atomic 64-bit store of a page table entry.
+ */
+static inline void
+pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
+{
+	/*
+	 * Load the new value into %ecx:%ebx
+	 * Load the old value into %edx:%eax
+	 * Compare-exchange-8bytes at address entryp (loaded in %edi)
+	 * If the compare succeeds, the new value will have been stored.
+	 * Otherwise, the old value changed and reloaded, so try again.
+	 */
+	asm volatile(
+		"	movl	(%0), %%eax	\n\t"
+		"	movl	4(%0), %%edx	\n\t"
+		"1:				\n\t"
+		"	cmpxchg8b (%0)		\n\t"
+		"	jnz 1b"
+		:
+		: "D" (entryp),
+		  "b" ((uint32_t)value),
+		  "c" ((uint32_t)(value >> 32))
+		: "eax", "edx", "memory");
+}
+
+/* in 64 bit spaces, the number of each type of page in the page tables */
+#define NPML4PGS        (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
+#define NPDPTPGS        (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
+#define NPDEPGS         (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t))))
+#define NPTEPGS         (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
+
+/*
+ * The 64-bit kernel is remapped in uber-space which is at the base
+ * the highest 4th-level directory (KERNEL_UBER_PML4_INDEX). That is,
+ * 512GB from the top of virtual space (or zero).
+ */
+#define KERNEL_UBER_PML4_INDEX	511
+#define KERNEL_UBER_BASE	(0ULL - NBPML4)
+#define KERNEL_UBER_BASE_HI32	((uint32_t)(KERNEL_UBER_BASE >> 32))
+
 #define	VM_WIMG_COPYBACK	VM_MEM_COHERENT
 #define	VM_WIMG_DEFAULT		VM_MEM_COHERENT
 /* ?? intel ?? */
@@ -154,56 +211,83 @@ typedef uint32_t        pmap_paddr_t;
 /* write combining mode, aka store gather */
 #define VM_WIMG_WCOMB		(VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT) 
 
+/*
+ * Pte related macros
+ */
+#define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDESHIFT)|((pti)<<PTESHIFT)))
+#define VADDR64(pmi, pdi, pti) ((vm_offset_t)(((pmi)<<PLM4SHIFT))((pdi)<<PDESHIFT)|((pti)<<PTESHIFT))
+
 /*
  * Size of Kernel address space.  This is the number of page table pages
  * (4MB each) to use for the kernel.  256 pages == 1 Gigabyte.
  * This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc).
  */
 #ifndef KVA_PAGES
-#define KVA_PAGES	256
+#define KVA_PAGES	1024
 #endif
 
-/*
- * Pte related macros
- */
-#define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDESHIFT)|((pti)<<PTESHIFT)))
-
 #ifndef NKPT
-#ifdef PAE
 #define	NKPT		500	/* actual number of kernel page tables */
-#else
-#define	NKPT	        32	/* initial number of kernel page tables */
-#endif
 #endif
 #ifndef NKPDE
 #define NKPDE	(KVA_PAGES - 1)	/* addressable number of page tables/pde's */
 #endif
 
+
+enum high_cpu_types {
+  HIGH_CPU_ISS0,
+  HIGH_CPU_ISS1,
+  HIGH_CPU_DESC,
+  HIGH_CPU_LDT_BEGIN,
+  HIGH_CPU_LDT_END = HIGH_CPU_LDT_BEGIN + (LDTSZ / 512) - 1,
+  HIGH_CPU_END
+};
+
+enum  high_fixed_addresses {
+  HIGH_FIXED_TRAMPS,  /* must be first */
+  HIGH_FIXED_TRAMPS_END,
+  HIGH_FIXED_GDT,
+  HIGH_FIXED_IDT,
+  HIGH_FIXED_LDT_BEGIN,
+  HIGH_FIXED_LDT_END = HIGH_FIXED_LDT_BEGIN + (LDTSZ / 512) - 1,
+  HIGH_FIXED_KTSS,
+  HIGH_FIXED_DFTSS,
+  HIGH_FIXED_DBTSS,
+  HIGH_FIXED_CPUS_BEGIN,
+  HIGH_FIXED_CPUS_END = HIGH_FIXED_CPUS_BEGIN + (HIGH_CPU_END * MAX_CPUS) - 1,
+};
+
+
+/* XXX64  below PTDI values need cleanup */
 /*
  * The *PTDI values control the layout of virtual memory
  *
  */
-#ifdef PAE
-#define        KPTDI           (0x600)/* start of kernel virtual pde's */
+#define        KPTDI           (0x000)/* start of kernel virtual pde's */
 #define        PTDPTDI         (0x7F4) /* ptd entry that points to ptd! */
 #define        APTDPTDI        (0x7F8) /* alt ptd entry that points to APTD */
-#define        UMAXPTDI        (0x5FC) /* ptd entry for user space end */
+#define        UMAXPTDI        (0x7F8) /* ptd entry for user space end */
 #define	UMAXPTEOFF	(NPTEPG)	/* pte entry for user space end */
-#else
-#define        KPTDI           (0x300)/* start of kernel virtual pde's */
-#define        PTDPTDI         (0x3FD) /* ptd entry that points to ptd! */
-#define        APTDPTDI        (0x3FE) /* alt ptd entry that points to APTD */
-#define        UMAXPTDI        (0x2FF) /* ptd entry for user space end */
-#define	UMAXPTEOFF	(NPTEPG)	/* pte entry for user space end */
-#endif
 
 #define KERNBASE       VADDR(KPTDI,0)
 
+/*
+ *	Convert address offset to directory address
+ *	containing the page table pointer - legacy
+ */
+/*#define pmap_pde(m,v) (&((m)->dirbase[(vm_offset_t)(v) >> PDESHIFT]))*/
+
+#define HIGH_MEM_BASE  ((uint32_t)( -NBPDE) )  /* shared gdt etc seg addr */ /* XXX64 ?? */
+#define pmap_index_to_virt(x)  (HIGH_MEM_BASE | ((unsigned)(x) << PAGE_SHIFT))
+
 /*
  *	Convert address offset to page descriptor index
  */
-#define pdenum(pmap, a) (((a) >> PDESHIFT) & PDEMASK)
+#define pdenum(pmap, a) (((vm_offset_t)(a) >> PDESHIFT) & PDEMASK)
 
+#define pdeidx(pmap, a)    (((a) >> PDSHIFT)   & ((1ULL<<(48 - PDSHIFT)) -1))
+#define pdptidx(pmap, a)   (((a) >> PDPTSHIFT) & ((1ULL<<(48 - PDPTSHIFT)) -1))
+#define pml4idx(pmap, a)   (((a) >> PML4SHIFT) & ((1ULL<<(48 - PML4SHIFT)) -1))
 
 /*
  *	Convert page descriptor index to user virtual address
@@ -213,7 +297,7 @@ typedef uint32_t        pmap_paddr_t;
 /*
  *	Convert address offset to page table index
  */
-#define ptenum(a)	(((a) >> PTESHIFT) & PTEMASK)
+#define ptenum(a)	(((vm_offset_t)(a) >> PTESHIFT) & PTEMASK)
 
 /*
  *	Hardware pte bit definitions (to be used directly on the ptes
@@ -231,13 +315,22 @@ typedef uint32_t        pmap_paddr_t;
 #define INTEL_PTE_PS            0x00000080
 #define INTEL_PTE_GLOBAL        0x00000100
 #define INTEL_PTE_WIRED		0x00000200
-#define INTEL_PTE_PFN		/*0xFFFFF000*/ (~0xFFF)
+#define INTEL_PTE_PFN		PG_FRAME
 #define INTEL_PTE_PTA		0x00000080
 
+#define INTEL_PTE_NX		(1ULL << 63)
+
+#define INTEL_PTE_INVALID       0
+
 #define	pa_to_pte(a)		((a) & INTEL_PTE_PFN) /* XXX */
 #define	pte_to_pa(p)		((p) & INTEL_PTE_PFN) /* XXX */
 #define	pte_increment_pa(p)	((p) += INTEL_OFFMASK+1)
 
+#define pte_kernel_rw(p)          ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_RW))
+#define pte_kernel_ro(p)          ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID))
+#define pte_user_rw(p)            ((pt_entry)t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER|INTEL_PTE_RW))
+#define pte_user_ro(p)            ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER))
+
 #define PMAP_DEFAULT_CACHE	0
 #define PMAP_INHIBIT_CACHE	1
 #define PMAP_GUARDED_CACHE	2
@@ -258,9 +351,14 @@ extern pt_entry_t PTmap[], APTmap[], Upte;
 extern pd_entry_t PTD[], APTD[], PTDpde[], APTDpde[], Upde;
 
 extern pd_entry_t *IdlePTD;	/* physical address of "Idle" state directory */
-#ifdef PAE
 extern pdpt_entry_t *IdlePDPT;
-#endif
+
+extern pmap_paddr_t lo_kernel_cr3;
+
+extern pml4_entry_t *IdlePML4;
+extern pdpt_entry_t *IdlePDPT64;
+extern addr64_t     kernel64_cr3;
+extern boolean_t    no_shared_cr3;
 
 /*
  * virtual address to page table entry and
@@ -268,7 +366,7 @@ extern pdpt_entry_t *IdlePDPT;
  * Note: these work recursively, thus vtopte of a pte will give
  * the corresponding pde that in turn maps it.
  */
-#define	vtopte(va)	(PTmap + i386_btop(va))
+#define	vtopte(va)	(PTmap + i386_btop((vm_offset_t)va))
 
 
 typedef	volatile long	cpu_set;	/* set of CPUs - must be <= 32 */
@@ -287,48 +385,51 @@ struct md_page {
  */
 
 struct pmap {
-#ifdef PMAP_QUEUE
-  queue_head_t     pmap_link;   /* unordered queue of in use pmaps */
-#endif
-	pd_entry_t	*dirbase;	/* page directory pointer register */
-	pd_entry_t	*pdirbase;	/* phys. address of dirbase */
-        vm_object_t     pm_obj;         /* object to hold pte's */
+        pd_entry_t      *dirbase;        /* page directory pointer */
+	pmap_paddr_t    pdirbase;        /* phys. address of dirbase */
+        vm_object_t     pm_obj;         /* object to hold pde's */
 	int		ref_count;	/* reference count */
+        int		nx_enabled;
+        boolean_t       pm_64bit;
+        boolean_t       pm_kernel_cr3;
+        boolean_t       pm_shared;
 	decl_simple_lock_data(,lock)	/* lock on map */
 	struct pmap_statistics	stats;	/* map statistics */
-	cpu_set		cpus_using;	/* bitmap of cpus using pmap */
-#ifdef PAE
 	vm_offset_t     pm_hold;        /* true pdpt zalloc addr */
-	pdpt_entry_t    *pm_pdpt;       /* KVA of pg dir ptr table */
-	vm_offset_t     pm_ppdpt;       /* phy addr pdpt
-					   should really be 32/64 bit */
-#endif
+	pmap_paddr_t    pm_cr3;         /* physical addr */
+        pdpt_entry_t    *pm_pdpt;       /* KVA of 3rd level page */
+	pml4_entry_t    *pm_pml4;       /* VKA of top level */
+	vm_object_t     pm_obj_pdpt;    /* holds pdpt pages */
+	vm_object_t     pm_obj_pml4;    /* holds pml4 pages */
+	vm_object_t     pm_obj_top;     /* holds single top level page */
 };
 
-#define PMAP_NWINDOWS	4
+
+#define PMAP_PDPT_FIRST_WINDOW 0
+#define PMAP_PDPT_NWINDOWS 4
+#define PMAP_PDE_FIRST_WINDOW (PMAP_PDPT_NWINDOWS)
+#define PMAP_PDE_NWINDOWS 4
+#define PMAP_PTE_FIRST_WINDOW (PMAP_PDE_FIRST_WINDOW + PMAP_PDE_NWINDOWS)
+#define PMAP_PTE_NWINDOWS 4
+
+#define PMAP_NWINDOWS_FIRSTFREE (PMAP_PTE_FIRST_WINDOW + PMAP_PTE_NWINDOWS)
+#define PMAP_WINDOW_SIZE 8
+#define PMAP_NWINDOWS (PMAP_NWINDOWS_FIRSTFREE + PMAP_WINDOW_SIZE)
+
 typedef struct {
 	pt_entry_t	*prv_CMAP;
 	caddr_t		prv_CADDR;
 } mapwindow_t;
 
 typedef struct cpu_pmap {
+        int                     pdpt_window_index;
+        int                     pde_window_index;
+        int                     pte_window_index;
 	mapwindow_t		mapwindow[PMAP_NWINDOWS];
-	struct pmap		*real_pmap;
-	struct pmap_update_list	*update_list;
-	volatile boolean_t	update_needed;
 } cpu_pmap_t;
 
-/*
- * Should be rewritten in asm anyway.
- */
-#define CM1 (current_cpu_datap()->cpu_pmap->mapwindow[0].prv_CMAP)
-#define CM2 (current_cpu_datap()->cpu_pmap->mapwindow[1].prv_CMAP)
-#define CM3 (current_cpu_datap()->cpu_pmap->mapwindow[2].prv_CMAP)
-#define CM4 (current_cpu_datap()->cpu_pmap->mapwindow[3].prv_CMAP)
-#define CA1 (current_cpu_datap()->cpu_pmap->mapwindow[0].prv_CADDR)
-#define CA2 (current_cpu_datap()->cpu_pmap->mapwindow[1].prv_CADDR)
-#define CA3 (current_cpu_datap()->cpu_pmap->mapwindow[2].prv_CADDR)
-#define CA4 (current_cpu_datap()->cpu_pmap->mapwindow[3].prv_CADDR)
+
+extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry);
 
 typedef struct pmap_memory_regions {
   ppnum_t base;
@@ -340,117 +441,62 @@ typedef struct pmap_memory_regions {
 unsigned pmap_memory_region_count;
 unsigned pmap_memory_region_current;
 
-#define PMAP_MEMORY_REGIONS_SIZE 32
+#define PMAP_MEMORY_REGIONS_SIZE 128
 
 extern pmap_memory_region_t pmap_memory_regions[];
 
-/* 
- * Optimization avoiding some TLB flushes when switching to
- * kernel-loaded threads.  This is effective only for i386:
- * Since user task, kernel task and kernel loaded tasks share the
- * same virtual space (with appropriate protections), any pmap
- * allows mapping kernel and kernel loaded tasks. 
- *
- * The idea is to avoid switching to another pmap unnecessarily when
- * switching to a kernel-loaded task, or when switching to the kernel
- * itself.
- *
- * We store the pmap we are really using (from which we fetched the
- * dirbase value) in current_cpu_datap()->cpu_pmap.real_pmap.
- *
- * Invariant:
- * current_pmap() == current_cpu_datap()->cpu_pmap.real_pmap ||
- *			current_pmap() == kernel_pmap.
- */
-#define	PMAP_REAL(my_cpu)	(cpu_datap(my_cpu)->cpu_pmap->real_pmap)
-
-#include <i386/proc_reg.h>
-/*
- * If switching to the kernel pmap, don't incur the TLB cost of switching
- * to its page tables, since all maps include the kernel map as a subset.
- * Simply record that this CPU is logically on the kernel pmap (see
- * pmap_destroy).
- * 
- * Similarly, if switching to a pmap (other than kernel_pmap that is already
- * in use, don't do anything to the hardware, to avoid a TLB flush.
- */
-
-#define	PMAP_CPU_SET(pmap, my_cpu) i_bit_set(my_cpu, &((pmap)->cpus_using))
-#define	PMAP_CPU_CLR(pmap, my_cpu) i_bit_clear(my_cpu, &((pmap)->cpus_using))
-
-#ifdef PAE
-#define PDIRBASE pm_ppdpt
-#else
-#define PDIRBASE pdirbase
-#endif
-#define	set_dirbase(mypmap, my_cpu) {					\
-	struct pmap	**ppmap = &PMAP_REAL(my_cpu);			\
-	pmap_paddr_t	pdirbase = (pmap_paddr_t)((mypmap)->PDIRBASE);  	\
-									\
-	if (*ppmap == (pmap_paddr_t)NULL) {				\
-		*ppmap = (mypmap);					\
-		PMAP_CPU_SET((mypmap), my_cpu);				\
-		set_cr3(pdirbase);					\
-	} else if ((mypmap) != kernel_pmap && (mypmap) != *ppmap ) {	\
-		if (*ppmap != kernel_pmap)				\
-			PMAP_CPU_CLR(*ppmap, my_cpu);			\
-		*ppmap = (mypmap);					\
-		PMAP_CPU_SET((mypmap), my_cpu);				\
-		set_cr3(pdirbase);					\
-	}								\
-	assert((mypmap) == *ppmap || (mypmap) == kernel_pmap);		\
+static inline void set_dirbase(pmap_t tpmap, __unused int tcpu) {
+    	current_cpu_datap()->cpu_task_cr3 = (pmap_paddr_t)((tpmap)->pm_cr3);
+	current_cpu_datap()->cpu_task_map = tpmap->pm_64bit ? TASK_MAP_64BIT : TASK_MAP_32BIT;
 }
 
-/*
- *	List of cpus that are actively using mapped memory.  Any
- *	pmap update operation must wait for all cpus in this list.
- *	Update operations must still be queued to cpus not in this
- *	list.
- */
-extern cpu_set		cpus_active;
-
-/*
- *	List of cpus that are idle, but still operating, and will want
- *	to see any kernel pmap updates when they become active.
- */
-extern cpu_set		cpus_idle;
-
-
-#define cpu_update_needed(cpu)	cpu_datap(cpu)->cpu_pmap->update_needed
-#define cpu_update_list(cpu)	cpu_datap(cpu)->cpu_pmap->update_list
-
 /*
  *	External declarations for PMAP_ACTIVATE.
  */
 
-extern void		process_pmap_updates(struct pmap *pmap);
+extern void		process_pmap_updates(void);
 extern void		pmap_update_interrupt(void);
 
 /*
  *	Machine dependent routines that are used only for i386/i486/i860.
  */
 
-extern vm_offset_t	(kvtophys)(
+extern addr64_t		(kvtophys)(
 				vm_offset_t	addr);
 
 extern pt_entry_t	*pmap_pte(
 				struct pmap	*pmap,
-				vm_offset_t	addr);
+				vm_map_offset_t	addr);
+
+extern pd_entry_t	*pmap_pde(
+				struct pmap	*pmap,
+				vm_map_offset_t	addr);
+
+extern pd_entry_t	*pmap64_pde(
+				struct pmap	*pmap,
+				vm_map_offset_t	addr);
+
+extern pdpt_entry_t	*pmap64_pdpt(
+				struct pmap	*pmap,
+				vm_map_offset_t	addr);
 
 extern vm_offset_t	pmap_map(
 				vm_offset_t	virt,
-				vm_offset_t	start,
-				vm_offset_t	end,
-				vm_prot_t	prot);
+				vm_map_offset_t	start,
+				vm_map_offset_t	end,
+				vm_prot_t	prot,
+				unsigned int	flags);
 
 extern vm_offset_t	pmap_map_bd(
 				vm_offset_t	virt,
-				vm_offset_t	start,
-				vm_offset_t	end,
-				vm_prot_t	prot);
+				vm_map_offset_t	start,
+				vm_map_offset_t	end,
+				vm_prot_t	prot,
+				unsigned int	flags);
 
 extern void		pmap_bootstrap(
-				vm_offset_t	load_start);
+				vm_offset_t	load_start,
+				boolean_t	IA32e);
 
 extern boolean_t	pmap_valid_page(
 				ppnum_t	pn);
@@ -460,14 +506,28 @@ extern int		pmap_list_resident_pages(
 				vm_offset_t	*listp,
 				int		space);
 
-extern void             pmap_commpage_init(
+extern void             pmap_commpage32_init(
 					   vm_offset_t kernel,
 					   vm_offset_t user,
 					   int count);
+extern void             pmap_commpage64_init(
+					   vm_offset_t	kernel,
+					   vm_map_offset_t user,
+					   int count);
+
 extern struct cpu_pmap	*pmap_cpu_alloc(
 				boolean_t	is_boot_cpu);
 extern void		pmap_cpu_free(
 				struct cpu_pmap	*cp);
+
+extern void		pmap_map_block(
+				pmap_t pmap, 
+				addr64_t va,
+				ppnum_t pa,
+				uint32_t size,
+				vm_prot_t prot,
+				int attr,
+				unsigned int flags);
 				
 extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
 extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
@@ -475,6 +535,24 @@ extern ppnum_t          pmap_find_phys(pmap_t map, addr64_t va);
 extern void pmap_sync_page_data_phys(ppnum_t pa);
 extern void pmap_sync_page_attributes_phys(ppnum_t pa);
 
+extern kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t nstart, uint64_t size);
+extern kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr);
+extern void pmap_map_sharedpage(task_t task, pmap_t pmap);
+extern void pmap_unmap_sharedpage(pmap_t pmap);
+extern void pmap_disable_NX(pmap_t pmap);
+extern void pmap_set_4GB_pagezero(pmap_t pmap);
+extern void pmap_clear_4GB_pagezero(pmap_t pmap);
+extern void pmap_load_kernel_cr3(void);
+extern vm_offset_t pmap_cpu_high_map_vaddr(int, enum high_cpu_types);
+extern vm_offset_t pmap_high_map_vaddr(enum high_cpu_types);
+extern vm_offset_t pmap_high_map(pt_entry_t, enum high_cpu_types);
+extern vm_offset_t pmap_cpu_high_shared_remap(int, enum high_cpu_types, vm_offset_t, int);
+extern vm_offset_t pmap_high_shared_remap(enum high_fixed_addresses, vm_offset_t, int);
+
+extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, int *, int *);
+
+
+
 /*
  *	Macros for speed.
  */
@@ -489,125 +567,114 @@ extern void pmap_sync_page_attributes_phys(ppnum_t pa);
 #undef PMAP_DEACTIVATE_USER
 #endif
 
-/*
- *	For multiple CPUS, PMAP_ACTIVATE and PMAP_DEACTIVATE must manage
- *	fields to control TLB invalidation on other CPUS.
- */
 
-#define	PMAP_ACTIVATE_KERNEL(my_cpu)	{				\
-									\
-	/*								\
-	 *	Let pmap updates proceed while we wait for this pmap.	\
-	 */								\
-	i_bit_clear((my_cpu), &cpus_active);				\
-									\
-	/*								\
-	 *	Lock the pmap to put this cpu in its active set.	\
-	 *	Wait for updates here.					\
-	 */								\
-	simple_lock(&kernel_pmap->lock);				\
-									\
-	/*								\
-	 *	Process invalidate requests for the kernel pmap.	\
-	 */								\
-	if (cpu_update_needed(my_cpu))					\
-	    process_pmap_updates(kernel_pmap);				\
-									\
-	/*								\
-	 *	Mark that this cpu is using the pmap.			\
-	 */								\
-	i_bit_set((my_cpu), &kernel_pmap->cpus_using);			\
-									\
-	/*								\
-	 *	Mark this cpu active - IPL will be lowered by		\
-	 *	load_context().						\
-	 */								\
-	i_bit_set((my_cpu), &cpus_active);				\
-									\
-	simple_unlock(&kernel_pmap->lock);				\
+#define PMAP_ACTIVATE_KERNEL(my_cpu)  {					\
+        spl_t		spl;						\
+                                                                        \
+        spl = splhigh();						\
+	if (current_cpu_datap()->cpu_tlb_invalid)			\
+	        process_pmap_updates();					\
+        splx(spl);							\
 }
 
-#define	PMAP_DEACTIVATE_KERNEL(my_cpu)	{				\
-	/*								\
-	 *	Mark pmap no longer in use by this cpu even if		\
-	 *	pmap is locked against updates.				\
-	 */								\
-	i_bit_clear((my_cpu), &kernel_pmap->cpus_using);		\
-	i_bit_clear((my_cpu), &cpus_active);				\
-	PMAP_REAL(my_cpu) = NULL;					\
+#define PMAP_DEACTIVATE_KERNEL(my_cpu) {				\
+        spl_t		spl;						\
+                                                                        \
+        spl = splhigh();						\
+	process_pmap_updates();						\
+        splx(spl);							\
 }
 
+
 #define PMAP_ACTIVATE_MAP(map, my_cpu)	{				\
 	register pmap_t		tpmap;					\
-									\
-	tpmap = vm_map_pmap(map);					\
-	if (tpmap == kernel_pmap) {					\
-	    /*								\
-	     *	If this is the kernel pmap, switch to its page tables.	\
-	     */								\
-	    set_dirbase(kernel_pmap, my_cpu);				\
-	}								\
-	else {								\
-	    /*								\
-	     *	Let pmap updates proceed while we wait for this pmap.	\
-	     */								\
-	    i_bit_clear((my_cpu), &cpus_active);			\
-									\
-	    /*								\
-	     *	Lock the pmap to put this cpu in its active set.	\
-	     *	Wait for updates here.					\
-	     */								\
-	    simple_lock(&tpmap->lock);					\
-									\
-	    /*								\
-	     *	No need to invalidate the TLB - the entire user pmap	\
-	     *	will be invalidated by reloading dirbase.		\
-	     */								\
-	    set_dirbase(tpmap, my_cpu);					\
-									\
-	    /*								\
-	     *	Mark this cpu active - IPL will be lowered by		\
-	     *	load_context().						\
-	     */								\
-	    i_bit_set((my_cpu), &cpus_active);				\
-									\
-	    simple_unlock(&tpmap->lock);				\
-	}								\
+                                                                        \
+        tpmap = vm_map_pmap(map);					\
+        set_dirbase(tpmap, my_cpu);					\
 }
 
-#define PMAP_DEACTIVATE_MAP(map, my_cpu)
+#define PMAP_DEACTIVATE_MAP(map, my_cpu)                                \
+        if (current_cpu_datap()->cpu_task_map == TASK_MAP_64BIT_SHARED) \
+            pmap_load_kernel_cr3();
 
-#define PMAP_ACTIVATE_USER(th, my_cpu)	{				\
-	spl_t		spl;						\
-									\
-	spl = splhigh();							\
-	PMAP_ACTIVATE_MAP(th->map, my_cpu)				\
-	splx(spl);							\
+#define PMAP_ACTIVATE_USER(th, my_cpu)  {				\
+        spl_t		spl;						\
+                                                                        \
+        spl = splhigh();						\
+        PMAP_ACTIVATE_MAP(th->map, my_cpu)				\
+        splx(spl);							\
 }
 
 #define PMAP_DEACTIVATE_USER(th, my_cpu)
 
+
 #define	PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) {			\
 	spl_t		spl;						\
-									\
+	pt_entry_t	*kpdp;						\
+	pt_entry_t	*updp;						\
+        int		i;						\
+        int		need_flush;					\
+                                                                        \
+        need_flush = 0;							\
+        spl = splhigh();						\
 	if (old_th->map != new_th->map) {				\
-		spl = splhigh();						\
 		PMAP_DEACTIVATE_MAP(old_th->map, my_cpu);		\
 		PMAP_ACTIVATE_MAP(new_th->map, my_cpu);			\
-		splx(spl);						\
 	}								\
+        kpdp = current_cpu_datap()->cpu_copywindow_pdp;			\
+        for (i = 0; i < NCOPY_WINDOWS; i++) {				\
+                if (new_th->machine.copy_window[i].user_base != (user_addr_t)-1) {	\
+	                updp = pmap_pde(new_th->map->pmap,		\
+                              new_th->machine.copy_window[i].user_base);\
+	                pmap_store_pte(kpdp,  updp ? *updp : 0);        \
+                }							\
+                kpdp++;							\
+        }								\
+	splx(spl);							\
+        if (new_th->machine.copyio_state == WINDOWS_OPENED)		\
+                need_flush = 1;						\
+        else								\
+                new_th->machine.copyio_state = WINDOWS_DIRTY;		\
+        if (new_th->machine.physwindow_pte) {				\
+                pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep),	\
+				 new_th->machine.physwindow_pte);	\
+                if (need_flush == 0)					\
+                        invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);\
+        }								\
+        if (need_flush)							\
+                flush_tlb();						\
 }
 
 #define	PMAP_SWITCH_USER(th, new_map, my_cpu) {				\
 	spl_t		spl;						\
 									\
-	spl = splhigh();							\
+	spl = splhigh();						\
 	PMAP_DEACTIVATE_MAP(th->map, my_cpu);				\
 	th->map = new_map;						\
 	PMAP_ACTIVATE_MAP(th->map, my_cpu);				\
 	splx(spl);							\
+        inval_copy_windows(th);						\
 }
 
+/*
+ * Marking the current cpu's cr3 inactive is achieved by setting its lsb.
+ * Marking the current cpu's cr3 active once more involves clearng this bit.
+ * Note that valid page tables are page-aligned and so the bottom 12 bits
+ * are noramlly zero.
+ * We can only mark the current cpu active/inactive but we can test any cpu.
+ */
+#define CPU_CR3_MARK_INACTIVE()						\
+	current_cpu_datap()->cpu_active_cr3 |= 1
+
+#define CPU_CR3_MARK_ACTIVE()	 					\
+	current_cpu_datap()->cpu_active_cr3 &= ~1
+
+#define CPU_CR3_IS_ACTIVE(cpu)						\
+	((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0)
+
+#define CPU_GET_ACTIVE_CR3(cpu)                                         \
+        (cpu_datap(cpu)->cpu_active_cr3 & ~1)
+
 #define MARK_CPU_IDLE(my_cpu)	{					\
 	/*								\
 	 *	Mark this cpu idle, and remove it from the active set,	\
@@ -617,13 +684,16 @@ extern void pmap_sync_page_attributes_phys(ppnum_t pa);
 	 *	becomes active.						\
 	 */								\
 	int	s = splhigh();						\
-	i_bit_set((my_cpu), &cpus_idle);				\
-	i_bit_clear((my_cpu), &cpus_active);				\
+	if (!cpu_mode_is64bit() || no_shared_cr3)			\
+		process_pmap_updates();					\
+	else								\
+		pmap_load_kernel_cr3();					\
+	CPU_CR3_MARK_INACTIVE();					\
+	__asm__ volatile("mfence");					\
 	splx(s);							\
-	set_led(my_cpu); 						\
 }
 
-#define MARK_CPU_ACTIVE(my_cpu)	{					\
+#define MARK_CPU_ACTIVE(my_cpu) {					\
 									\
 	int	s = splhigh();						\
 	/*								\
@@ -636,23 +706,20 @@ extern void pmap_sync_page_attributes_phys(ppnum_t pa);
 	 *	set assures that we will receive another update		\
 	 *	interrupt if this happens.				\
 	 */								\
-	i_bit_clear((my_cpu), &cpus_idle);				\
-									\
-	if (cpu_update_needed(my_cpu))					\
-	    pmap_update_interrupt();					\
+	CPU_CR3_MARK_ACTIVE();						\
+	__asm__ volatile("mfence");					\
 									\
-	/*								\
-	 *	Mark that this cpu is now active.			\
-	 */								\
-	i_bit_set((my_cpu), &cpus_active);				\
+	if (current_cpu_datap()->cpu_tlb_invalid)			\
+	    process_pmap_updates();					\
 	splx(s);							\
-	clear_led(my_cpu); 						\
 }
 
 #define PMAP_CONTEXT(pmap, thread)
 
 #define pmap_kernel_va(VA)	\
-	(((VA) >= VM_MIN_KERNEL_ADDRESS) && ((VA) <= VM_MAX_KERNEL_ADDRESS))
+	((((vm_offset_t) (VA)) >= vm_min_kernel_address) &&	\
+	 (((vm_offset_t) (VA)) <= vm_max_kernel_address))
+
 
 #define pmap_resident_count(pmap)	((pmap)->stats.resident_count)
 #define	pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr)
@@ -663,4 +730,8 @@ extern void pmap_sync_page_attributes_phys(ppnum_t pa);
 
 #endif	/* ASSEMBLER */
 
+
 #endif	/* _PMAP_MACHINE_ */
+
+
+#endif  /* KERNEL_PRIVATE */
diff --git a/osfmk/i386/postcode.h b/osfmk/i386/postcode.h
index 2b9ecfab0..a9a637e6d 100644
--- a/osfmk/i386/postcode.h
+++ b/osfmk/i386/postcode.h
@@ -39,7 +39,7 @@
 /* The POSTCODE is port 0x80 */
 #define	POSTPORT 0x80
 
-#define	SPINCOUNT	100000000
+#define	SPINCOUNT	300000000
 #define CPU_PAUSE()	rep; nop
 
 #if DEBUG
@@ -47,6 +47,7 @@
  * Macro to output byte value to postcode, destoying register al.
  * Additionally, if POSTCODE_DELAY, spin for about a second.
  */
+
 #if POSTCODE_DELAY
 #define POSTCODE_AL			\
         outb    %al,$(POSTPORT);	\
@@ -102,37 +103,50 @@
  * The following postcodes are defined for stages of early startup:
  */
 
-#define	PSTART_ENTRY		0xFF
-#define PSTART_PAGE_TABLES	0xFE
-#define PSTART_BEFORE_PAGING	0xFD
-#define VSTART_ENTRY		0xFC
-#define VSTART_STACK_SWITCH	0xFB
-#define VSTART_EXIT		0xFA
-#define	I386_INIT_ENTRY		0xF9
-#define	CPU_INIT_D		0xF8
-#define	PROCESSOR_BOOTSTRAP_D	0xF7
-#define	PE_INIT_PLATFORM_D	0xF6
-#define	THREAD_BOOTSTRAP_D	0xF5
-
-#define	SLAVE_PSTART_ENTRY	0xEF
-#define	REAL_TO_PROT_ENTRY	0xEE
-#define	REAL_TO_PROT_EXIT	0xED
-#define	STARTPROG_ENTRY		0xEC
-#define	STARTPROG_EXIT		0xEB
-#define	SLAVE_START_ENTRY	0xEA
-#define	SLAVE_START_EXIT	0xE9
-#define	SVSTART_ENTRY		0xE8
-#define	SVSTART_DESC_INIT	0xE7
-#define	SVSTART_STACK_SWITCH	0xE6
-#define	SVSTART_EXIT		0xE5
-#define	I386_INIT_SLAVE		0xE4
-
-#define	MP_KDP_ENTER		0xDB		/* Machine in kdp DeBugger */
-#define	PANIC_HLT		0xD1		/* Die an early death */ 
-
-#define ACPI_WAKE_START_ENTRY	0xCF
-#define ACPI_WAKE_PROT_ENTRY	0xCE
-#define ACPI_WAKE_PAGED_ENTRY	0xCD
+#define	_PSTART_ENTRY			0xFF
+#define	_PSTART_RELOC			0xFE
+#define	PSTART_ENTRY			0xFD
+#define PSTART_PAGE_TABLES		0xFC
+#define PSTART_BEFORE_PAGING		0xFB
+#define VSTART_ENTRY			0xFA
+#define VSTART_STACK_SWITCH		0xF9
+#define VSTART_EXIT			0xF8
+#define	I386_INIT_ENTRY			0xF7
+#define	CPU_INIT_D			0xF6
+#define	PE_INIT_PLATFORM_D		0xF5
+
+#define	SLAVE_RSTART_ENTRY		0xEF
+#define	SLAVE_REAL_TO_PROT_ENTRY	0xEE
+#define	SLAVE_REAL_TO_PROT_EXIT		0xED
+#define	SLAVE_STARTPROG_ENTRY		0xEC
+#define	SLAVE_STARTPROG_EXIT		0xEB
+#define	SLAVE_PSTART_ENTRY		0xEA
+#define	SLAVE_PSTART_EXIT		0xE9
+#define	SLAVE_VSTART_ENTRY		0xE8
+#define	SLAVE_VSTART_DESC_INIT		0xE7
+#define	SLAVE_VSTART_STACK_SWITCH	0xE6
+#define	SLAVE_VSTART_EXIT		0xE5
+#define	I386_INIT_SLAVE			0xE4
+
+#define	PANIC_DOUBLE_FAULT		0xDF	/* Double Fault exception */
+#define	PANIC_MACHINE_CHECK		0xDE	/* Machine-Check */
+#define	MP_KDP_ENTER			0xDB	/* Machine in kdp DeBugger */
+#define	PANIC_HLT			0xD1	/* Die an early death */ 
+#define	NO_64BIT			0x64	/* No 64-bit support yet */
+
+#define ACPI_WAKE_START_ENTRY		0xCF
+#define ACPI_WAKE_PROT_ENTRY		0xCE
+#define ACPI_WAKE_PAGED_ENTRY		0xCD
+
+#define	CPU_IA32_ENABLE_ENTRY		0xBF
+#define	CPU_IA32_ENABLE_EXIT		0xBE
+#define ML_LOAD_DESC64_ENTRY		0xBD
+#define ML_LOAD_DESC64_GDT		0xBC
+#define ML_LOAD_DESC64_IDT		0xBB
+#define ML_LOAD_DESC64_LDT		0xBA
+#define ML_LOAD_DESC64_EXIT		0xB9
+#define	CPU_IA32_DISABLE_ENTRY		0xB8
+#define	CPU_IA32_DISABLE_EXIT		0xB7
 
 #ifndef ASSEMBLER
 inline static void
diff --git a/osfmk/i386/proc_reg.h b/osfmk/i386/proc_reg.h
index 38c09a1b7..cb8fb43da 100644
--- a/osfmk/i386/proc_reg.h
+++ b/osfmk/i386/proc_reg.h
@@ -195,9 +195,17 @@ static inline void set_cr3(unsigned int value)
 	__asm__ volatile("mov %0, %%cr3" : : "r" (value));
 }
 
-/* Implemented in locore: */
-extern uint32_t	get_cr4(void);
-extern void	set_cr4(uint32_t);
+static inline uint32_t get_cr4(void)
+{
+	uint32_t cr4;
+	__asm__ volatile("mov %%cr4, %0" : "=r" (cr4));
+	return(cr4);
+}
+
+static inline void set_cr4(uint32_t value)
+{
+	__asm__ volatile("mov %0, %%cr4" : : "r" (value));
+}
 
 static inline void clear_ts(void)
 {
@@ -216,23 +224,30 @@ static inline void set_tr(unsigned int seg)
 	__asm__ volatile("ltr %0" : : "rm" ((unsigned short)(seg)));
 }
 
-static inline unsigned short get_ldt(void)
+static inline unsigned short sldt(void)
 {
 	unsigned short seg;
 	__asm__ volatile("sldt %0" : "=rm" (seg));
 	return(seg);
 }
 
-static inline void set_ldt(unsigned int seg)
+static inline void lldt(unsigned int seg)
 {
 	__asm__ volatile("lldt %0" : : "rm" ((unsigned short)(seg)));
 }
 
+#ifdef MACH_KERNEL_PRIVATE
+extern void flush_tlb64(void);
 static inline void flush_tlb(void)
 {
 	unsigned long	cr3_temp;
+	if (cpu_mode_is64bit()) {
+		flush_tlb64();
+		return;
+	}
 	__asm__ volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (cr3_temp) :: "memory");
 }
+#endif	/* MACH_KERNEL_PRIVATE */
 
 static inline void wbinvd(void)
 {
@@ -310,8 +325,14 @@ __END_DECLS
 #define MSR_IA32_PERFCTR0		0xc1
 #define MSR_IA32_PERFCTR1		0xc2
 
+#define MSR_PMG_CST_CONFIG_CONTROL	0xe2
+
 #define MSR_IA32_BBL_CR_CTL		0x119
 
+#define MSR_IA32_SYSENTER_CS		0x174
+#define MSR_IA32_SYSENTER_ESP		0x175
+#define MSR_IA32_SYSENTER_EIP		0x176
+
 #define MSR_IA32_MCG_CAP		0x179
 #define MSR_IA32_MCG_STATUS		0x17a
 #define MSR_IA32_MCG_CTL		0x17b
@@ -319,6 +340,9 @@ __END_DECLS
 #define MSR_IA32_EVNTSEL0		0x186
 #define MSR_IA32_EVNTSEL1		0x187
 
+#define MSR_IA32_PERF_STS		0x198
+#define MSR_IA32_PERF_CTL		0x199
+
 #define MSR_IA32_MISC_ENABLE		0x1a0
 
 #define MSR_IA32_DEBUGCTLMSR		0x1d9
@@ -350,4 +374,20 @@ __END_DECLS
 #define MSR_IA32_MTRR_FIX4K_F0000	0x26e
 #define MSR_IA32_MTRR_FIX4K_F8000	0x26f
 
+
+#define	MSR_IA32_EFER		0xC0000080
+#define	MSR_IA32_EFER_SCE		0x00000001
+#define	MSR_IA32_EFER_LME		0x00000100
+#define	MSR_IA32_EFER_LMA		0x00000400
+#define	MSR_IA32_EFER_NXE		0x00000800
+
+#define	MSR_IA32_STAR		0xC0000081
+#define	MSR_IA32_LSTAR		0xC0000082
+#define	MSR_IA32_CSTAR		0xC0000083
+#define	MSR_IA32_FMASK		0xC0000084
+
+#define MSR_IA32_FS_BASE	0xC0000100
+#define MSR_IA32_GS_BASE	0xC0000101
+#define MSR_IA32_KERNEL_GS_BASE	0xC0000102
+
 #endif	/* _I386_PROC_REG_H_ */
diff --git a/osfmk/i386/rtclock.c b/osfmk/i386/rtclock.c
index 1c7984497..4a39287c3 100644
--- a/osfmk/i386/rtclock.c
+++ b/osfmk/i386/rtclock.c
@@ -57,7 +57,7 @@
 #include <vm/vm_kern.h>		/* for kernel_map */
 #include <i386/ipl.h>
 #include <i386/pit.h>
-#include <i386/pio.h>
+#include <architecture/i386/pio.h>
 #include <i386/misc_protos.h>
 #include <i386/proc_reg.h>
 #include <i386/machine_cpu.h>
@@ -67,11 +67,13 @@
 #include <i386/cpu_threads.h>
 #include <i386/perfmon.h>
 #include <i386/machine_routines.h>
-#include <i386/AT386/bbclock_entries.h>
 #include <pexpert/pexpert.h>
 #include <machine/limits.h>
 #include <machine/commpage.h>
 #include <sys/kdebug.h>
+#include <i386/tsc.h>
+#include <i386/hpet.h>
+#include <i386/rtclock.h>
 
 #define MAX(a,b) (((a)>(b))?(a):(b))
 #define MIN(a,b) (((a)>(b))?(b):(a))
@@ -80,429 +82,62 @@
 
 #define UI_CPUFREQ_ROUNDING_FACTOR	10000000
 
-int		sysclk_config(void);
+int		rtclock_config(void);
 
-int		sysclk_init(void);
+int		rtclock_init(void);
 
-kern_return_t	sysclk_gettime(
-	mach_timespec_t			*cur_time);
+uint64_t	rtc_decrementer_min;
 
-kern_return_t	sysclk_getattr(
-	clock_flavor_t			flavor,
-	clock_attr_t			attr,
-	mach_msg_type_number_t	*count);
+void			rtclock_intr(x86_saved_state_t *regs);
+static uint64_t		maxDec;			/* longest interval our hardware timer can handle (nsec) */
 
-void		sysclk_setalarm(
-	mach_timespec_t			*alarm_time);
+/* XXX this should really be in a header somewhere */
+extern clock_timer_func_t	rtclock_timer_expire;
 
-/*
- * Lists of clock routines.
- */
-struct clock_ops  sysclk_ops = {
-	sysclk_config,			sysclk_init,
-	sysclk_gettime,			0,
-	sysclk_getattr,			0,
-	sysclk_setalarm,
-};
-
-int		calend_config(void);
-
-int		calend_init(void);
-
-kern_return_t	calend_gettime(
-	mach_timespec_t			*cur_time);
-
-kern_return_t	calend_getattr(
-	clock_flavor_t			flavor,
-	clock_attr_t			attr,
-	mach_msg_type_number_t	*count);
-
-struct clock_ops calend_ops = {
-	calend_config,			calend_init,
-	calend_gettime,			0,
-	calend_getattr,			0,
-	0,
-};
-
-/* local data declarations */
-
-static clock_timer_func_t	rtclock_timer_expire;
-
-static timer_call_data_t	rtclock_alarm_timer;
-
-static void	rtclock_alarm_expire(
-			timer_call_param_t	p0,
-			timer_call_param_t	p1);
-
-struct	{
-	mach_timespec_t			calend_offset;
-	boolean_t			calend_is_set;
-
-	int64_t				calend_adjtotal;
-	int32_t				calend_adjdelta;
+static void	rtc_set_timescale(uint64_t cycles);
+static uint64_t	rtc_export_speed(uint64_t cycles);
 
-	uint32_t			boottime;
+extern void		rtc_nanotime_store(
+					uint64_t		tsc,
+					uint64_t		nsec,
+					uint32_t		scale,
+					uint32_t		shift,
+					rtc_nanotime_t	*dst);
 
-        mach_timebase_info_data_t	timebase_const;
+extern void		rtc_nanotime_load(
+					rtc_nanotime_t	*src,
+					rtc_nanotime_t	*dst);
 
-	decl_simple_lock_data(,lock)	/* real-time clock device lock */
-} rtclock;
-
-boolean_t		rtc_initialized = FALSE;
-clock_res_t		rtc_intr_nsec = NSEC_PER_HZ;	/* interrupt res */
-uint64_t		rtc_cycle_count;	/* clocks in 1/20th second */
-uint64_t		rtc_cyc_per_sec;	/* processor cycles per sec */
-uint32_t		rtc_boot_frequency;	/* provided by 1st speed-step */
-uint32_t		rtc_quant_scale;	/* clock to nanos multiplier */
-uint32_t		rtc_quant_shift;	/* clock to nanos right shift */
-uint64_t		rtc_decrementer_min;
-
-static	mach_timebase_info_data_t	rtc_lapic_scale; /* nsec to lapic count */
-
-/*
- *	Macros to lock/unlock real-time clock data.
- */
-#define RTC_INTRS_OFF(s)		\
-	(s) = splclock()
-
-#define RTC_INTRS_ON(s)			\
-	splx(s)
-
-#define RTC_LOCK(s)			\
-MACRO_BEGIN				\
-	RTC_INTRS_OFF(s);		\
-	simple_lock(&rtclock.lock);	\
-MACRO_END
-
-#define RTC_UNLOCK(s)			\
-MACRO_BEGIN				\
-	simple_unlock(&rtclock.lock);	\
-	RTC_INTRS_ON(s);		\
-MACRO_END
+rtc_nanotime_t	rtc_nanotime_info;
 
 /*
- * i8254 control.  ** MONUMENT **
- *
- * The i8254 is a traditional PC device with some arbitrary characteristics.
- * Basically, it is a register that counts at a fixed rate and can be
- * programmed to generate an interrupt every N counts.  The count rate is
- * clknum counts per sec (see pit.h), historically 1193167=14.318MHz/12
- * but the more accurate value is 1193182=14.31818MHz/12. [14.31818 MHz being
- * the master crystal oscillator reference frequency since the very first PC.]
- * Various constants are computed based on this value, and we calculate
- * them at init time for execution efficiency.  To obtain sufficient
- * accuracy, some of the calculation are most easily done in floating
- * point and then converted to int.
+ * tsc_to_nanoseconds:
  *
+ * Basic routine to convert a raw 64 bit TSC value to a
+ * 64 bit nanosecond value.  The conversion is implemented
+ * based on the scale factor and an implicit 32 bit shift.
  */
-
-/*
- * Forward decl.
- */
-
-static uint64_t	rtc_set_cyc_per_sec(uint64_t cycles);
-uint64_t	rtc_nanotime_read(void);
-
-/*
- * create_mul_quant_GHZ
- *   create a constant used to multiply the TSC by to convert to nanoseconds.
- *   This is a 32 bit number and the TSC *MUST* have a frequency higher than
- *   1000Mhz for this routine to work.
- *
- * The theory here is that we know how many TSCs-per-sec the processor runs at.
- * Normally to convert this to nanoseconds you would multiply the current
- * timestamp by 1000000000 (a billion) then divide by TSCs-per-sec.
- * Unfortunatly the TSC is 64 bits which would leave us with 96 bit intermediate
- * results from the multiply that must be divided by.
- * Usually thats
- *   uint96 = tsc * numer
- *   nanos = uint96 / denom
- * Instead, we create this quant constant and it becomes the numerator,
- * the denominator can then be 0x100000000 which makes our division as simple as
- * forgetting the lower 32 bits of the result. We can also pass this number to
- * user space as the numer and pass 0xFFFFFFFF (RTC_FAST_DENOM) as the denom to
- * convert raw counts * to nanos. The difference is so small as to be
- * undetectable by anything.
- *
- * Unfortunatly we can not do this for sub GHZ processors. In this case, all
- * we do is pass the CPU speed in raw as the denom and we pass in 1000000000
- * as the numerator. No short cuts allowed
- */
-#define RTC_FAST_DENOM	0xFFFFFFFF
-inline static uint32_t
-create_mul_quant_GHZ(int shift, uint32_t quant)
-{
-	return (uint32_t)((((uint64_t)NSEC_PER_SEC/20) << shift) / quant);
-}
-/*
- * This routine takes a value of raw TSC ticks and applies the passed mul_quant
- * generated by create_mul_quant() This is our internal routine for creating
- * nanoseconds.
- * Since we don't really have uint96_t this routine basically does this....
- *   uint96_t intermediate = (*value) * scale
- *   return (intermediate >> 32)
- */
-inline static uint64_t
-fast_get_nano_from_abs(uint64_t value, int scale)
-{
-    asm (" 	movl	%%edx,%%esi	\n\t"
-         "      mull	%%ecx		\n\t"
-         "      movl	%%edx,%%edi	\n\t"
-         "      movl	%%esi,%%eax	\n\t"
-         "      mull	%%ecx		\n\t"
-         "      xorl	%%ecx,%%ecx	\n\t"	
-         "      addl	%%edi,%%eax	\n\t"	
-         "      adcl	%%ecx,%%edx	    "
-		: "+A" (value)
-		: "c" (scale)
-		: "%esi", "%edi");
-    return value;
-}
-
-/*
- * This routine basically does this...
- * ts.tv_sec = nanos / 1000000000;	create seconds
- * ts.tv_nsec = nanos % 1000000000;	create remainder nanos
- */
-inline static mach_timespec_t 
-nanos_to_timespec(uint64_t nanos)
-{
-	union {
-		mach_timespec_t ts;
-		uint64_t u64;
-	} ret;
-        ret.u64 = nanos;
-        asm volatile("divl %1" : "+A" (ret.u64) : "r" (NSEC_PER_SEC));
-        return ret.ts;
-}
-
-/*
- * The following two routines perform the 96 bit arithmetic we need to
- * convert generic absolute<->nanoseconds
- * The multiply routine takes a uint64_t and a uint32_t and returns the result
- * in a uint32_t[3] array.
- * The divide routine takes this uint32_t[3] array and divides it by a uint32_t
- * returning a uint64_t
- */
-inline static void
-longmul(uint64_t	*abstime, uint32_t multiplicand, uint32_t *result)
-{
-    asm volatile(
-        " pushl	%%ebx			\n\t"	
-        " movl	%%eax,%%ebx		\n\t"
-        " movl	(%%eax),%%eax		\n\t"
-        " mull	%%ecx			\n\t"
-        " xchg	%%eax,%%ebx		\n\t"
-        " pushl	%%edx			\n\t"
-        " movl	4(%%eax),%%eax		\n\t"
-        " mull	%%ecx			\n\t"
-        " movl	%2,%%ecx		\n\t"
-        " movl	%%ebx,(%%ecx)		\n\t"
-        " popl	%%ebx			\n\t"
-        " addl	%%ebx,%%eax		\n\t"
-        " popl	%%ebx			\n\t"
-        " movl	%%eax,4(%%ecx)		\n\t"
-        " adcl	$0,%%edx		\n\t"
-        " movl	%%edx,8(%%ecx)	// and save it"
-        : : "a"(abstime), "c"(multiplicand), "m"(result));
-    
-}
-
-inline static uint64_t
-longdiv(uint32_t *numer, uint32_t denom)
-{
-    uint64_t	result;
-    asm volatile(
-        " pushl	%%ebx			\n\t"
-        " movl	%%eax,%%ebx		\n\t"
-        " movl	8(%%eax),%%edx		\n\t"
-        " movl	4(%%eax),%%eax		\n\t"
-        " divl	%%ecx			\n\t"
-        " xchg	%%ebx,%%eax		\n\t"
-        " movl	(%%eax),%%eax		\n\t"
-        " divl	%%ecx			\n\t"
-        " xchg	%%ebx,%%edx		\n\t"
-        " popl	%%ebx			\n\t"
-        : "=A"(result) : "a"(numer),"c"(denom));
-    return result;
-}
-
-/*
- * Enable or disable timer 2.
- * Port 0x61 controls timer 2:
- *   bit 0 gates the clock,
- *   bit 1 gates output to speaker.
- */
-inline static void
-enable_PIT2(void)
-{
-    asm volatile(
-        " inb   $0x61,%%al      \n\t"
-        " and   $0xFC,%%al       \n\t"
-        " or    $1,%%al         \n\t"
-        " outb  %%al,$0x61      \n\t"
-        : : : "%al" );
-}
-
-inline static void
-disable_PIT2(void)
-{
-    asm volatile(
-        " inb   $0x61,%%al      \n\t"
-        " and   $0xFC,%%al      \n\t"
-        " outb  %%al,$0x61      \n\t"
-        : : : "%al" );
-}
-
-inline static void
-set_PIT2(int value)
-{
-/*
- * First, tell the clock we are going to write 16 bits to the counter
- *   and enable one-shot mode (command 0xB8 to port 0x43)
- * Then write the two bytes into the PIT2 clock register (port 0x42).
- * Loop until the value is "realized" in the clock,
- * this happens on the next tick.
- */
-    asm volatile(
-        " movb  $0xB8,%%al      \n\t"
-        " outb	%%al,$0x43	\n\t"
-        " movb	%%dl,%%al	\n\t"
-        " outb	%%al,$0x42	\n\t"
-        " movb	%%dh,%%al	\n\t"
-        " outb	%%al,$0x42	\n"
-"1:	  inb	$0x42,%%al	\n\t" 
-        " inb	$0x42,%%al	\n\t"
-        " cmp	%%al,%%dh	\n\t"
-        " jne	1b"
-        : : "d"(value) : "%al");
-}
-
-inline static uint64_t
-get_PIT2(unsigned int *value)
-{
-    register uint64_t	result;
-/*
- * This routine first latches the time (command 0x80 to port 0x43),
- * then gets the time stamp so we know how long the read will take later.
- * Read (from port 0x42) and return the current value of the timer.
- */
-    asm volatile(
-        " xorl	%%ecx,%%ecx	\n\t"
-        " movb	$0x80,%%al	\n\t"
-        " outb	%%al,$0x43	\n\t"
-        " rdtsc			\n\t"
-        " pushl	%%eax		\n\t"
-        " inb	$0x42,%%al	\n\t"
-        " movb	%%al,%%cl	\n\t"
-        " inb	$0x42,%%al	\n\t"
-        " movb	%%al,%%ch	\n\t"
-        " popl	%%eax	"
-        : "=A"(result), "=c"(*value));
-    return result;
-}
-
-/*
- * timeRDTSC()
- * This routine sets up PIT counter 2 to count down 1/20 of a second.
- * It pauses until the value is latched in the counter
- * and then reads the time stamp counter to return to the caller.
- */
-static uint64_t
-timeRDTSC(void)
+static inline uint64_t
+_tsc_to_nanoseconds(uint64_t value)
 {
-    int		attempts = 0;
-    uint64_t	latchTime;
-    uint64_t	saveTime,intermediate;
-    unsigned int timerValue, lastValue;
-    boolean_t   int_enabled;
-    /*
-     * Table of correction factors to account for
-     *   - timer counter quantization errors, and
-     *   - undercounts 0..5
-     */
-#define	SAMPLE_CLKS_EXACT	(((double) CLKNUM) / 20.0)
-#define	SAMPLE_CLKS_INT		((int) CLKNUM / 20)
-#define SAMPLE_NSECS		(2000000000LL)
-#define SAMPLE_MULTIPLIER	(((double)SAMPLE_NSECS)*SAMPLE_CLKS_EXACT)
-#define ROUND64(x)		((uint64_t)((x) + 0.5))
-    uint64_t	scale[6] = {
-	ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-0)), 
-	ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-1)), 
-	ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-2)), 
-	ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-3)), 
-	ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-4)), 
-	ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-5))
-    };
-                            
-    int_enabled = ml_set_interrupts_enabled(FALSE);
-    
-restart:
-    if (attempts >= 2)
-	panic("timeRDTSC() calibation failed with %d attempts\n", attempts);
-    attempts++;
-    enable_PIT2();      // turn on PIT2
-    set_PIT2(0);	// reset timer 2 to be zero
-    latchTime = rdtsc64();	// get the time stamp to time 
-    latchTime = get_PIT2(&timerValue) - latchTime; // time how long this takes
-    set_PIT2(SAMPLE_CLKS_INT);	// set up the timer for (almost) 1/20th a second
-    saveTime = rdtsc64();	// now time how long a 20th a second is...
-    get_PIT2(&lastValue);
-    get_PIT2(&lastValue);	// read twice, first value may be unreliable
-    do {
-        intermediate = get_PIT2(&timerValue);
-        if (timerValue > lastValue) {
-	    printf("Hey we are going backwards! %u -> %u, restarting timing\n",
-			timerValue,lastValue);
-	    set_PIT2(0);
-	    disable_PIT2();
-	    goto restart;
-	}
-        lastValue = timerValue;
-    } while (timerValue > 5);
-    kprintf("timerValue   %d\n",timerValue);
-    kprintf("intermediate 0x%016llx\n",intermediate);
-    kprintf("saveTime     0x%016llx\n",saveTime);
-    
-    intermediate -= saveTime;		// raw count for about 1/20 second
-    intermediate *= scale[timerValue];	// rescale measured time spent
-    intermediate /= SAMPLE_NSECS;	// so its exactly 1/20 a second
-    intermediate += latchTime;		// add on our save fudge
-    
-    set_PIT2(0);			// reset timer 2 to be zero
-    disable_PIT2();     		// turn off PIT 2
-
-    ml_set_interrupts_enabled(int_enabled);
-    return intermediate;
-}
+    asm volatile("movl	%%edx,%%esi	;"
+				 "mull	%%ecx		;"
+				 "movl	%%edx,%%edi	;"
+				 "movl	%%esi,%%eax	;"
+				 "mull	%%ecx		;"
+				 "addl	%%edi,%%eax	;"	
+				 "adcl	$0,%%edx	 "
+				 		: "+A" (value) : "c" (rtc_nanotime_info.scale) : "esi", "edi");
 
-static uint64_t
-tsc_to_nanoseconds(uint64_t abstime)
-{
-        uint32_t	numer;
-        uint32_t	denom;
-        uint32_t	intermediate[3];
-        
-        numer = rtclock.timebase_const.numer;
-        denom = rtclock.timebase_const.denom;
-        if (denom == RTC_FAST_DENOM) {
-            abstime = fast_get_nano_from_abs(abstime, numer);
-        } else {
-            longmul(&abstime, numer, intermediate);
-            abstime = longdiv(intermediate, denom);
-        }
-        return abstime;
+    return (value);
 }
 
-inline static mach_timespec_t 
-tsc_to_timespec(void)
+uint64_t
+tsc_to_nanoseconds(uint64_t value)
 {
-        uint64_t	currNanos;
-        currNanos = rtc_nanotime_read();
-        return nanos_to_timespec(currNanos);
+	return _tsc_to_nanoseconds(value);
 }
 
-#define	DECREMENTER_MAX		UINT_MAX
 static uint32_t
 deadline_to_decrementer(
 	uint64_t	deadline,
@@ -514,86 +149,28 @@ deadline_to_decrementer(
 		return rtc_decrementer_min;
 	else {
 		delta = deadline - now;
-		return MIN(MAX(rtc_decrementer_min,delta),DECREMENTER_MAX); 
+		return MIN(MAX(rtc_decrementer_min,delta),maxDec); 
 	}
 }
 
-static inline uint64_t
-lapic_time_countdown(uint32_t initial_count)
-{
-	boolean_t		state;
-	uint64_t		start_time;
-	uint64_t		stop_time;
-	lapic_timer_count_t	count;
-
-	state = ml_set_interrupts_enabled(FALSE);
-	lapic_set_timer(FALSE, one_shot, divide_by_1, initial_count);
-	start_time = rdtsc64();
-	do {
-		lapic_get_timer(NULL, NULL, NULL, &count);
-	} while (count > 0);
-	stop_time = rdtsc64();
-	ml_set_interrupts_enabled(state);
-
-	return tsc_to_nanoseconds(stop_time - start_time);
-}
-
-static void
-rtc_lapic_timer_calibrate(void)
-{
-	uint32_t	nsecs;
-	uint64_t	countdown;
-
-	if (!(cpuid_features() & CPUID_FEATURE_APIC))
-		return;
-
-	/*
-	 * Set the local apic timer counting down to zero without an interrupt.
-	 * Use the timestamp to calculate how long this takes.
-	 */ 
-	nsecs = (uint32_t) lapic_time_countdown(rtc_intr_nsec);
-
-	/*
-	 * Compute a countdown ratio for a given time in nanoseconds.
-	 * That is, countdown = time * numer / denom.
-	 */
-	countdown = (uint64_t)rtc_intr_nsec * (uint64_t)rtc_intr_nsec / nsecs;
-
-	nsecs = (uint32_t) lapic_time_countdown((uint32_t) countdown);
-
-	rtc_lapic_scale.numer = countdown;
-	rtc_lapic_scale.denom = nsecs;
-
-	kprintf("rtc_lapic_timer_calibrate() scale: %d/%d\n",
-		(uint32_t) countdown, nsecs);
-}
-
-static void
-rtc_lapic_set_timer(
-	uint32_t	interval)
-{
-	uint64_t	count;
-
-	assert(rtc_lapic_scale.denom);
-
-	count = interval * (uint64_t) rtc_lapic_scale.numer;
-	count /= rtc_lapic_scale.denom;
-
-	lapic_set_timer(TRUE, one_shot, divide_by_1, (uint32_t) count);
-}
-
 static void
 rtc_lapic_start_ticking(void)
 {
 	uint64_t	abstime;
 	uint64_t	first_tick;
-	uint64_t	decr;
+	cpu_data_t      *cdp = current_cpu_datap();
 
 	abstime = mach_absolute_time();
-	first_tick = abstime + NSEC_PER_HZ;
-	current_cpu_datap()->cpu_rtc_tick_deadline = first_tick;
-	decr = deadline_to_decrementer(first_tick, abstime);
-	rtc_lapic_set_timer(decr);
+	rtclock_tick_interval = NSEC_PER_HZ;
+
+	first_tick = abstime + rtclock_tick_interval;
+	cdp->rtclock_intr_deadline = first_tick;
+
+	/*
+	 * Force a complete re-evaluation of timer deadlines.
+	 */
+	cdp->rtcPop = EndOfAllTime;
+	etimer_resync_deadlines();
 }
 
 /*
@@ -602,20 +179,9 @@ rtc_lapic_start_ticking(void)
  */
 
 int
-sysclk_config(void)
+rtclock_config(void)
 {
-
-	mp_disable_preemption();
-	if (cpu_number() != master_cpu) {
-		mp_enable_preemption();
-		return(1);
-	}
-	mp_enable_preemption();
-
-	timer_call_setup(&rtclock_alarm_timer, rtclock_alarm_expire, NULL);
-
-	simple_lock_init(&rtclock.lock, 0);
-
+	/* nothing to do */
 	return (1);
 }
 
@@ -623,235 +189,169 @@ sysclk_config(void)
 /*
  * Nanotime/mach_absolutime_time
  * -----------------------------
- * The timestamp counter (tsc) - which counts cpu clock cycles and can be read
- * efficient by the kernel and in userspace - is the reference for all timing.
- * However, the cpu clock rate is not only platform-dependent but can change
- * (speed-step) dynamically. Hence tsc is converted into nanoseconds which is
- * identical to mach_absolute_time. The conversion to tsc to nanoseconds is
- * encapsulated by nanotime.
+ * The timestamp counter (TSC) - which counts cpu clock cycles and can be read
+ * efficiently by the kernel and in userspace - is the reference for all timing.
+ * The cpu clock rate is platform-dependent and may stop or be reset when the
+ * processor is napped/slept.  As a result, nanotime is the software abstraction
+ * used to maintain a monotonic clock, adjusted from an outside reference as needed.
  *
  * The kernel maintains nanotime information recording:
- * 	- the current ratio of tsc to nanoseconds
+ * 	- the ratio of tsc to nanoseconds
  *	  with this ratio expressed as a 32-bit scale and shift
  *	  (power of 2 divider);
- *	- the tsc (step_tsc) and nanotime (step_ns) at which the current
- *	  ratio (clock speed) began.
- * So a tsc value can be converted to nanotime by:
+ *	- { tsc_base, ns_base } pair of corresponding timestamps.
  *
- *	nanotime = (((tsc - step_tsc)*scale) >> shift) + step_ns
+ * The tuple {tsc_base, ns_base, scale, shift} is exported in the commpage 
+ * for the userspace nanotime routine to read.
  *
- * In general, (tsc - step_tsc) is a 64-bit quantity with the scaling
- * involving a 96-bit intermediate value. However, by saving the converted 
- * values at each tick (or at any intervening speed-step) - base_tsc and
- * base_ns - we can perform conversions relative to these and be assured that
- * (tsc - tick_tsc) is 32-bits. Hence:
- *
- * 	fast_nanotime = (((tsc - base_tsc)*scale) >> shift) + base_ns  
- *
- * The tuple {base_tsc, base_ns, scale, shift} is exported in the commpage 
- * for the userspace nanotime routine to read. A duplicate check_tsc is
- * appended so that the consistency of the read can be verified. Note that
- * this scheme is essential for MP systems in which the commpage is updated
- * by the master cpu but may be read concurrently by other cpus.
- * 
+ * All of the routines which update the nanotime data are non-reentrant.  This must
+ * be guaranteed by the caller.
  */
 static inline void
 rtc_nanotime_set_commpage(rtc_nanotime_t *rntp)
 {
-	commpage_nanotime_t	cp_nanotime;
-
-	/* Only the master cpu updates the commpage */
-	if (cpu_number() != master_cpu)
-		return;
+	commpage_set_nanotime(rntp->tsc_base, rntp->ns_base, rntp->scale, rntp->shift);
+}
 
-	cp_nanotime.nt_base_tsc = rntp->rnt_tsc;
-	cp_nanotime.nt_base_ns = rntp->rnt_nanos;
-	cp_nanotime.nt_scale = rntp->rnt_scale;
-	cp_nanotime.nt_shift = rntp->rnt_shift;
+/*
+ * rtc_nanotime_init:
+ *
+ * Intialize the nanotime info from the base time.  Since
+ * the base value might be from a lower resolution clock,
+ * we compare it to the TSC derived value, and use the
+ * greater of the two values.
+ */
+static inline void
+_rtc_nanotime_init(rtc_nanotime_t *rntp, uint64_t base)
+{
+	uint64_t	nsecs, tsc = rdtsc64();
 
-	commpage_set_nanotime(&cp_nanotime);
+	nsecs = _tsc_to_nanoseconds(tsc);
+	rtc_nanotime_store(tsc, MAX(nsecs, base), rntp->scale, rntp->shift, rntp);
 }
 
 static void
-rtc_nanotime_init(void)
+rtc_nanotime_init(uint64_t base)
 {
-	rtc_nanotime_t	*rntp = &current_cpu_datap()->cpu_rtc_nanotime;
-	rtc_nanotime_t	*master_rntp = &cpu_datap(master_cpu)->cpu_rtc_nanotime;
+	rtc_nanotime_t	*rntp = &rtc_nanotime_info;
 
-	if (cpu_number() == master_cpu) {
-		rntp->rnt_tsc = rdtsc64();
-		rntp->rnt_nanos = tsc_to_nanoseconds(rntp->rnt_tsc);
-		rntp->rnt_scale = rtc_quant_scale;
-		rntp->rnt_shift = rtc_quant_shift;
-		rntp->rnt_step_tsc = 0ULL;
-		rntp->rnt_step_nanos = 0ULL;
-	} else {
-		/*
-		 * Copy master processor's nanotime info.
-		 * Loop required in case this changes while copying.
-		 */
-		do {
-			*rntp = *master_rntp;
-		} while (rntp->rnt_tsc != master_rntp->rnt_tsc);
-	}
+	_rtc_nanotime_init(rntp, base);
+	rtc_nanotime_set_commpage(rntp);
 }
 
-static inline void
-_rtc_nanotime_update(rtc_nanotime_t *rntp, uint64_t	tsc)
+/*
+ * rtc_nanotime_init:
+ *
+ * Call back from the commpage initialization to
+ * cause the commpage data to be filled in once the
+ * commpages have been created.
+ */
+void
+rtc_nanotime_init_commpage(void)
 {
-	uint64_t	tsc_delta;
-	uint64_t	ns_delta;
+	spl_t			s = splclock();
 
-	tsc_delta = tsc - rntp->rnt_step_tsc;
-	ns_delta = tsc_to_nanoseconds(tsc_delta);
-	rntp->rnt_nanos = rntp->rnt_step_nanos + ns_delta;
-	rntp->rnt_tsc = tsc;
+	rtc_nanotime_set_commpage(&rtc_nanotime_info);
+
+	splx(s);
 }
 
-static void
-rtc_nanotime_update(void)
+/*
+ * rtc_nanotime_update:
+ *
+ * Update the nanotime info from the base time.  Since
+ * the base value might be from a lower resolution clock,
+ * we compare it to the TSC derived value, and use the
+ * greater of the two values.
+ *
+ * N.B. In comparison to the above init routine, this assumes
+ * that the TSC has remained monotonic compared to the tsc_base
+ * value, which is not the case after S3 sleep.
+ */
+static inline void
+_rtc_nanotime_update(rtc_nanotime_t *rntp, uint64_t	base)
 {
-	rtc_nanotime_t	*rntp = &current_cpu_datap()->cpu_rtc_nanotime;
+	uint64_t	nsecs, tsc = rdtsc64();
 
-	assert(get_preemption_level() > 0);
-	assert(!ml_get_interrupts_enabled());
-        
-	_rtc_nanotime_update(rntp, rdtsc64());
-	rtc_nanotime_set_commpage(rntp);
+	nsecs = rntp->ns_base + _tsc_to_nanoseconds(tsc - rntp->tsc_base);
+	rtc_nanotime_store(tsc, MAX(nsecs, base), rntp->scale, rntp->shift, rntp);
 }
 
 static void
-rtc_nanotime_scale_update(void)
+rtc_nanotime_update(
+	uint64_t		base)
 {
-	rtc_nanotime_t	*rntp = &current_cpu_datap()->cpu_rtc_nanotime;
-	uint64_t	tsc = rdtsc64();
+	rtc_nanotime_t	*rntp = &rtc_nanotime_info;
 
 	assert(!ml_get_interrupts_enabled());
         
-	/*
-	 * Update time based on past scale.
-	 */
-	_rtc_nanotime_update(rntp, tsc);
-
-	/*
-	 * Update scale and timestamp this update.
-	 */
-	rntp->rnt_scale = rtc_quant_scale;
-	rntp->rnt_shift = rtc_quant_shift;
-	rntp->rnt_step_tsc = rntp->rnt_tsc;
-	rntp->rnt_step_nanos = rntp->rnt_nanos;
-
-	/* Export update to userland */
+	_rtc_nanotime_update(rntp, base);
 	rtc_nanotime_set_commpage(rntp);
 }
 
+/*
+ * rtc_nanotime_read:
+ *
+ * Returns the current nanotime value, accessable from any
+ * context.
+ */
 static uint64_t
-_rtc_nanotime_read(void)
-{
-	rtc_nanotime_t	*rntp = &current_cpu_datap()->cpu_rtc_nanotime;
-	uint64_t	rnt_tsc;
-	uint32_t	rnt_scale;
-	uint32_t	rnt_shift;
-	uint64_t	rnt_nanos;
-	uint64_t	tsc;
-	uint64_t	tsc_delta;
-
-	rnt_scale = rntp->rnt_scale;
-	if (rnt_scale == 0)
-		return 0ULL;
-
-	rnt_shift = rntp->rnt_shift;
-	rnt_nanos = rntp->rnt_nanos;
-	rnt_tsc = rntp->rnt_tsc;
-	tsc = rdtsc64();
-
-	tsc_delta = tsc - rnt_tsc;
-	if ((tsc_delta >> 32) != 0)
-		return rnt_nanos + tsc_to_nanoseconds(tsc_delta);
-
-	/* Let the compiler optimize(?): */
-	if (rnt_shift == 32)
-		return rnt_nanos + ((tsc_delta * rnt_scale) >> 32);	
-	else 
-		return rnt_nanos + ((tsc_delta * rnt_scale) >> rnt_shift);
-}
-
-uint64_t
 rtc_nanotime_read(void)
 {
-	uint64_t	result;
-	uint64_t	rnt_tsc;
-	rtc_nanotime_t	*rntp = &current_cpu_datap()->cpu_rtc_nanotime;
+	rtc_nanotime_t	rnt, *rntp = &rtc_nanotime_info;
+	uint64_t		result;
 
-	/*
-	 * Use timestamp to ensure the uptime record isn't changed.
-	 * This avoids disabling interrupts.
-	 * And not this is a per-cpu structure hence no locking.
-	 */
 	do {
-		rnt_tsc = rntp->rnt_tsc;
-		result = _rtc_nanotime_read();
-	} while (rnt_tsc != rntp->rnt_tsc);
+		rtc_nanotime_load(rntp, &rnt);
+		result = rnt.ns_base + _tsc_to_nanoseconds(rdtsc64() - rnt.tsc_base);
+	} while (rntp->tsc_base != rnt.tsc_base);
 
-	return result;
+	return (result);
 }
 
-
 /*
- * This function is called by the speed-step driver when a
- * change of cpu clock frequency is about to occur.
- * The scale is not changed until rtc_clock_stepped() is called.
- * Between these times there is an uncertainty is exactly when
- * the change takes effect. FIXME: by using another timing source
- * we could eliminate this error.
+ * rtc_clock_napped:
+ *
+ * Invoked from power manangement when we have awoken from a nap (C3/C4)
+ * during which the TSC lost counts.  The nanotime data is updated according
+ * to the provided nanosecond base value.
+ *
+ * The caller must guarantee non-reentrancy.
  */
+void
+rtc_clock_napped(
+	uint64_t		base)
+{
+	rtc_nanotime_update(base);
+}
+
 void
 rtc_clock_stepping(__unused uint32_t new_frequency,
 		   __unused uint32_t old_frequency)
 {
-	boolean_t	istate;
-
-	istate = ml_set_interrupts_enabled(FALSE);
-	rtc_nanotime_scale_update();
-	ml_set_interrupts_enabled(istate);
+	panic("rtc_clock_stepping unsupported");
 }
 
-/*
- * This function is called by the speed-step driver when a
- * change of cpu clock frequency has just occured. This change
- * is expressed as a ratio relative to the boot clock rate.
- */
 void
-rtc_clock_stepped(uint32_t new_frequency, uint32_t old_frequency)
+rtc_clock_stepped(__unused uint32_t new_frequency,
+		  __unused uint32_t old_frequency)
 {
-	boolean_t	istate;
-
-	istate = ml_set_interrupts_enabled(FALSE);
-	if (rtc_boot_frequency == 0) {
-		/*
-		 * At the first ever stepping, old frequency is the real
-		 * initial clock rate. This step and all others are based
-		 * relative to this initial frequency at which the tsc
-		 * calibration was made. Hence we must remember this base
-		 * frequency as reference.
-		 */
-		rtc_boot_frequency = old_frequency;
-	}
-	rtc_set_cyc_per_sec(rtc_cycle_count * new_frequency /
-				rtc_boot_frequency);
-	rtc_nanotime_scale_update();
-	ml_set_interrupts_enabled(istate);
+	panic("rtc_clock_stepping unsupported");
 }
 
 /*
- * rtc_sleep_wakeup() is called from acpi on awakening from a S3 sleep
+ * rtc_sleep_wakeup:
+ *
+ * Invoked from power manageent when we have awoken from a sleep (S3)
+ * and the TSC has been reset.  The nanotime data is updated based on
+ * the HPET value.
+ *
+ * The caller must guarantee non-reentrancy.
  */
 void
 rtc_sleep_wakeup(void)
 {
-	rtc_nanotime_t	*rntp = &current_cpu_datap()->cpu_rtc_nanotime;
-
-	boolean_t	istate;
+	boolean_t		istate;
 
 	istate = ml_set_interrupts_enabled(FALSE);
 
@@ -859,12 +359,8 @@ rtc_sleep_wakeup(void)
 	 * Reset nanotime.
 	 * The timestamp counter will have been reset
 	 * but nanotime (uptime) marches onward.
-	 * We assume that we're still at the former cpu frequency.
 	 */
-	rntp->rnt_tsc = rdtsc64();
-	rntp->rnt_step_tsc = 0ULL;
-	rntp->rnt_step_nanos = rntp->rnt_nanos;
-	rtc_nanotime_set_commpage(rntp);
+	rtc_nanotime_init(tmrCvt(rdHPET(), hpetCvtt2n));
 
 	/* Restart tick interrupts from the LAPIC timer */
 	rtc_lapic_start_ticking();
@@ -877,100 +373,68 @@ rtc_sleep_wakeup(void)
  * In addition, various variables used to support the clock are initialized.
  */
 int
-sysclk_init(void)
+rtclock_init(void)
 {
 	uint64_t	cycles;
 
-	mp_disable_preemption();
+	assert(!ml_get_interrupts_enabled());
+
 	if (cpu_number() == master_cpu) {
+
+		assert(tscFreq);
+		rtc_set_timescale(tscFreq);
+
 		/*
-		 * Perform calibration.
-		 * The PIT is used as the reference to compute how many
-		 * TCS counts (cpu clock cycles) occur per second.
+		 * Adjust and set the exported cpu speed.
 		 */
-        	rtc_cycle_count = timeRDTSC();
-		cycles = rtc_set_cyc_per_sec(rtc_cycle_count);
+		cycles = rtc_export_speed(tscFreq);
 
 		/*
 		 * Set min/max to actual.
 		 * ACPI may update these later if speed-stepping is detected.
 		 */
-        	gPEClockFrequencyInfo.cpu_frequency_min_hz = cycles;
-        	gPEClockFrequencyInfo.cpu_frequency_max_hz = cycles;
-		printf("[RTCLOCK] frequency %llu (%llu)\n",
-		       cycles, rtc_cyc_per_sec);
+		gPEClockFrequencyInfo.cpu_frequency_min_hz = cycles;
+		gPEClockFrequencyInfo.cpu_frequency_max_hz = cycles;
 
-		rtc_lapic_timer_calibrate();
+		/*
+		 * Compute the longest interval we can represent.
+		 */
+		maxDec = tmrCvt(0x7fffffffULL, busFCvtt2n);
+		kprintf("maxDec: %lld\n", maxDec);
 
 		/* Minimum interval is 1usec */
-		rtc_decrementer_min = deadline_to_decrementer(NSEC_PER_USEC,
-								0ULL);
+		rtc_decrementer_min = deadline_to_decrementer(NSEC_PER_USEC, 0ULL);
 		/* Point LAPIC interrupts to hardclock() */
 		lapic_set_timer_func((i386_intr_func_t) rtclock_intr);
 
 		clock_timebase_init();
-		rtc_initialized = TRUE;
+		ml_init_lock_timeout();
 	}
 
-	rtc_nanotime_init();
-
 	rtc_lapic_start_ticking();
 
-	mp_enable_preemption();
-
 	return (1);
 }
 
-/*
- * Get the clock device time. This routine is responsible
- * for converting the device's machine dependent time value
- * into a canonical mach_timespec_t value.
- */
-static kern_return_t
-sysclk_gettime_internal(
-	mach_timespec_t	*cur_time)	/* OUT */
-{
-	*cur_time = tsc_to_timespec();
-	return (KERN_SUCCESS);
-}
+// utility routine 
+// Code to calculate how many processor cycles are in a second...
 
-kern_return_t
-sysclk_gettime(
-	mach_timespec_t	*cur_time)	/* OUT */
+static void
+rtc_set_timescale(uint64_t cycles)
 {
-	return sysclk_gettime_internal(cur_time);
-}
+	rtc_nanotime_info.scale = ((uint64_t)NSEC_PER_SEC << 32) / cycles;
+	rtc_nanotime_info.shift = 32;
 
-void
-sysclk_gettime_interrupts_disabled(
-	mach_timespec_t	*cur_time)	/* OUT */
-{
-	(void) sysclk_gettime_internal(cur_time);
+	rtc_nanotime_init(0);
 }
 
-// utility routine 
-// Code to calculate how many processor cycles are in a second...
-
 static uint64_t
-rtc_set_cyc_per_sec(uint64_t cycles)
+rtc_export_speed(uint64_t cyc_per_sec)
 {
+	uint64_t	cycles;
 
-        if (cycles > (NSEC_PER_SEC/20)) {
-            // we can use just a "fast" multiply to get nanos
-	    rtc_quant_shift = 32;
-            rtc_quant_scale = create_mul_quant_GHZ(rtc_quant_shift, cycles);
-            rtclock.timebase_const.numer = rtc_quant_scale; // timeRDTSC is 1/20
-	    rtclock.timebase_const.denom = RTC_FAST_DENOM;
-        } else {
-	    rtc_quant_shift = 26;
-            rtc_quant_scale = create_mul_quant_GHZ(rtc_quant_shift, cycles);
-            rtclock.timebase_const.numer = NSEC_PER_SEC/20; // timeRDTSC is 1/20
-            rtclock.timebase_const.denom = cycles;
-        }
-	rtc_cyc_per_sec = cycles*20;	// multiply it by 20 and we are done..
-					// BUT we also want to calculate...
-
-        cycles = ((rtc_cyc_per_sec + (UI_CPUFREQ_ROUNDING_FACTOR/2))
+	/* Round: */
+        cycles = ((cyc_per_sec + (UI_CPUFREQ_ROUNDING_FACTOR/2))
 			/ UI_CPUFREQ_ROUNDING_FACTOR)
 				* UI_CPUFREQ_ROUNDING_FACTOR;
 
@@ -984,7 +448,7 @@ rtc_set_cyc_per_sec(uint64_t cycles)
         }
         gPEClockFrequencyInfo.cpu_frequency_hz = cycles;
 
-	kprintf("[RTCLOCK] frequency %llu (%llu)\n", cycles, rtc_cyc_per_sec);
+	kprintf("[RTCLOCK] frequency %llu (%llu)\n", cycles, cyc_per_sec);
 	return(cycles);
 }
 
@@ -993,12 +457,17 @@ clock_get_system_microtime(
 	uint32_t			*secs,
 	uint32_t			*microsecs)
 {
-	mach_timespec_t		now;
-
-	(void) sysclk_gettime_internal(&now);
+	uint64_t	now = rtc_nanotime_read();
+	uint32_t	remain;
 
-	*secs = now.tv_sec;
-	*microsecs = now.tv_nsec / NSEC_PER_USEC;
+	asm volatile(
+			"divl %3"
+				: "=a" (*secs), "=d" (remain)
+				: "A" (now), "r" (NSEC_PER_SEC));
+	asm volatile(
+			"divl %3"
+				: "=a" (*microsecs)
+				: "0" (remain), "d" (0), "r" (NSEC_PER_USEC));
 }
 
 void
@@ -1006,291 +475,39 @@ clock_get_system_nanotime(
 	uint32_t			*secs,
 	uint32_t			*nanosecs)
 {
-	mach_timespec_t		now;
-
-	(void) sysclk_gettime_internal(&now);
-
-	*secs = now.tv_sec;
-	*nanosecs = now.tv_nsec;
-}
-
-/*
- * Get clock device attributes.
- */
-kern_return_t
-sysclk_getattr(
-	clock_flavor_t		flavor,
-	clock_attr_t		attr,		/* OUT */
-	mach_msg_type_number_t	*count)		/* IN/OUT */
-{
-	if (*count != 1)
-		return (KERN_FAILURE);
-	switch (flavor) {
-
-	case CLOCK_GET_TIME_RES:	/* >0 res */
-		*(clock_res_t *) attr = rtc_intr_nsec;
-		break;
-
-	case CLOCK_ALARM_CURRES:	/* =0 no alarm */
-	case CLOCK_ALARM_MAXRES:
-	case CLOCK_ALARM_MINRES:
-		*(clock_res_t *) attr = 0;
-		break;
-
-	default:
-		return (KERN_INVALID_VALUE);
-	}
-	return (KERN_SUCCESS);
-}
-
-/*
- * Set next alarm time for the clock device. This call
- * always resets the time to deliver an alarm for the
- * clock.
- */
-void
-sysclk_setalarm(
-	mach_timespec_t	*alarm_time)
-{
-	timer_call_enter(&rtclock_alarm_timer,
-			 (uint64_t) alarm_time->tv_sec * NSEC_PER_SEC
-				+ alarm_time->tv_nsec);
-}
-
-/*
- * Configure the calendar clock.
- */
-int
-calend_config(void)
-{
-	return bbc_config();
-}
-
-/*
- * Initialize calendar clock.
- */
-int
-calend_init(void)
-{
-	return (1);
-}
-
-/*
- * Get the current clock time.
- */
-kern_return_t
-calend_gettime(
-	mach_timespec_t	*cur_time)	/* OUT */
-{
-	spl_t		s;
-
-	RTC_LOCK(s);
-	if (!rtclock.calend_is_set) {
-		RTC_UNLOCK(s);
-		return (KERN_FAILURE);
-	}
-
-	(void) sysclk_gettime_internal(cur_time);
-	ADD_MACH_TIMESPEC(cur_time, &rtclock.calend_offset);
-	RTC_UNLOCK(s);
-
-	return (KERN_SUCCESS);
-}
-
-void
-clock_get_calendar_microtime(
-	uint32_t			*secs,
-	uint32_t			*microsecs)
-{
-	mach_timespec_t		now;
-
-	calend_gettime(&now);
-
-	*secs = now.tv_sec;
-	*microsecs = now.tv_nsec / NSEC_PER_USEC;
-}
-
-void
-clock_get_calendar_nanotime(
-	uint32_t			*secs,
-	uint32_t			*nanosecs)
-{
-	mach_timespec_t		now;
+	uint64_t	now = rtc_nanotime_read();
 
-	calend_gettime(&now);
-
-	*secs = now.tv_sec;
-	*nanosecs = now.tv_nsec;
+	asm volatile(
+			"divl %3"
+				: "=a" (*secs), "=d" (*nanosecs)
+				: "A" (now), "r" (NSEC_PER_SEC));
 }
 
 void
-clock_set_calendar_microtime(
-	uint32_t			secs,
-	uint32_t			microsecs)
-{
-	mach_timespec_t		new_time, curr_time;
-	uint32_t			old_offset;
-	spl_t		s;
-
-	new_time.tv_sec = secs;
-	new_time.tv_nsec = microsecs * NSEC_PER_USEC;
-
-	RTC_LOCK(s);
-	old_offset = rtclock.calend_offset.tv_sec;
-	(void) sysclk_gettime_internal(&curr_time);
-	rtclock.calend_offset = new_time;
-	SUB_MACH_TIMESPEC(&rtclock.calend_offset, &curr_time);
-	rtclock.boottime += rtclock.calend_offset.tv_sec - old_offset;
-	rtclock.calend_is_set = TRUE;
-	RTC_UNLOCK(s);
-
-	(void) bbc_settime(&new_time);
-
-	host_notify_calendar_change();
-}
-
-/*
- * Get clock device attributes.
- */
-kern_return_t
-calend_getattr(
-	clock_flavor_t		flavor,
-	clock_attr_t		attr,		/* OUT */
-	mach_msg_type_number_t	*count)		/* IN/OUT */
-{
-	if (*count != 1)
-		return (KERN_FAILURE);
-	switch (flavor) {
-
-	case CLOCK_GET_TIME_RES:	/* >0 res */
-		*(clock_res_t *) attr = rtc_intr_nsec;
-		break;
-
-	case CLOCK_ALARM_CURRES:	/* =0 no alarm */
-	case CLOCK_ALARM_MINRES:
-	case CLOCK_ALARM_MAXRES:
-		*(clock_res_t *) attr = 0;
-		break;
-
-	default:
-		return (KERN_INVALID_VALUE);
-	}
-	return (KERN_SUCCESS);
-}
-
-#define tickadj		(40*NSEC_PER_USEC)	/* "standard" skew, ns / tick */
-#define	bigadj		(NSEC_PER_SEC)		/* use 10x skew above bigadj ns */
-
-uint32_t
-clock_set_calendar_adjtime(
-	int32_t				*secs,
-	int32_t				*microsecs)
-{
-	int64_t			total, ototal;
-	uint32_t		interval = 0;
-	spl_t			s;
-
-	total = (int64_t)*secs * NSEC_PER_SEC + *microsecs * NSEC_PER_USEC;
-
-	RTC_LOCK(s);
-	ototal = rtclock.calend_adjtotal;
-
-	if (total != 0) {
-		int32_t		delta = tickadj;
-
-		if (total > 0) {
-			if (total > bigadj)
-				delta *= 10;
-			if (delta > total)
-				delta = total;
-		}
-		else {
-			if (total < -bigadj)
-				delta *= 10;
-			delta = -delta;
-			if (delta < total)
-				delta = total;
-		}
-
-		rtclock.calend_adjtotal = total;
-		rtclock.calend_adjdelta = delta;
-
-		interval = NSEC_PER_HZ;
-	}
-	else
-		rtclock.calend_adjdelta = rtclock.calend_adjtotal = 0;
-
-	RTC_UNLOCK(s);
-
-	if (ototal == 0)
-		*secs = *microsecs = 0;
-	else {
-		*secs = ototal / NSEC_PER_SEC;
-		*microsecs = ototal % NSEC_PER_SEC;
-	}
-
-	return (interval);
-}
-
-uint32_t
-clock_adjust_calendar(void)
-{
-	uint32_t		interval = 0;
-	int32_t			delta;
-	spl_t			s;
-
-	RTC_LOCK(s);
-	delta = rtclock.calend_adjdelta;
-	ADD_MACH_TIMESPEC_NSEC(&rtclock.calend_offset, delta);
-
-	rtclock.calend_adjtotal -= delta;
-
-	if (delta > 0) {
-		if (delta > rtclock.calend_adjtotal)
-			rtclock.calend_adjdelta = rtclock.calend_adjtotal;
-	}
-	else
-	if (delta < 0) {
-		if (delta < rtclock.calend_adjtotal)
-			rtclock.calend_adjdelta = rtclock.calend_adjtotal;
-	}
-
-	if (rtclock.calend_adjdelta != 0)
-		interval = NSEC_PER_HZ;
+clock_gettimeofday_set_commpage(
+	uint64_t				abstime,
+	uint64_t				epoch,
+	uint64_t				offset,
+	uint32_t				*secs,
+	uint32_t				*microsecs)
+{
+	uint64_t	now = abstime;
+	uint32_t	remain;
 
-	RTC_UNLOCK(s);
+	now += offset;
 
-	return (interval);
-}
+	asm volatile(
+			"divl %3"
+				: "=a" (*secs), "=d" (remain)
+				: "A" (now), "r" (NSEC_PER_SEC));
+	asm volatile(
+			"divl %3"
+				: "=a" (*microsecs)
+				: "0" (remain), "d" (0), "r" (NSEC_PER_USEC));
 
-void
-clock_initialize_calendar(void)
-{
-	mach_timespec_t	bbc_time, curr_time;
-	spl_t		s;
-
-	if (bbc_gettime(&bbc_time) != KERN_SUCCESS)
-		return;
-
-	RTC_LOCK(s);
-	if (rtclock.boottime == 0)
-		rtclock.boottime = bbc_time.tv_sec;
-	(void) sysclk_gettime_internal(&curr_time);
-	rtclock.calend_offset = bbc_time;
-	SUB_MACH_TIMESPEC(&rtclock.calend_offset, &curr_time);
-	rtclock.calend_is_set = TRUE;
-	RTC_UNLOCK(s);
-
-	host_notify_calendar_change();
-}
+	*secs += epoch;
 
-void
-clock_get_boottime_nanotime(
-	uint32_t			*secs,
-	uint32_t			*nanosecs)
-{
-	*secs = rtclock.boottime;
-	*nanosecs = 0;
+	commpage_set_timestamp(abstime - remain, *secs, NSEC_PER_SEC);
 }
 
 void
@@ -1300,37 +517,6 @@ clock_timebase_info(
 	info->numer = info->denom =  1;
 }	
 
-void
-clock_set_timer_deadline(
-	uint64_t			deadline)
-{
-	spl_t		s;
-	cpu_data_t	*pp = current_cpu_datap();
-	rtclock_timer_t	*mytimer = &pp->cpu_rtc_timer;
-	uint64_t	abstime;
-	uint64_t	decr;
-
-	assert(get_preemption_level() > 0);
-	assert(rtclock_timer_expire);
-
-	RTC_INTRS_OFF(s);
-	mytimer->deadline = deadline;
-	mytimer->is_set = TRUE;
-	if (!mytimer->has_expired) {
-		abstime = mach_absolute_time();
-		if (mytimer->deadline < pp->cpu_rtc_tick_deadline) {
-			decr = deadline_to_decrementer(mytimer->deadline,
-						       abstime);
-			rtc_lapic_set_timer(decr);
-			pp->cpu_rtc_intr_deadline = mytimer->deadline;
-			KERNEL_DEBUG_CONSTANT(
-				MACHDBG_CODE(DBG_MACH_EXCP_DECI, 1) |
-					DBG_FUNC_NONE, decr, 2, 0, 0, 0);
-		}
-	}
-	RTC_INTRS_ON(s);
-}
-
 void
 clock_set_timer_func(
 	clock_timer_func_t		func)
@@ -1343,87 +529,99 @@ clock_set_timer_func(
  * Real-time clock device interrupt.
  */
 void
-rtclock_intr(struct i386_interrupt_state *regs)
+rtclock_intr(
+	x86_saved_state_t	*tregs)
 {
+        uint64_t	rip;
+	boolean_t	user_mode = FALSE;
 	uint64_t	abstime;
 	uint32_t	latency;
-	uint64_t	decr;
-	uint64_t	decr_tick;
-	uint64_t	decr_timer;
 	cpu_data_t	*pp = current_cpu_datap();
-	rtclock_timer_t	*mytimer = &pp->cpu_rtc_timer;
 
 	assert(get_preemption_level() > 0);
 	assert(!ml_get_interrupts_enabled());
 
-        abstime = _rtc_nanotime_read();
-	latency = (uint32_t) abstime - pp->cpu_rtc_intr_deadline;
-	if (pp->cpu_rtc_tick_deadline <= abstime) {
-		rtc_nanotime_update();
-		clock_deadline_for_periodic_event(
-			NSEC_PER_HZ, abstime, &pp->cpu_rtc_tick_deadline);
-		hertz_tick(
-#if STAT_TIME
-			   NSEC_PER_HZ,
-#endif
-			   (regs->efl & EFL_VM) || ((regs->cs & 0x03) != 0),
-			   regs->eip);
-	}
+	abstime = rtc_nanotime_read();
+	latency = (uint32_t) abstime - pp->rtcPop;
 
-	abstime = _rtc_nanotime_read();
-	if (mytimer->is_set && mytimer->deadline <= abstime) {
-		mytimer->has_expired = TRUE;
-		mytimer->is_set = FALSE;
-		(*rtclock_timer_expire)(abstime);
-		assert(!ml_get_interrupts_enabled());
-		mytimer->has_expired = FALSE;
-	}
+	if (is_saved_state64(tregs) == TRUE) {
+	        x86_saved_state64_t	*regs;
+		  
+		regs = saved_state64(tregs);
 
-	/* Log the interrupt service latency (-ve value expected by tool) */
-	KERNEL_DEBUG_CONSTANT(
-		MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE,
-		-latency, (uint32_t)regs->eip, 0, 0, 0);
+		user_mode = TRUE;
+		rip = regs->isf.rip;
+	} else {
+	        x86_saved_state32_t	*regs;
 
-	abstime = _rtc_nanotime_read();
-	decr_tick = deadline_to_decrementer(pp->cpu_rtc_tick_deadline, abstime);
-	decr_timer = (mytimer->is_set) ?
-			deadline_to_decrementer(mytimer->deadline, abstime) :
-			DECREMENTER_MAX;
-	decr = MIN(decr_tick, decr_timer);
-	pp->cpu_rtc_intr_deadline = abstime + decr;
+		regs = saved_state32(tregs);
 
-	rtc_lapic_set_timer(decr);
+		if (regs->cs & 0x03)
+		        user_mode = TRUE;
+		rip = regs->eip;
+	}
 
-	/* Log the new decrementer value */
+	/* Log the interrupt service latency (-ve value expected by tool) */
 	KERNEL_DEBUG_CONSTANT(
-		MACHDBG_CODE(DBG_MACH_EXCP_DECI, 1) | DBG_FUNC_NONE,
-		decr, 3, 0, 0, 0);
+		MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE,
+		-latency, (uint32_t)rip, user_mode, 0, 0);
 
+	/* call the generic etimer */
+	etimer_intr(user_mode, rip);
 }
 
-static void
-rtclock_alarm_expire(
-	__unused timer_call_param_t	p0,
-	__unused timer_call_param_t	p1)
+/*
+ *	Request timer pop from the hardware 
+ */
+
+int
+setPop(
+	uint64_t time)
 {
-	mach_timespec_t	clock_time;
+	uint64_t now;
+	uint32_t decr;
+	uint64_t count;
+	
+	now = rtc_nanotime_read();		/* The time in nanoseconds */
+	decr = deadline_to_decrementer(time, now);
 
-	(void) sysclk_gettime_internal(&clock_time);
+	count = tmrCvt(decr, busFCvtn2t);
+	lapic_set_timer(TRUE, one_shot, divide_by_1, (uint32_t) count);
 
-	clock_alarm_intr(SYSTEM_CLOCK, &clock_time);
+	return decr;				/* Pass back what we set */
 }
 
+
 void
-clock_get_uptime(
-	uint64_t		*result)
+resetPop(void)
 {
-        *result = rtc_nanotime_read();
+	uint64_t	now;
+	uint32_t	decr;
+	uint64_t	count;
+	cpu_data_t	*cdp = current_cpu_datap();
+
+	now = rtc_nanotime_read();
+
+	decr = deadline_to_decrementer(cdp->rtcPop, now);
+
+	count = tmrCvt(decr, busFCvtn2t);
+	lapic_set_timer(TRUE, one_shot, divide_by_1, (uint32_t)count);
 }
 
+
 uint64_t
 mach_absolute_time(void)
 {
-        return rtc_nanotime_read();
+	return rtc_nanotime_read();
+}
+
+void
+clock_interval_to_absolutetime_interval(
+	uint32_t		interval,
+	uint32_t		scale_factor,
+	uint64_t		*result)
+{
+	*result = (uint64_t)interval * scale_factor;
 }
 
 void
@@ -1445,37 +643,24 @@ absolutetime_to_microtime(
 }
 
 void
-clock_interval_to_deadline(
-	uint32_t		interval,
-	uint32_t		scale_factor,
-	uint64_t		*result)
-{
-	uint64_t		abstime;
-
-	clock_get_uptime(result);
-
-	clock_interval_to_absolutetime_interval(interval, scale_factor, &abstime);
-
-	*result += abstime;
-}
-
-void
-clock_interval_to_absolutetime_interval(
-	uint32_t		interval,
-	uint32_t		scale_factor,
-	uint64_t		*result)
+absolutetime_to_nanotime(
+	uint64_t			abstime,
+	uint32_t			*secs,
+	uint32_t			*nanosecs)
 {
-	*result = (uint64_t)interval * scale_factor;
+	asm volatile(
+			"divl %3"
+			: "=a" (*secs), "=d" (*nanosecs)
+			: "A" (abstime), "r" (NSEC_PER_SEC));
 }
 
 void
-clock_absolutetime_interval_to_deadline(
-	uint64_t		abstime,
-	uint64_t		*result)
+nanotime_to_absolutetime(
+	uint32_t			secs,
+	uint32_t			nanosecs,
+	uint64_t			*result)
 {
-	clock_get_uptime(result);
-
-	*result += abstime;
+	*result = ((uint64_t)secs * NSEC_PER_SEC) + nanosecs;
 }
 
 void
diff --git a/iokit/Drivers/platform/drvAppleIntelClock/IntelClock.cpp b/osfmk/i386/rtclock.h
similarity index 71%
rename from iokit/Drivers/platform/drvAppleIntelClock/IntelClock.cpp
rename to osfmk/i386/rtclock.h
index b38defb9c..7bee1e8e3 100644
--- a/iokit/Drivers/platform/drvAppleIntelClock/IntelClock.cpp
+++ b/osfmk/i386/rtclock.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -25,26 +25,25 @@
  * 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-//
-// Backdoor hack for Intel Clock.
-//
-//
+/*
+ * @OSF_COPYRIGHT@
+ */
+/*
+ * @APPLE_FREE_COPYRIGHT@
+ */
+/*
+ *	File:		rtclock.h
+ *	Purpose:	Routines for handling the machine dependent
+ *				real-time clock.
+ */
+
+#ifndef _I386_RTCLOCK_H_
+#define _I386_RTCLOCK_H_
 
-#include "AppleIntelClock.h"
+#include <kern/etimer.h>
 
-#define super IOService
-OSDefineMetaClassAndStructors(AppleIntelClock, IOService);
+void resetPop(void);
 
-bool
-AppleIntelClock::start(IOService *provider)
-{
-	if (!super::start(provider))
-		return false;
+struct cpu_data;
 
-	/*
-	 * The clock is already provided by the kernel, so all we need 
-	 * here is publish its availability for any IOKit client to use.
-	 */
-	publishResource("IORTC", this);
-	return true;
-}
+#endif /* _I386_RTCLOCK_H_ */
diff --git a/osfmk/i386/seg.h b/osfmk/i386/seg.h
index 7475af199..7083298df 100644
--- a/osfmk/i386/seg.h
+++ b/osfmk/i386/seg.h
@@ -61,6 +61,7 @@
 
 #include <mach_kdb.h>
 #include <stdint.h>
+#include <mach/vm_types.h>
 #include <architecture/i386/sel.h>
 
 /*
@@ -93,21 +94,14 @@ selector_to_sel(uint16_t selector)
     return (tconv.sel);
 }
 
-#define	LDTSZ		15		/* size of the kernel ldt in entries*/
+#define LDTSZ		8192		/* size of the kernel ldt in entries */
+#define	LDTSZ_MIN	17		/* kernel ldt entries used by the system */
 
 #if	MACH_KDB
-#ifdef MACH_BSD
-#define	GDTSZ		14
+#define	GDTSZ		19
 #else
-#define	GDTSZ		11
+#define	GDTSZ		18
 #endif
-#else	/* MACH_KDB */
-#ifdef	MACH_BSD
-#define	GDTSZ		13
-#else
-#define	GDTSZ		10
-#endif
-#endif	/* MACH_KDB */
 
 /*
  * Interrupt table is always 256 entries long.
@@ -122,7 +116,7 @@ selector_to_sel(uint16_t selector)
  * Real segment descriptor.
  */
 struct real_descriptor {
-	unsigned int	limit_low:16,	/* limit 0..15 */
+	uint32_t	limit_low:16,	/* limit 0..15 */
 			base_low:16,	/* base  0..15 */
 			base_med:8,	/* base  16..23 */
 			access:8,	/* access byte */
@@ -130,14 +124,34 @@ struct real_descriptor {
 			granularity:4,	/* granularity */
 			base_high:8;	/* base 24..31 */
 };
-
+struct real_descriptor64 {
+	uint32_t	limit_low16:16,	/* limit 0..15 */
+			base_low16:16,	/* base  0..15 */
+			base_med8:8,	/* base  16..23 */
+			access8:8,	/* access byte */
+			limit_high4:4,	/* limit 16..19 */
+			granularity4:4,	/* granularity */
+			base_high8:8,	/* base 24..31 */
+			base_top32:32,	/* base 32..63 */
+			reserved32:32;	/* reserved/zero */
+};
 struct real_gate {
-	unsigned int	offset_low:16,	/* offset 0..15 */
+	uint32_t	offset_low:16,	/* offset 0..15 */
 			selector:16,
 			word_count:8,
 			access:8,
 			offset_high:16;	/* offset 16..31 */
 };
+struct real_gate64 {
+	uint32_t	offset_low16:16,	/* offset 0..15 */
+			selector16:16,
+			IST:3,
+			zeroes5:5,
+			access8:8,
+			offset_high16:16,	/* offset 16..31 */
+			offset_top32:32,	/* offset 32..63 */
+			reserved32:32;		/* reserved/zero */
+};
 
 /*
  * We build descriptors and gates in a 'fake' format to let the
@@ -145,28 +159,56 @@ struct real_gate {
  * at runtime.
  */
 struct fake_descriptor {
-	unsigned int	offset:32;		/* offset */
-	unsigned int	lim_or_seg:20;		/* limit */
+	uint32_t	offset:32;		/* offset */
+	uint32_t	lim_or_seg:20;		/* limit */
 						/* or segment, for gate */
-	unsigned int	size_or_wdct:4;		/* size/granularity */
+	uint32_t	size_or_wdct:4;		/* size/granularity */
 						/* word count, for gate */
-	unsigned int	access:8;		/* access */
+	uint32_t	access:8;		/* access */
 };
+struct fake_descriptor64 {
+	uint32_t	offset[2];		/* offset [0..31,32..63] */
+	uint32_t	lim_or_seg:20;		/* limit */
+						/* or segment, for gate */
+	uint32_t	size_or_IST:4;		/* size/granularity */
+						/* IST for gates */
+	uint32_t	access:8;		/* access */
+	uint32_t	reserved:32;		/* reserved/zero */
+};
+#define	FAKE_UBER64(addr32)	{ (uint32_t) (addr32), KERNEL_UBER_BASE_HI32 }
+#define	FAKE_COMPAT(addr32)	{ (uint32_t) (addr32), 0x0 }
+#define	UBER64(addr32)		((addr64_t) addr32 + KERNEL_UBER_BASE)
 
 /*
  * Boot-time data for master (or only) CPU
  */
-extern struct fake_descriptor	idt[IDTSZ];
-extern struct fake_descriptor	gdt[GDTSZ];
-extern struct fake_descriptor	ldt[LDTSZ];
-extern struct i386_tss		ktss;
+extern struct fake_descriptor	master_idt[IDTSZ];
+extern struct fake_descriptor	master_gdt[GDTSZ];
+extern struct fake_descriptor	master_ldt[LDTSZ];
+extern struct i386_tss		master_ktss;
+extern struct sysenter_stack	master_sstk;
+
+extern struct fake_descriptor64	master_idt64[IDTSZ];
+extern struct fake_descriptor64	kernel_ldt_desc64;
+extern struct fake_descriptor64	kernel_tss_desc64;
+extern struct x86_64_tss	master_ktss64;
 
 __BEGIN_DECLS
 
+extern char			df_task_stack[];
+extern char			df_task_stack_end[];
+extern struct i386_tss		master_dftss;
+extern void			df_task_start(void);
+
+extern char			mc_task_stack[];
+extern char			mc_task_stack_end[];
+extern struct i386_tss		master_mctss;
+extern void			mc_task_start(void);
+
 #if	MACH_KDB
 extern char			db_stack_store[];
 extern char			db_task_stack_store[];
-extern struct i386_tss		dbtss;
+extern struct i386_tss		master_dbtss;
 extern void			db_task_start(void);
 #endif	/* MACH_KDB */
 
@@ -174,6 +216,7 @@ __END_DECLS
 
 #endif	/*__ASSEMBLER__*/
 
+#define	SZ_64		0x2			/* 64-bit segment */
 #define	SZ_32		0x4			/* 32-bit segment */
 #define	SZ_G		0x8			/* 4K limit field */
 
@@ -221,18 +264,27 @@ __END_DECLS
  * Convert selector to descriptor table index.
  */
 #define	sel_idx(sel)	(selector_to_sel(sel).index)
+#define SEL_TO_INDEX(s)	((s)>>3)
 
 #define NULL_SEG	0
 
 /*
  * User descriptors for MACH - 32-bit flat address space
  */
-#define	USER_SCALL	0x07		/* system call gate */
-#define	USER_RPC	0x0f		/* mach rpc call gate */
-#define	USER_CS		0x17		/* user code segment */
-#define	USER_DS		0x1f		/* user data segment */
-#define	USER_CTHREAD	0x27		/* user cthread area */
-#define	USER_SETTABLE	0x2f		/* start of user settable ldt entries */
+#define	SYSENTER_CS	0x07		/* sysenter kernel code segment */
+#define	SYSENTER_DS	0x0f		/* sysenter kernel data segment */
+#define	USER_CS		0x17		/* user code segment
+					   Must be SYSENTER_CS+16 for sysexit */
+/* Special case: sysenter with EFL_TF (trace bit) set - use iret not sysexit */
+#define SYSENTER_TF_CS	(USER_CS|0x10000)
+#define	USER_DS		0x1f		/* user data segment 
+					   Must be SYSENTER_CS+24 for sysexit */
+#define	USER64_CS	0x27		/* 64-bit user code segment 
+					   Must be USER_CS+16 for sysret */
+#define	USER64_DS	USER_DS		/* 64-bit user data segment == 32-bit */
+#define	SYSCALL_CS	0x2f		/* 64-bit syscall pseudo-segment */
+#define	USER_CTHREAD	0x37		/* user cthread area */
+#define	USER_SETTABLE	0x3f		/* start of user settable ldt entries */
 #define	USLDTSZ		10		/* number of user settable entries */
 
 /*
@@ -241,29 +293,41 @@ __END_DECLS
 #define	KERNEL_CS	0x08		/* kernel code */
 #define	KERNEL_DS	0x10		/* kernel data */
 #define	KERNEL_LDT	0x18		/* master LDT */
-#define	KERNEL_TSS	0x20		/* master TSS (uniprocessor) */
-#ifdef	MACH_BSD
-#define	BSD_SCALL_SEL	0x28		/* BSD System calls */
-#define	MK25_SCALL_SEL	0x30		/* MK25 System Calls */
-#define	MACHDEP_SCALL_SEL 0x38		/* Machdep SYstem calls */
-#else
-#define	USER_LDT	0x28		/* place for per-thread LDT */
-#define	USER_TSS	0x30		/* place for per-thread TSS
-					   that holds IO bitmap */
-#define	FPE_CS		0x38		/* floating-point emulator code */
-#endif
-#define	USER_FPREGS	0x40		/* user-mode access to saved
-					   floating-point registers */
+#define	KERNEL_LDT_2	0x20		/* master LDT expanded for 64-bit */
+#define	KERNEL_TSS	0x28		/* master TSS */
+#define	KERNEL_TSS_2	0x30		/* master TSS expanded for 64-bit */
+
+#define	MC_TSS		0x38		/* machine-check handler TSS */
+
 #define	CPU_DATA_GS	0x48		/* per-cpu data */
 
-#ifdef	MACH_BSD
+#define	DF_TSS		0x50		/* double-fault handler TSS */
+
 #define	USER_LDT	0x58
 #define	USER_TSS	0x60
 #define	FPE_CS		0x68
-#endif
+
+#define USER_WINDOW_SEL	0x70		/* window for copyin/copyout */
+#define PHYS_WINDOW_SEL	0x78		/* window for copyin/copyout */
+
+#define	KERNEL64_CS	0x80		/* kernel 64-bit code */
+#define	KERNEL64_SS	0x88		/* kernel 64-bit (syscall) stack */
 
 #if	MACH_KDB
-#define	DEBUG_TSS	0x50		/* debug TSS (uniprocessor) */
+#define	DEBUG_TSS	0x90		/* debug TSS (uniprocessor) */
 #endif
 
+struct __gdt_desc_struct {
+  unsigned short size;
+  unsigned long address __attribute__((packed));
+  unsigned short pad;
+} __attribute__ ((packed));
+
+struct __idt_desc_struct {
+  unsigned short size;
+  unsigned long address __attribute__((packed));
+  unsigned short pad;
+} __attribute__ ((packed));
+
+
 #endif	/* _I386_SEG_H_ */
diff --git a/osfmk/i386/simple_lock.h b/osfmk/i386/simple_lock.h
index 67d99cebf..ad2c33893 100644
--- a/osfmk/i386/simple_lock.h
+++ b/osfmk/i386/simple_lock.h
@@ -100,6 +100,8 @@ typedef struct slock {
 	uslock_debug	debug;
 } usimple_lock_data_t, *usimple_lock_t;
 
+extern void			i386_lock_unlock_with_flush(
+					hw_lock_t);
 #else
 
 typedef	struct slock {
diff --git a/osfmk/i386/start.s b/osfmk/i386/start.s
index 84a96b2e8..08f671378 100644
--- a/osfmk/i386/start.s
+++ b/osfmk/i386/start.s
@@ -68,6 +68,7 @@
 
 #include <i386/mp.h>
 #include <i386/mp_slave_boot.h>
+#include <i386/cpuid.h>
 
 /*
  * GAS won't handle an intersegment jump with a relocatable offset.
@@ -79,38 +80,26 @@
 
 
 
-#define KVTOPHYS	(-KERNELBASE)
-#define	KVTOLINEAR	LINEAR_KERNELBASE
+#define	PA(addr)	(addr)
+#define	VA(addr)	(addr)
 
-
-#define	PA(addr)	((addr)+KVTOPHYS)
-#define	VA(addr)	((addr)-KVTOPHYS)
-
-	.data
-#if 0	/* Anyone need this? */
-	.align 	2
-	.globl	EXT(_kick_buffer_)
-EXT(_kick_buffer_):
-	.long	1
-	.long	3
-	.set	.,.+16836
-#endif	/* XXX */
 /*
  * Interrupt and bootup stack for initial processor.
  */
+
         /* in the __HIB section since the hibernate restore code uses this stack. */
         .section __HIB, __data
-	.align	ALIGN
+	.align	12
 
-	.globl	EXT(intstack)
-EXT(intstack):
+	.globl	EXT(low_intstack)
+EXT(low_intstack):
 	.globl  EXT(gIOHibernateRestoreStack)
 EXT(gIOHibernateRestoreStack):
 
 	.set	., .+INTSTACK_SIZE
 
-	.globl	EXT(eintstack)
-EXT(eintstack:)
+	.globl	EXT(low_eintstack)
+EXT(low_eintstack:)
 	.globl  EXT(gIOHibernateRestoreStackEnd)
 EXT(gIOHibernateRestoreStackEnd):
 
@@ -119,26 +108,51 @@ EXT(gIOHibernateRestoreStackEnd):
  */
 	.align	ALIGN
 	.globl	EXT(gdtptr)
+	/* align below properly */
+	.word   0 
 LEXT(gdtptr)
 	.word	Times(8,GDTSZ)-1
-	.long	EXT(gdt)
+	.long	EXT(master_gdt)
 
 	.align	ALIGN
 	.globl	EXT(idtptr)
+	/* align below properly */
+	.word   0 
 LEXT(idtptr)
 	.word	Times(8,IDTSZ)-1
-	.long	EXT(idt)
+	.long	EXT(master_idt)
 
-          /* back to the regular __DATA section. */
+        /* back to the regular __DATA section. */
 
           .section __DATA, __data
 
+/*
+ * Stack for last-gasp double-fault handler.
+ */
+	.align	12
+	.globl	EXT(df_task_stack)
+EXT(df_task_stack):
+	.set	., .+INTSTACK_SIZE
+	.globl	EXT(df_task_stack_end)
+EXT(df_task_stack_end):
+
+
+/*
+ * Stack for machine-check handler.
+ */
+	.align	12
+	.globl	EXT(mc_task_stack)
+EXT(mc_task_stack):
+	.set	., .+INTSTACK_SIZE
+	.globl	EXT(mc_task_stack_end)
+EXT(mc_task_stack_end):
+
 
 #if	MACH_KDB
 /*
  * Kernel debugger stack for each processor.
  */
-	.align	ALIGN
+	.align	12
 	.globl	EXT(db_stack_store)
 EXT(db_stack_store):
 	.set	., .+(INTSTACK_SIZE*MAX_CPUS)
@@ -146,7 +160,7 @@ EXT(db_stack_store):
 /*
  * Stack for last-ditch debugger task for each processor.
  */
-	.align	ALIGN
+	.align	12
 	.globl	EXT(db_task_stack_store)
 EXT(db_task_stack_store):
 	.set	., .+(INTSTACK_SIZE*MAX_CPUS)
@@ -161,54 +175,46 @@ EXT(kgdb_stack_store):
 #endif	/* MACH_KDB */
 
 	.data
-	/*
-	 *	start_lock is very special.  We initialize the
-	 *	lock at allocation time rather than at run-time.
-	 *	Although start_lock should be an instance of a
-	 *	hw_lock, we hand-code all manipulation of the lock
-	 *	because the hw_lock code may require function calls;
-	 *	and we'd rather not introduce another dependency on
-	 *	a working stack at this point.
-	 */
-	.globl	EXT(start_lock)
-EXT(start_lock):
-	.long	0			/* synchronizes processor startup */
-
-	.globl	EXT(master_is_up)
-EXT(master_is_up):
-	.long	0			/* 1 when OK for other processors */
-					/* to start */
-	.globl	EXT(mp_boot_pde)
-EXT(mp_boot_pde):
-	.long	0
-
-_KERNend:	.long	0			/* phys addr end of kernel (just after bss) */
-physfree:	.long	0			/* phys addr of next free page */
-
-	.globl	_IdlePTD
-_IdlePTD:	.long	0			/* phys addr of kernel PTD */
+physfree:
+	.long	0			/* phys addr of next free page */
+
+	.globl	EXT(IdlePTD)
+EXT(IdlePTD):
+	.long	0			/* phys addr of kernel PTD */
 #ifdef PAE
-	.globl	_IdlePDPT
-_IdlePDPT:	.long 0				/* phys addr of kernel PDPT */
+	.globl	EXT(IdlePDPT)
+EXT(IdlePDPT):
+	.long 0				/* phys addr of kernel PDPT */
+#endif
+#ifdef X86_64
+	.globl EXT(IdlePML4)
+EXT(IdlePML4):
+	.long 0
+	.globl EXT(IdlePDPT64)
+EXT(IdlePDPT64):
+	.long 0
 #endif
 
-	.globl	_KPTphys
-
-_KPTphys:	.long	0			/* phys addr of kernel page tables */
+KPTphys:
+	.long	0			/* phys addr of kernel page tables */
 
+	.globl	EXT(KernelRelocOffset)
+EXT(KernelRelocOffset):
+	.long	0					/* Kernel relocation offset  */
+	
 
 /*   Some handy macros */
 
-#define ALLOCPAGES(npages) \
-	movl	PA(physfree), %esi ; \
-	movl	$((npages) * PAGE_SIZE), %eax ;	\
-	addl	%esi, %eax ; \
-	movl	%eax, PA(physfree) ; \
-	movl	%esi, %edi ; \
-	movl	$((npages) * PAGE_SIZE / 4),%ecx ;	\
-	xorl	%eax,%eax ; \
-	cld ; \
-	rep ; \
+#define ALLOCPAGES(npages)			   \
+	movl	PA(physfree), %esi		 ; \
+	movl	$((npages) * PAGE_SIZE), %eax	 ; \
+	addl	%esi, %eax			 ; \
+	movl	%eax, PA(physfree)		 ; \
+	movl	%esi, %edi			 ; \
+	movl	$((npages) * PAGE_SIZE / 4),%ecx ; \
+	xorl	%eax,%eax			 ; \
+	cld					 ; \
+	rep					 ; \
 	stosl
 
 /*
@@ -220,7 +226,7 @@ _KPTphys:	.long	0			/* phys addr of kernel page tables */
  *	prot = protection bits
  */
 #define	fillkpt(base, prot)		  \
-	shll	$(PTEINDX),%ebx			; \
+	shll	$(PTEINDX),%ebx		; \
 	addl	base,%ebx		; \
 	orl	$(PTE_V) ,%eax		; \
 	orl	prot,%eax		; \
@@ -238,11 +244,11 @@ _KPTphys:	.long	0			/* phys addr of kernel page tables */
 #define	fillkptphys(prot)		  \
 	movl	%eax, %ebx		; \
 	shrl	$(PAGE_SHIFT), %ebx	; \
-	fillkpt(PA(EXT(KPTphys)), prot)
+	fillkpt(PA(KPTphys), prot)
 
-	
 /*
- * All CPUs start here.
+ * BSP CPU start here.
+ *	eax points to kernbootstruct
  *
  * Environment:
  *	protected mode, no paging, flat 32-bit address space.
@@ -250,50 +256,103 @@ _KPTphys:	.long	0			/* phys addr of kernel page tables */
  */
 	.text
 	.align	ALIGN
-	.globl	EXT(pstart)
 	.globl	EXT(_start)
+	.globl	EXT(_pstart)
 LEXT(_start)
+LEXT(_pstart)
+	mov	%ds, %bx
+	mov	%bx, %es
+	mov	%eax, %ebp		// Move kernbootstruct to ebp
+	POSTCODE(_PSTART_ENTRY)
+	movl	KADDR(%ebp), %ebx	// Load boot image phys addr
+	movl	%ebx, %edx		// Set edx with boot load phys addr
+	addl	KSIZE(%ebp), %edx	// Add boot image size
+	addl	$(NBPG-1), %edx		// Round to a page size
+	andl	$(-NBPG), %edx		// Set edx to first free page
+	movl	%edx, %esp		// Set temporay stack
+	addl	$(NBPG), %esp		// add page size
+	call	Ls1
+Ls1:	popl	%esi			// Get return address
+	cmpl	$(PA(Ls1)), %esi 	// Compare with static physicall addr 
+	je	EXT(pstart)		// Branch if equal
+	subl	$(PA(Ls1)), %esi	// Extract relocation offset
+	movl	%esi, %esp		// Store relocation offset in esp
+	leal	(PA(Lreloc_start))(%esp),%esi
+					// Set esi to reloc_start boot phys addr
+	movl	%edx, %edi		// Set edi to first free page
+	movl	$(Lreloc_end-Lreloc_start), %ecx
+					// Set ecx to copy code size
+	cld				// count up
+	rep
+	movsb				// copy reloc copy code
+	wbinvd				// Write back and Invalidate cache
+	movl	%ebx, %esi		// Set esi to kernbootstruct kaddr
+	movl	KADDR(%ebp), %edi	// Load boot image phys addr
+	subl	%esp,  %edi		// Adjust to static phys addr
+	movl	KSIZE(%ebp), %ecx	// Set ecx to kernbootstruct ksize
+	addl	$(NBPG-1), %ecx		// Add NBPG-1 to ecx
+	andl	$(-NBPG), %ecx		// Truncate  ecx to a page aligned addr
+	sarl	$2, %ecx		// Divide ecx by 4
+	movl	%esp, (PA(EXT(KernelRelocOffset)))(%esp)
+					// Store relocation offset
+	movl	%edi, KADDR(%ebp)	// Relocate kaddr in kernbootstruct
+	subl	%esp, MEMORYMAP(%ebp)	// And relocate MemoryMap 
+	subl	%esp, DEVICETREEP(%ebp)	// And relocate deviceTreeP
+	subl	%esp, %ebp		// Set ebp with relocated phys addr
+	jmp	*%edx			// Branch to relocated copy code
+Lreloc_start:
+	POSTCODE(_PSTART_RELOC)
+	rep
+	movsl				// Copy boot image at BASE_KERNEL_PADDR
+	wbinvd				// Write back and Invalidate cache
+	movl	$(PA(EXT(pstart))), %edx	// Set branch target
+	jmp	*%edx			// Far jmp to pstart phys addr
+Lreloc_end:
+	/* NOTREACHED */
+	hlt
+
+	.text
+	.globl __start
+	.set __start, PA(EXT(_pstart))
+
+/*
+ * BSP CPU continues here after possible relocation.
+ *	ebp points to kernbootstruct
+ */
+	.align	ALIGN
+	.globl	EXT(pstart)
 LEXT(pstart)
-	mov     %eax, %ebx		/* save pointer to kernbootstruct */
+	mov     %ebp, %ebx		/* get pointer to kernbootstruct */
 
-	POSTCODE(PSTART_ENTRY);
+	POSTCODE(PSTART_ENTRY)
 
 	mov	$0,%ax			/* fs must be zeroed; */
 	mov	%ax,%fs			/* some bootstrappers don`t do this */
 	mov	%ax,%gs
 
-	jmp	1f
-0:	cmpl	$0,PA(EXT(start_lock))
-	jne	0b
-1:	movb	$1,%eax
-	xchgl	%eax,PA(EXT(start_lock)) /* locked */
-	testl	%eax,%eax
-	jnz	0b
-
-	cmpl	$0,PA(EXT(master_is_up))	/* are we first? */
-	jne	EXT(slave_start)		/* no -- system already up. */
-	movl	$1,PA(EXT(master_is_up))	/* others become slaves */
-	jmp	3f
-3:
-
 /*
  * Get startup parameters.
  */
-
-        movl	%ebx,PA(EXT(boot_args_start))  /* Save KERNBOOTSTRUCT */
-	
 	movl	KADDR(%ebx), %eax
 	addl	KSIZE(%ebx), %eax
 	addl	$(NBPG-1),%eax
 	andl	$(-NBPG), %eax
-	movl	%eax, PA(EXT(KERNend))
 	movl	%eax, PA(physfree)
 	cld
 
 /* allocate kernel page table pages */
 	ALLOCPAGES(NKPT)
-	movl	%esi,PA(EXT(KPTphys))
+	movl	%esi,PA(KPTphys)
 
+#ifdef X86_64
+/* allocate PML4 page */
+	ALLOCPAGES(1)
+	movl	%esi,EXT(IdlePML4)
+/* allocate new 3rd level directory page */
+	ALLOCPAGES(1)
+	movl	%esi,EXT(IdlePDPT64)
+#endif
+	
 #ifdef PAE
 /* allocate Page Table Directory Page */
 	ALLOCPAGES(1)
@@ -315,19 +374,23 @@ LEXT(pstart)
 	movl	PA(EXT(IdlePDPT)), %eax
 	movl	$1, %ecx
 	fillkptphys( $(PTE_W) )
+	
+	movl	PA(EXT(IdlePDPT64)), %eax
+	movl	$1, %ecx
+	fillkptphys( $(PTE_W) )
 #endif
 	movl	PA(EXT(IdlePTD)),%eax
 	movl	$(NPGPTD), %ecx
 	fillkptphys( $(PTE_W) )
 
 /* install a pde for temp double map of bottom of VA */
-	movl	PA(EXT(KPTphys)),%eax
+	movl	PA(KPTphys),%eax
 	xorl	%ebx,%ebx
 	movl	$(NKPT), %ecx
 	fillkpt(PA(EXT(IdlePTD)), $(PTE_W))
 
 /* install pde's for page tables */
-	movl	PA(EXT(KPTphys)),%eax
+	movl	PA(KPTphys),%eax
 	movl	$(KPTDI),%ebx
 	movl	$(NKPT),%ecx
 	fillkpt(PA(EXT(IdlePTD)), $(PTE_W))
@@ -404,24 +467,24 @@ LEXT(pstart)
 	loop	1b
 #endif
 	
-	POSTCODE(PSTART_PAGE_TABLES);
+	POSTCODE(PSTART_PAGE_TABLES)
 
 /*
  * Fix initial descriptor tables.
  */
-	lea	PA(EXT(idt)),%esi	/* fix IDT */
+	lea	PA(EXT(master_idt)),%esi	/* fix IDT */
 	movl	$(IDTSZ),%ecx
 	movl	$(PA(fix_idt_ret)),%ebx
 	jmp	fix_desc_common		/* (cannot use stack) */
 fix_idt_ret:
 
-	lea	PA(EXT(gdt)),%esi	/* fix GDT */
+	lea	PA(EXT(master_gdt)),%esi	/* fix GDT */
 	movl	$(GDTSZ),%ecx
 	movl	$(PA(fix_gdt_ret)),%ebx
 	jmp	fix_desc_common		/* (cannot use stack) */
 fix_gdt_ret:
 
-	lea	PA(EXT(ldt)),%esi	/* fix LDT */
+	lea	PA(EXT(master_ldt)),%esi	/* fix LDT */
 	movl	$(LDTSZ),%ecx
 	movl	$(PA(fix_ldt_ret)),%ebx
 	jmp	fix_desc_common		/* (cannot use stack) */
@@ -434,7 +497,7 @@ fix_ldt_ret:
 	lgdt	PA(EXT(gdtptr))		/* load GDT */
 	lidt	PA(EXT(idtptr))		/* load IDT */
 
-	POSTCODE(PSTART_BEFORE_PAGING);
+	POSTCODE(PSTART_BEFORE_PAGING)
 
 /*
  * Turn on paging.
@@ -444,8 +507,21 @@ fix_ldt_ret:
 	movl	%eax, %cr3
 
 	movl	%cr4, %eax
-	orl	$(CR4_PAE), %eax
+	orl	$(CR4_PAE|CR4_PGE), %eax
 	movl	%eax, %cr4
+
+	movl	$0x80000001, %eax
+	cpuid
+	and	$(CPUID_EXTFEATURE_XD), %edx	/* clear all but bit 20 */
+	cmp	$0, %edx		/* skip setting NXE if 20 is not set */
+	je	1f
+	
+	movl	$(MSR_IA32_EFER), %ecx			/* MSR number in ecx */
+	rdmsr						/* MSR value return in edx: eax */
+	orl	$(MSR_IA32_EFER_NXE), %eax		/* Set NXE bit in low 32-bits */
+	wrmsr						/* Update Extended Feature Enable reg */
+1:
+
 #else	
 	movl	PA(EXT(IdlePTD)), %eax
 	movl	%eax,%cr3
@@ -458,7 +534,7 @@ fix_ldt_ret:
 	LJMP(KERNEL_CS,EXT(vstart))	/* switch to kernel code segment */
 
 /*
- * Master is now running with correct addresses.
+ * BSP is now running with correct addresses.
  */
 LEXT(vstart)
 	POSTCODE(VSTART_ENTRY)	; 
@@ -467,20 +543,24 @@ LEXT(vstart)
 	mov	%ax,%ds
 	mov	%ax,%es
 	mov	%ax,%ss
-	mov	%ax,EXT(ktss)+TSS_SS0	/* set kernel stack segment */
+	mov	%ax,EXT(master_ktss)+TSS_SS0	/* set kernel stack segment */
 					/* for traps to kernel */
+
 #if	MACH_KDB
-	mov	%ax,EXT(dbtss)+TSS_SS0	/* likewise for debug task switch */
+	mov	%ax,EXT(master_dbtss)+TSS_SS0	/* likewise for debug task switch */
 	mov	%cr3,%eax		/* get PDBR into debug TSS */
-	mov	%eax,EXT(dbtss)+TSS_PDBR
+	mov	%eax,EXT(master_dbtss)+TSS_PDBR
 	mov	$0,%eax
 #endif
+	mov	%cr3,%eax		/* get PDBR into DF TSS */
+	mov	%eax,EXT(master_dftss)+TSS_PDBR
+	mov	%eax,EXT(master_mctss)+TSS_PDBR
 
 	movw	$(KERNEL_LDT),%ax	/* get LDT segment */
 	lldt	%ax			/* load LDT */
 #if	MACH_KDB
-	mov	%ax,EXT(ktss)+TSS_LDT	/* store LDT in two TSS, as well... */
-	mov	%ax,EXT(dbtss)+TSS_LDT	/*   ...matters if we switch tasks */
+	mov	%ax,EXT(master_ktss)+TSS_LDT	/* store LDT in two TSS, as well... */
+	mov	%ax,EXT(master_dbtss)+TSS_LDT	/*   ...matters if we switch tasks */
 #endif
 	movw	$(KERNEL_TSS),%ax
 	ltr	%ax			/* set up KTSS */
@@ -488,72 +568,56 @@ LEXT(vstart)
 	mov	$(CPU_DATA_GS),%ax
 	mov	%ax,%gs
 
-	POSTCODE(VSTART_STACK_SWITCH);
+	POSTCODE(VSTART_STACK_SWITCH)
 
-	lea	EXT(eintstack),%esp	/* switch to the bootup stack */
-	call EXT(i386_preinit)
+	lea	EXT(low_eintstack),%esp	/* switch to the bootup stack */
+	pushl	%ebp			/* push boot args addr */
+	xorl	%ebp,%ebp		/* clear stack frame ptr */
 
-	POSTCODE(VSTART_EXIT);
+	POSTCODE(VSTART_EXIT)
 
 	call	EXT(i386_init)		/* run C code */
 	/*NOTREACHED*/
 	hlt
 
-	.text
-	.globl __start
-	.set __start, PA(EXT(pstart))
-
-	
-/*
- * master_up is used by the master cpu to signify that it is done
- * with the interrupt stack, etc. See the code in pstart and svstart
- * that this interlocks with.
- */
-	.align	ALIGN
-	.globl	EXT(master_up)
-LEXT(master_up)
-	pushl	%ebp			/* set up */
-	movl	%esp,%ebp		/* stack frame */
-	movl	$0,%ecx			/* unlock start_lock */
-	xchgl	%ecx,EXT(start_lock)	/* since we are no longer using */
-					/* bootstrap stack */
-	leave				/* pop stack frame */
-	ret
 
 /*
- * We aren't the first.  Call slave_main to initialize the processor
- * and get Mach going on it.
+ * AP (slave) CPUs enter here.
+ *
+ * Environment:
+ *	protected mode, no paging, flat 32-bit address space.
+ *	(Code/data/stack segments have base == 0, limit == 4G)
  */
 	.align	ALIGN
-	.globl	EXT(slave_start)
-LEXT(slave_start)
+	.globl	EXT(slave_pstart)
+LEXT(slave_pstart)
 	cli				/* disable interrupts, so we don`t */
 					/* need IDT for a while */
 
-	POSTCODE(SLAVE_START_ENTRY);
+	POSTCODE(SLAVE_PSTART_ENTRY)
 /*
  * Turn on paging.
  */
-	movl	$(EXT(spag_start)),%edx /* first paged code address */
-
 #ifdef PAE
-	movl	$(0x4000), %eax
-	movl	%eax, %cr3
-
 	movl	%cr4, %eax
-	orl	$(CR4_PAE), %eax
+	orl	$(CR4_PAE|CR4_PGE), %eax
 	movl	%eax, %cr4
-#else
+
+	movl	$(MSR_IA32_EFER), %ecx			/* MSR number in ecx */
+	rdmsr						/* MSR value return in edx: eax */
+	orl	$(MSR_IA32_EFER_NXE), %eax		/* Set NXE bit in low 32-bits */
+	wrmsr						/* Update Extended Feature Enable reg */
+#endif
 	movl	$(0x4000),%eax  /* tmp until we get mapped */
 	movl	%eax,%cr3
-#endif
 
 	movl	%cr0,%eax
 	orl	$(CR0_PG|CR0_WP|CR0_PE),%eax
 	movl	%eax,%cr0		/* to enable paging */
 
-	POSTCODE(SLAVE_START_EXIT);
+	POSTCODE(SLAVE_PSTART_EXIT)
 
+	movl	$(EXT(spag_start)),%edx /* first paged code address */
 	jmp	*%edx			/* flush prefetch queue */
 
 /*
@@ -564,15 +628,15 @@ LEXT(spag_start)
 	lgdt	PA(EXT(gdtptr))		/* load GDT */
 	lidt	PA(EXT(idtptr))		/* load IDT */
 
-	LJMP(KERNEL_CS,EXT(svstart))	/* switch to kernel code segment */
+	LJMP(KERNEL_CS,EXT(slave_vstart))	/* switch to kernel code segment */
 
 
 /*
  * Slave is now running with correct addresses.
  */
-LEXT(svstart)
+LEXT(slave_vstart)
 
-	POSTCODE(SVSTART_ENTRY);
+	POSTCODE(SLAVE_VSTART_ENTRY)
 
 #ifdef PAE
 	movl	PA(EXT(IdlePDPT)), %eax
@@ -598,19 +662,18 @@ LEXT(svstart)
 /*
  * Switch to the per-cpu descriptor tables
  */
-	POSTCODE(SVSTART_DESC_INIT);
+	POSTCODE(SLAVE_VSTART_DESC_INIT)
 
 	CPU_NUMBER_FROM_LAPIC(%eax)
 	movl	CX(EXT(cpu_data_ptr),%eax),%ecx
-	movl	CPU_DESC_TABLEP(%ecx), %ecx
 
 	movw	$(GDTSZ*8-1),0(%esp)	/* set GDT size in GDT descriptor */
-	leal	MP_GDT(%ecx),%edx
+	movl	CPU_DESC_INDEX+CDI_GDT(%ecx),%edx
 	movl	%edx,2(%esp)		/* point to local GDT (linear addr) */
 	lgdt	0(%esp)			/* load new GDT */
 	
 	movw	$(IDTSZ*8-1),0(%esp)	/* set IDT size in IDT descriptor */
-	leal	MP_IDT(%ecx),%edx
+	movl	CPU_DESC_INDEX+CDI_IDT(%ecx),%edx
 	movl	%edx,2(%esp)		/* point to local IDT (linear addr) */
 	lidt	0(%esp)			/* load new IDT */
 	
@@ -626,15 +689,12 @@ LEXT(svstart)
 /*
  * Get stack top from pre-cpu data and switch
  */
-	POSTCODE(SVSTART_STACK_SWITCH);
+	POSTCODE(SLAVE_VSTART_STACK_SWITCH)
 
 	movl	%gs:CPU_INT_STACK_TOP,%esp
 	xorl    %ebp,%ebp               /* for completeness */
 
-	movl	$0,%eax			/* unlock start_lock */
-	xchgl	%eax,EXT(start_lock)	/* since we are no longer using */
-					/* bootstrap stack */
-	POSTCODE(SVSTART_EXIT);
+	POSTCODE(SLAVE_VSTART_EXIT)
 
 	call	EXT(i386_init_slave)	/* start MACH */
 	/*NOTREACHED*/
diff --git a/osfmk/i386/start64.s b/osfmk/i386/start64.s
new file mode 100644
index 000000000..447ad1acd
--- /dev/null
+++ b/osfmk/i386/start64.s
@@ -0,0 +1,182 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <platforms.h>
+#include <mach_kdb.h>
+
+#include <i386/asm.h>
+#include <i386/asm64.h>
+#include <i386/proc_reg.h>
+#include <i386/postcode.h>
+#include <assym.s>
+
+	.data
+	.align 3
+	.globl EXT(gdtptr64)
+	/* align below right */
+	.word	0
+LEXT(gdtptr64)
+	.word	Times(8,GDTSZ)-1
+	/* XXX really want .quad here */
+	.long	EXT(master_gdt)
+	.long	KERNEL_UBER_BASE_HI32  /* must be in uber-space */
+
+	.align 3
+	.globl EXT(idtptr64)
+	/* align below right */
+	.word	0
+LEXT(idtptr64)
+	.word	Times(16,IDTSZ)-1
+	/* XXX really want .quad here */
+	.long	EXT(master_idt64)
+	.long	KERNEL_UBER_BASE_HI32  /* must be in uber-space */
+
+	.text
+
+Entry(ml_load_desc64)
+
+	ENTER_64BIT_MODE()
+
+	POSTCODE(ML_LOAD_DESC64_ENTRY)
+	
+	lgdt	EXT(gdtptr64)		/* load GDT */
+
+	POSTCODE(ML_LOAD_DESC64_GDT)
+
+	lidt	EXT(idtptr64)		/* load IDT */
+
+	POSTCODE(ML_LOAD_DESC64_IDT)
+
+	movw	$(KERNEL_LDT),%ax	/* get LDT segment */
+	lldt	%ax			/* load LDT */
+
+	POSTCODE(ML_LOAD_DESC64_LDT)
+
+	movw	$(KERNEL_TSS),%ax
+	ltr	%ax			/* set up KTSS */
+
+	POSTCODE(ML_LOAD_DESC64_EXIT)
+
+	ENTER_COMPAT_MODE()
+
+	ret
+
+
+Entry(ml_64bit_wrmsr64)
+	/* (uint32_t msr, uint64_t value) */
+	/* (uint32_t msr, uint32_t lo, uint32_t hi) */
+
+	FRAME
+
+	ENTER_64BIT_MODE()
+
+	movl	B_ARG0, %ecx
+	movl	B_ARG1, %eax
+	movl	B_ARG2, %edx
+	wrmsr
+
+	ENTER_COMPAT_MODE()
+
+	EMARF
+	ret
+
+
+Entry(ml_64bit_lldt)
+	/* (int32_t selector) */
+
+	FRAME
+
+	ENTER_64BIT_MODE()
+
+	movl	B_ARG0, %eax
+	lldt	%ax
+
+	ENTER_COMPAT_MODE()
+
+	EMARF
+	ret
+
+Entry(set_64bit_debug_regs)
+	/* x86_debug_state64_t *ds */
+
+	FRAME
+
+	ENTER_64BIT_MODE()
+
+	mov	B_ARG0, %edx
+	mov	DS64_DR0(%edx), %rax
+	mov	%rax, %dr0
+	mov	DS64_DR1(%edx), %rax
+	mov	%rax, %dr1
+	mov	DS64_DR2(%edx), %rax
+	mov	%rax, %dr2
+	mov	DS64_DR3(%edx), %rax
+	mov	%rax, %dr3
+
+	ENTER_COMPAT_MODE()
+
+	EMARF
+	ret
+
+Entry(flush_tlb64)
+
+	FRAME
+
+	ENTER_64BIT_MODE()
+
+	mov	%cr3, %rax
+	mov	%rax, %cr3
+
+	ENTER_COMPAT_MODE()
+
+	EMARF
+	ret
+
+/* FXSAVE and FXRSTOR operate in a mode dependent fashion, hence these variants.
+* Must be called with interrupts disabled.
+* We clear pending x87 exceptions here; this is technically incorrect, since we should
+* propagate those to the user, but the compatibility mode kernel is currently not
+* prepared to handle exceptions originating in 64-bit kernel mode. However, it may be possible
+* to work around this should it prove necessary.
+*/
+
+Entry(fxsave64)
+	movl		S_ARG0,%eax
+	ENTER_64BIT_MODE()
+	fnclex
+	fxsave		0(%eax)
+	ENTER_COMPAT_MODE()
+	ret
+
+Entry(fxrstor64)
+	movl		S_ARG0,%eax
+	ENTER_64BIT_MODE()
+	fnclex
+	fxrstor		0(%rax)
+	ENTER_COMPAT_MODE()
+	ret
diff --git a/osfmk/i386/startup64.c b/osfmk/i386/startup64.c
new file mode 100644
index 000000000..29d21fac8
--- /dev/null
+++ b/osfmk/i386/startup64.c
@@ -0,0 +1,348 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <string.h>
+
+#include <mach/machine/vm_types.h>
+
+#include <mach/boolean.h>
+#include <kern/thread.h>
+#include <kern/zalloc.h>
+
+#include <kern/lock.h>
+#include <kern/kalloc.h>
+#include <kern/spl.h>
+
+#include <vm/pmap.h>
+#include <vm/vm_map.h>
+#include <vm/vm_kern.h>
+#include <mach/vm_param.h>
+#include <mach/vm_prot.h>
+#include <vm/vm_object.h>
+#include <vm/vm_page.h>
+
+#include <mach/machine/vm_param.h>
+#include <machine/thread.h>
+
+#include <kern/misc_protos.h>			/* prototyping */
+#include <i386/misc_protos.h>
+
+#include <i386/cpuid.h>
+#include <i386/cpu_data.h>
+#include <i386/mp.h>
+#include <i386/cpu_number.h>
+#include <i386/machine_cpu.h>
+#include <i386/mp_slave_boot.h>
+#include <i386/seg.h>
+
+#include <vm/vm_protos.h>
+
+#include <sys/kdebug.h>
+
+#include <i386/postcode.h>
+
+void
+cpu_IA32e_enable(cpu_data_t *cdp)
+{
+	uint32_t	cr0 = get_cr0();
+        uint64_t	efer = rdmsr64(MSR_IA32_EFER);
+
+	assert(!ml_get_interrupts_enabled());
+
+	postcode(CPU_IA32_ENABLE_ENTRY);
+
+	/* Turn paging off - works because we're identity mapped */
+	set_cr0(cr0 & ~CR0_PG);
+
+	/* pop in new top level phys pg addr */
+	set_cr3((vm_offset_t) kernel64_cr3);
+
+	wrmsr64(MSR_IA32_EFER, efer | MSR_IA32_EFER_LME); /* set  mode */
+
+	/* Turn paging on */
+        set_cr0(cr0 | CR0_PG);
+
+	/* this call is required to re-activate paging */
+	kprintf("cpu_IA32e_enable(%p)\n", cdp);
+
+	if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) == 0)
+		panic("cpu_IA32e_enable() MSR_IA32_EFER_LMA not asserted");
+
+	cdp->cpu_kernel_cr3 = kernel64_cr3;
+
+	postcode(CPU_IA32_ENABLE_EXIT);
+}
+
+void
+cpu_IA32e_disable(cpu_data_t *cdp)
+{
+	uint32_t	cr0 = get_cr0();
+        uint64_t	efer = rdmsr64(MSR_IA32_EFER);
+
+	assert(!ml_get_interrupts_enabled());
+
+	postcode(CPU_IA32_DISABLE_ENTRY);
+
+	if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) == 0)
+		panic("cpu_IA32e_disable() MSR_IA32_EFER_LMA clear on entry");
+
+	/* Turn paging off - works because we're identity mapped */
+	set_cr0(cr0 & ~CR0_PG);
+
+	/* pop in legacy top level phys pg addr */
+	set_cr3((vm_offset_t) lo_kernel_cr3);
+
+	wrmsr64(MSR_IA32_EFER, efer & ~MSR_IA32_EFER_LME); /* reset mode */
+
+	/* Turn paging on */
+        set_cr0(cr0 | CR0_PG);
+
+	/* this call is required to re-activate paging */
+	kprintf("cpu_IA32e_disable(%p)\n", cdp);
+
+	if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) != 0)
+		panic("cpu_IA32e_disable() MSR_IA32_EFER_LMA not cleared");
+
+	cdp->cpu_kernel_cr3 = 0ULL;
+
+	postcode(CPU_IA32_DISABLE_EXIT);
+}
+
+void
+fix_desc64(void *descp, int count)
+{
+	struct fake_descriptor64	*fakep;
+	union {
+		struct real_gate64		gate;
+		struct real_descriptor64	desc;
+	}				real;
+	int				i;
+
+	fakep = (struct fake_descriptor64 *) descp;
+	
+	for (i = 0; i < count; i++, fakep++) {
+		/*
+		 * Construct the real decriptor locally.
+		 */
+
+		bzero((void *) &real, sizeof(real));
+
+		switch (fakep->access & ACC_TYPE) {
+		case 0:
+			break;
+		case ACC_CALL_GATE:
+		case ACC_INTR_GATE:
+		case ACC_TRAP_GATE:
+			real.gate.offset_low16 = fakep->offset[0] & 0xFFFF;
+			real.gate.selector16 = fakep->lim_or_seg & 0xFFFF;
+			real.gate.IST = fakep->size_or_IST & 0x7;
+			real.gate.access8 = fakep->access;
+			real.gate.offset_high16 = (fakep->offset[0]>>16)&0xFFFF;
+			real.gate.offset_top32 = (uint32_t)fakep->offset[1];
+			break;
+		default:	/* Otherwise */
+			real.desc.limit_low16 = fakep->lim_or_seg & 0xFFFF;
+			real.desc.base_low16 = fakep->offset[0] & 0xFFFF;
+			real.desc.base_med8 = (fakep->offset[0] >> 16) & 0xFF;
+			real.desc.access8 = fakep->access;
+			real.desc.limit_high4 = (fakep->lim_or_seg >> 16) & 0xFF;
+			real.desc.granularity4 = fakep->size_or_IST;
+			real.desc.base_high8 = (fakep->offset[0] >> 24) & 0xFF;
+			real.desc.base_top32 = (uint32_t) fakep->offset[1];
+		}
+
+		/*
+		 * Now copy back over the fake structure.
+		 */
+		bcopy((void *) &real, (void *) fakep, sizeof(real));
+	}
+}
+
+#if DEBUG
+extern void dump_gdt(void *);
+extern void dump_ldt(void *);
+extern void dump_idt(void *);
+extern void dump_tss(void *);
+extern void dump_frame32(x86_saved_state_compat32_t *scp);
+extern void dump_frame64(x86_saved_state64_t *sp);
+extern void dump_frame(x86_saved_state_t *sp);
+
+void
+dump_frame(x86_saved_state_t *sp)
+{
+	if (is_saved_state32(sp))
+		dump_frame32((x86_saved_state_compat32_t *) sp);
+	else if (is_saved_state64(sp))
+		dump_frame64(&sp->ss_64);
+	else
+		kprintf("dump_frame(%p) unknown type %d\n", sp, sp->flavor);
+}
+
+void
+dump_frame32(x86_saved_state_compat32_t *scp)
+{
+	unsigned int	i;
+	uint32_t	*ip = (uint32_t *) scp;
+
+	kprintf("dump_frame32(0x%08x):\n", scp);
+	
+	for (i = 0;
+	     i < sizeof(x86_saved_state_compat32_t)/sizeof(uint32_t);
+	     i++, ip++)
+		kprintf("0x%08x: 0x%08x\n", ip, *ip);
+
+	kprintf("scp->isf64.err:    0x%016llx\n", scp->isf64.err);
+	kprintf("scp->isf64.rip:    0x%016llx\n", scp->isf64.rip);
+	kprintf("scp->isf64.cs:     0x%016llx\n", scp->isf64.cs);
+	kprintf("scp->isf64.rflags: 0x%016llx\n", scp->isf64.rflags);
+	kprintf("scp->isf64.rsp:    0x%016llx\n", scp->isf64.rsp);
+	kprintf("scp->isf64.ss:     0x%016llx\n", scp->isf64.ss);
+
+	kprintf("scp->iss32.tag:    0x%08x\n", scp->iss32.tag);
+	kprintf("scp->iss32.state.gs:     0x%08x\n", scp->iss32.state.gs);
+	kprintf("scp->iss32.state.fs:     0x%08x\n", scp->iss32.state.fs);
+	kprintf("scp->iss32.state.es:     0x%08x\n", scp->iss32.state.es);
+	kprintf("scp->iss32.state.ds:     0x%08x\n", scp->iss32.state.ds);
+	kprintf("scp->iss32.state.edi:    0x%08x\n", scp->iss32.state.edi);
+	kprintf("scp->iss32.state.esi:    0x%08x\n", scp->iss32.state.esi);
+	kprintf("scp->iss32.state.ebp:    0x%08x\n", scp->iss32.state.ebp);
+	kprintf("scp->iss32.state.cr2:    0x%08x\n", scp->iss32.state.cr2);
+	kprintf("scp->iss32.state.ebx:    0x%08x\n", scp->iss32.state.ebx);
+	kprintf("scp->iss32.state.edx:    0x%08x\n", scp->iss32.state.edx);
+	kprintf("scp->iss32.state.ecx:    0x%08x\n", scp->iss32.state.ecx);
+	kprintf("scp->iss32.state.eax:    0x%08x\n", scp->iss32.state.eax);
+	kprintf("scp->iss32.state.trapno: 0x%08x\n", scp->iss32.state.eax);
+	kprintf("scp->iss32.state.eip:    0x%08x\n", scp->iss32.state.eip);
+	kprintf("scp->iss32.state.cs:     0x%08x\n", scp->iss32.state.cs);
+	kprintf("scp->iss32.state.efl:    0x%08x\n", scp->iss32.state.efl);
+	kprintf("scp->iss32.state.uesp:   0x%08x\n", scp->iss32.state.uesp);
+	kprintf("scp->iss32.state.ss:     0x%08x\n", scp->iss32.state.ss);
+
+	postcode(0x99);
+}
+
+void
+dump_frame64(x86_saved_state64_t *sp)
+{
+	unsigned int	i;
+	uint64_t	*ip = (uint64_t *) sp;
+
+	kprintf("dump_frame64(%p):\n", sp);
+	
+	for (i = 0;
+	     i < sizeof(x86_saved_state64_t)/sizeof(uint64_t);
+	     i++, ip++)
+		kprintf("0x%08x: 0x%016x\n", ip, *ip);
+
+	kprintf("sp->isf.trapno: 0x%08x\n", sp->isf.trapno);
+	kprintf("sp->isf.trapfn: 0x%08x\n", sp->isf.trapfn);
+	kprintf("sp->isf.err:    0x%016llx\n", sp->isf.err);
+	kprintf("sp->isf.rip:    0x%016llx\n", sp->isf.rip);
+	kprintf("sp->isf.cs:     0x%016llx\n", sp->isf.cs);
+	kprintf("sp->isf.rflags: 0x%016llx\n", sp->isf.rflags);
+	kprintf("sp->isf.rsp:    0x%016llx\n", sp->isf.rsp);
+	kprintf("sp->isf.ss:     0x%016llx\n", sp->isf.ss);
+
+	kprintf("sp->fs:         0x%016x\n", sp->fs);
+	kprintf("sp->gs:         0x%016x\n", sp->gs);
+	kprintf("sp->rax:        0x%016llx\n", sp->rax);
+	kprintf("sp->rcx:        0x%016llx\n", sp->rcx);
+	kprintf("sp->rbx:        0x%016llx\n", sp->rbx);
+	kprintf("sp->rbp:        0x%016llx\n", sp->rbp);
+	kprintf("sp->r11:        0x%016llx\n", sp->r11);
+	kprintf("sp->r12:        0x%016llx\n", sp->r12);
+	kprintf("sp->r13:        0x%016llx\n", sp->r13);
+	kprintf("sp->r14:        0x%016llx\n", sp->r14);
+	kprintf("sp->r15:        0x%016llx\n", sp->r15);
+	kprintf("sp->cr2:        0x%016llx\n", sp->cr2);
+	kprintf("sp->v_arg8:     0x%016llx\n", sp->v_arg8);
+	kprintf("sp->v_arg7:     0x%016llx\n", sp->v_arg7);
+	kprintf("sp->v_arg6:     0x%016llx\n", sp->v_arg6);
+	kprintf("sp->r9:         0x%016llx\n", sp->r9);
+	kprintf("sp->r8:         0x%016llx\n", sp->r8);
+	kprintf("sp->r10:        0x%016llx\n", sp->r10);
+	kprintf("sp->rdx:        0x%016llx\n", sp->rdx);
+	kprintf("sp->rsi:        0x%016llx\n", sp->rsi);
+	kprintf("sp->rdi:        0x%016llx\n", sp->rdi);
+
+	postcode(0x98);
+}
+
+void
+dump_gdt(void *gdtp)
+{
+	unsigned int	i;
+	uint32_t	*ip = (uint32_t *) gdtp;
+
+	kprintf("GDT:\n", ip);
+	for (i = 0; i < GDTSZ; i++, ip += 2) {
+		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
+		kprintf("%p: 0x%08x\n", ip+1, *(ip+1));
+	}
+}
+
+void
+dump_ldt(void *ldtp)
+{
+	unsigned int	i;
+	uint32_t	*ip = (uint32_t *) ldtp;
+
+	kprintf("LDT:\n", ip);
+	for (i = 0; i < LDTSZ_MIN; i++, ip += 2) {
+		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
+		kprintf("%p: 0x%08x\n", ip+1, *(ip+1));
+	}
+}
+
+void
+dump_idt(void *idtp)
+{
+	unsigned int	i;
+	uint32_t	*ip = (uint32_t *) idtp;
+
+	kprintf("IDT64:\n", ip);
+	for (i = 0; i < 16; i++, ip += 4) {
+		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
+		kprintf("%p: 0x%08x\n", ip+1, *(ip+1));
+		kprintf("%p: 0x%08x\n", ip+2, *(ip+2));
+		kprintf("%p: 0x%08x\n", ip+3, *(ip+3));
+	}
+}
+
+void
+dump_tss(void *tssp)
+{
+	unsigned int	i;
+	uint32_t	*ip = (uint32_t *) tssp;
+
+	kprintf("TSS64:\n", ip);
+	for (i = 0; i < sizeof(master_ktss64)/sizeof(uint32_t); i++, ip++) {
+		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
+	}
+}
+#endif /* DEBUG */
diff --git a/osfmk/i386/task.h b/osfmk/i386/task.h
index c35069ab6..d7df80459 100644
--- a/osfmk/i386/task.h
+++ b/osfmk/i386/task.h
@@ -55,8 +55,7 @@
  */
 
 /*
- * No machine dependant task fields
+ * Machine dependant task fields
  */
 
-#define MACHINE_TASK
-
+#define MACHINE_TASK struct user_ldt *       i386_ldt;
diff --git a/osfmk/i386/thread.h b/osfmk/i386/thread.h
index b40bae4db..d29d7ac26 100644
--- a/osfmk/i386/thread.h
+++ b/osfmk/i386/thread.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -89,119 +89,77 @@
  *	For performance, it is also used directly in syscall exceptions
  *	if the server has requested i386_THREAD_STATE flavor for the exception
  *	port.
- *
- *	We define the following as an alias for the "esp" field of the
- *	structure, because we actually save cr2 here, not the kernel esp.
  */
-#define cr2	esp
 
 /*
  *	Save area for user floating-point state.
  *	Allocated only when necessary.
  */
 
-struct i386_fpsave_state {
+struct x86_fpsave_state {
 	boolean_t		fp_valid;
-	struct i386_fp_save	fp_save_state;
-	struct i386_fp_regs	fp_regs;
-        struct i386_fx_save 	fx_save_state __attribute__ ((aligned (16)));
-	int			fp_save_flavor;
-};
-
-/*
- *	v86_assist_state:
- *
- *	This structure provides data to simulate 8086 mode
- *	interrupts.  It lives in the pcb.
- */
-
-struct v86_assist_state {
-	vm_offset_t		int_table;
-	unsigned short		int_count;
-	unsigned short		flags;	/* 8086 flag bits */
+	enum {
+		FXSAVE32 = 1,
+		FXSAVE64 = 2
+	} fp_save_layout;
+        struct x86_fx_save 	fx_save_state __attribute__ ((aligned (16)));
 };
-#define	V86_IF_PENDING		0x8000	/* unused bit */
-
-/*
- *	i386_interrupt_state:
- *
- *	This structure describes the set of registers that must
- *	be pushed on the current ring-0 stack by an interrupt before
- *	we can switch to the interrupt stack.
- */
 
-struct i386_interrupt_state {
-        int     gs;
-        int     fs;
-	int	es;
-	int	ds;
-	int	edx;
-	int	ecx;
-	int	eax;
-	int	eip;
-	int	cs;
-	int	efl;
-};
 
 /*
- *	i386_kernel_state:
+ *	x86_kernel_state32:
  *
  *	This structure corresponds to the state of kernel registers
  *	as saved in a context-switch.  It lives at the base of the stack.
+ *      kernel only runs in 32 bit mode for now
  */
 
-struct i386_kernel_state {
+struct x86_kernel_state32 {
 	int			k_ebx;	/* kernel context */
 	int			k_esp;
 	int			k_ebp;
 	int			k_edi;
 	int			k_esi;
 	int			k_eip;
+	/*
+	 * Kernel stacks are 16-byte aligned with a 4-byte i386_exception_link at
+	 * the top, followed by an x86_kernel_state32.  After both structs have
+	 * been pushed, we want to be 16-byte aligned.  A dummy int gets us there.
+	 */
+	int			dummy;
 };
 
-/*
- *	i386_machine_state:
- *
- *	This structure corresponds to special machine state.
- *	It lives in the pcb.  It is not saved by default.
- */
-
-struct i386_machine_state {
-	iopb_tss_t		io_tss;
-	struct user_ldt	*	ldt;
-	struct i386_fpsave_state *ifps;
-	struct v86_assist_state	v86s;
-};
 
 typedef struct pcb {
-	struct i386_interrupt_state iis[2];	/* interrupt and NMI */
-	struct i386_saved_state iss;
-	struct i386_machine_state ims;
+	void			*sf;
+	x86_saved_state_t	*iss;
+	struct x86_fpsave_state	*ifps;
 #ifdef	MACH_BSD
-	unsigned long	cthread_self;		/* for use of cthread package */
+	uint64_t	cthread_self;		/* for use of cthread package */
         struct real_descriptor cthread_desc;
-	unsigned long  uldt_selector;          /* user ldt selector to set */
-	struct real_descriptor uldt_desc;      /* the actual user setable ldt data */
+	unsigned long	uldt_selector;		/* user ldt selector to set */
+	struct real_descriptor uldt_desc;	/* the actual user setable ldt data */
 #endif
-	decl_simple_lock_data(,lock)
+	decl_simple_lock_data(,lock);
+	uint64_t	iss_pte0;
+	uint64_t	iss_pte1;
+	void		*ids;
 } *pcb_t;
 
+
 /*
  * Maps state flavor to number of words in the state:
  */
 __private_extern__ unsigned int _MachineStateCount[];
 
-#define USER_REGS(ThrAct)	(&(ThrAct)->machine.pcb->iss)
-
-#define act_machine_state_ptr(ThrAct)	(thread_state_t)USER_REGS(ThrAct)
+#define USER_STATE(ThrAct)	((ThrAct)->machine.pcb->iss)
+#define USER_REGS32(ThrAct)	(saved_state32(USER_STATE(ThrAct)))
+#define USER_REGS64(ThrAct)	(saved_state64(USER_STATE(ThrAct)))
 
+#define	user_pc(ThrAct)		(is_saved_state32(USER_STATE(ThrAct)) ?	\
+					USER_REGS32(ThrAct)->eip :	\
+					USER_REGS64(ThrAct)->isf.rip )
 
-#define	is_user_thread(ThrAct)	\
-  	((USER_REGS(ThrAct)->efl & EFL_VM) \
-	 || ((USER_REGS(ThrAct)->cs & 0x03) != 0))
-
-#define	user_pc(ThrAct)		(USER_REGS(ThrAct)->eip)
-#define	user_sp(ThrAct)		(USER_REGS(ThrAct)->uesp)
 
 struct machine_thread {
 	/*
@@ -211,9 +169,24 @@ struct machine_thread {
 	struct pcb xxx_pcb;
 	pcb_t pcb;
 
+	uint32_t	specFlags;
+#define		OnProc	0x1
+  
+        struct {
+	        user_addr_t	user_base;
+	} copy_window[NCOPY_WINDOWS];
+        int		nxt_window;
+        int		copyio_state;
+#define		WINDOWS_DIRTY	0
+#define		WINDOWS_CLEAN	1
+#define		WINDOWS_CLOSED	2
+#define		WINDOWS_OPENED	3
+        uint64_t	physwindow_pte;
+        int		physwindow_busy;
 };
 
-extern struct i386_saved_state *get_user_regs(thread_t);
+
+extern void *get_user_regs(thread_t);
 
 extern void *act_thread_csave(void);
 extern void act_thread_catt(void *ctx);
@@ -226,7 +199,7 @@ extern void act_thread_cfree(void *ctx);
  *	It points to the current thread`s user registers.
  */
 struct i386_exception_link {
-	struct i386_saved_state *saved_state;
+	x86_saved_state_t	*saved_state;
 };
 
 
@@ -239,7 +212,7 @@ struct i386_exception_link {
  */
 
 #define STACK_IKS(stack)	\
-	((struct i386_kernel_state *)((stack) + KERNEL_STACK_SIZE) - 1)
+	((struct x86_kernel_state32 *)((stack) + KERNEL_STACK_SIZE) - 1)
 #define STACK_IEL(stack)	\
 	((struct i386_exception_link *)STACK_IKS(stack) - 1)
 
diff --git a/osfmk/i386/trap.c b/osfmk/i386/trap.c
index 75071bb3a..34976d6f6 100644
--- a/osfmk/i386/trap.c
+++ b/osfmk/i386/trap.c
@@ -69,6 +69,7 @@
 #include <i386/trap.h>
 #include <i386/pmap.h>
 #include <i386/fpu.h>
+#include <i386/misc_protos.h> /* panic_io_port_read() */
 
 #include <mach/exception.h>
 #include <mach/kern_return.h>
@@ -88,17 +89,14 @@
 #include <kern/spl.h>
 #include <kern/misc_protos.h>
 
+#include <sys/kdebug.h>
+
 #if	MACH_KGDB
 #include <kgdb/kgdb_defs.h>
 #endif	/* MACH_KGDB */
 
-#include <i386/intel_read_fault.h>
-
-#if     MACH_KGDB
-#include <kgdb/kgdb_defs.h>
-#endif  /* MACH_KGDB */
-
 #if 	MACH_KDB
+#include <debug.h>
 #include <ddb/db_watch.h>
 #include <ddb/db_run.h>
 #include <ddb/db_break.h>
@@ -108,31 +106,56 @@
 #include <string.h>
 
 #include <i386/io_emulate.h>
+#include <i386/postcode.h>
+#include <i386/mp_desc.h>
+#include <i386/proc_reg.h>
+#include <i386/machine_check.h>
+#include <mach/i386/syscall_sw.h>
 
 /*
  * Forward declarations
  */
-extern void		user_page_fault_continue(
-				kern_return_t		kr);
+static void user_page_fault_continue(kern_return_t kret);
+static void panic_trap(x86_saved_state32_t *saved_state);
+static void set_recovery_ip(x86_saved_state32_t *saved_state, vm_offset_t ip);
 
-extern boolean_t	v86_assist(
-				thread_t		thread,
-				struct i386_saved_state	*regs);
-
-extern boolean_t	check_io_fault(
-				struct i386_saved_state	*regs);
-
-extern int		inst_fetch(
-				int			eip,
-				int			cs);
+perfCallback perfTrapHook = NULL; /* Pointer to CHUD trap hook routine */
+perfCallback perfASTHook  = NULL; /* Pointer to CHUD AST hook routine */
 
 void
 thread_syscall_return(
         kern_return_t ret)
 {
-        register thread_t   thr_act = current_thread();
-        register struct i386_saved_state *regs = USER_REGS(thr_act);
-        regs->eax = ret;
+        thread_t	thr_act = current_thread();
+
+        if (thread_is_64bit(thr_act)) {
+	        x86_saved_state64_t	*regs;
+		
+		regs = USER_REGS64(thr_act);
+
+		if (kdebug_enable && ((regs->rax & SYSCALL_CLASS_MASK) == (SYSCALL_CLASS_MACH << SYSCALL_CLASS_SHIFT))) {
+		        /* Mach trap */
+		        KERNEL_DEBUG_CONSTANT(
+					      MACHDBG_CODE(DBG_MACH_EXCP_SC, ((int) (regs->rax & SYSCALL_NUMBER_MASK)))
+					      | DBG_FUNC_END,
+					      ret, 0, 0, 0, 0);
+		}
+		regs->rax = ret;
+
+	} else {
+	        x86_saved_state32_t	*regs;
+		
+		regs = USER_REGS32(thr_act);
+
+		if (kdebug_enable && ((int) regs->eax < 0)) {
+		        /* Mach trap */
+		        KERNEL_DEBUG_CONSTANT(
+					      MACHDBG_CODE(DBG_MACH_EXCP_SC, -((int) regs->eax))
+					      | DBG_FUNC_END,
+					      ret, 0, 0, 0, 0);
+		}
+		regs->eax = ret;
+	}
         thread_exception_return();
         /*NOTREACHED*/
 }
@@ -147,27 +170,71 @@ extern boolean_t db_breakpoints_inserted;
 void
 thread_kdb_return(void)
 {
-	register thread_t	thread = current_thread();
-	register struct i386_saved_state *regs = USER_REGS(thread);
+	thread_t		thr_act = current_thread();
+	x86_saved_state_t	*iss = USER_STATE(thr_act);
 
-	if (kdb_trap(regs->trapno, regs->err, regs)) {
-#if		MACH_LDEBUG
-		assert(thread->mutex_count == 0); 
-#endif		/* MACH_LDEBUG */
-		thread_exception_return();
-		/*NOTREACHED*/
+        if (is_saved_state64(iss)) {
+	        x86_saved_state64_t	*regs;
+		
+		regs = saved_state64(iss);
+
+		if (kdb_trap(regs->isf.trapno, (int)regs->isf.err, (void *)regs)) {
+		        thread_exception_return();
+			/*NOTREACHED*/
+		}
+
+	} else {
+	        x86_saved_state32_t	*regs;
+		
+		regs = saved_state32(iss);
+
+		if (kdb_trap(regs->trapno, regs->err, (void *)regs)) {
+		        thread_exception_return();
+			/*NOTREACHED*/
+		}
 	}
 }
-boolean_t let_ddb_vm_fault = FALSE;
 
 #endif	/* MACH_KDB */
 
 void
 user_page_fault_continue(
-	kern_return_t	kr)
+			 kern_return_t	kr)
 {
-	register thread_t	thread = current_thread();
-	register struct i386_saved_state *regs = USER_REGS(thread);
+	thread_t	thread = current_thread();
+	x86_saved_state_t *regs = USER_STATE(thread);
+	ast_t		*myast;
+	boolean_t	intr;
+	user_addr_t	vaddr;
+#if	MACH_KDB
+	int		err;
+	int		trapno;
+#endif
+
+	assert((is_saved_state32(regs) && !thread_is_64bit(thread)) ||
+	       (is_saved_state64(regs) &&  thread_is_64bit(thread)));
+
+        if (thread_is_64bit(thread)) {
+	        x86_saved_state64_t	*uregs;
+
+		uregs = USER_REGS64(thread);
+
+#if	MACH_KDB
+		trapno = uregs->isf.trapno;
+		err = uregs->isf.err;
+#endif
+		vaddr = (user_addr_t)uregs->cr2;
+	} else {
+	        x86_saved_state32_t	*uregs;
+
+		uregs = USER_REGS32(thread);
+
+#if	MACH_KDB
+		trapno = uregs->trapno;
+		err = uregs->err;
+#endif
+		vaddr = uregs->cr2;
+	}
 
 	if ((kr == KERN_SUCCESS) || (kr == KERN_ABORTED)) {
 #if	MACH_KDB
@@ -176,28 +243,34 @@ user_page_fault_continue(
 		}
 		if (db_watchpoint_list &&
 		    db_watchpoints_inserted &&
-		    (regs->err & T_PF_WRITE) &&
+		    (err & T_PF_WRITE) &&
 		    db_find_watchpoint(thread->map,
-				       (vm_offset_t)regs->cr2,
+				       (vm_offset_t)vaddr,
 				       regs))
 			kdb_trap(T_WATCHPOINT, 0, regs);
 #endif	/* MACH_KDB */
+		intr = ml_set_interrupts_enabled(FALSE);
+		myast = ast_pending();
+		while (*myast & AST_ALL) {
+			ast_taken(AST_ALL, intr);
+			ml_set_interrupts_enabled(FALSE);
+			myast = ast_pending();
+		}
+		ml_set_interrupts_enabled(intr);
+
 		thread_exception_return();
 		/*NOTREACHED*/
 	}
 
 #if	MACH_KDB
 	if (debug_all_traps_with_kdb &&
-	    kdb_trap(regs->trapno, regs->err, regs)) {
-#if		MACH_LDEBUG
-		assert(thread->mutex_count == 0);
-#endif		/* MACH_LDEBUG */
+	    kdb_trap(trapno, err, regs)) {
 		thread_exception_return();
 		/*NOTREACHED*/
 	}
 #endif	/* MACH_KDB */
 
-	i386_exception(EXC_BAD_ACCESS, kr, regs->cr2);
+	i386_exception(EXC_BAD_ACCESS, kr, vaddr);
 	/*NOTREACHED*/
 }
 
@@ -212,56 +285,163 @@ struct recovery {
 extern struct recovery	recover_table[];
 extern struct recovery	recover_table_end[];
 
-/*
- * Recovery from Successful fault in copyout does not
- * return directly - it retries the pte check, since
- * the 386 ignores write protection in kernel mode.
- */
-extern struct recovery	retry_table[];
-extern struct recovery	retry_table_end[];
-
-const char *		trap_type[] = {TRAP_NAMES};
-int	TRAP_TYPES = sizeof(trap_type)/sizeof(trap_type[0]);
-
+const char *	trap_type[] = {TRAP_NAMES};
+unsigned 	TRAP_TYPES = sizeof(trap_type)/sizeof(trap_type[0]);
 
+static inline void
+reset_dr7(void)
+{
+	uint32_t dr7 = 0x400; /* magic dr7 reset value */
+	__asm__ volatile("movl %0,%%dr7" : : "r" (dr7));
+}
+#if MACH_KDP
+unsigned kdp_has_active_watchpoints = 0;
+#endif
 /*
  * Trap from kernel mode.  Only page-fault errors are recoverable,
  * and then only in special circumstances.  All other errors are
  * fatal.  Return value indicates if trap was handled.
  */
-boolean_t
+void
 kernel_trap(
-	register struct i386_saved_state	*regs)
+	x86_saved_state_t	*state)
 {
+	x86_saved_state32_t	*saved_state;
 	int			code;
-	unsigned int		subcode;
-	int			interruptible = THREAD_UNINT;
-	register int		type;
+	user_addr_t		vaddr;
+	int			type;
 	vm_map_t		map;
 	kern_return_t		result = KERN_FAILURE;
-	register thread_t	thread;
+	thread_t		thread;
+	ast_t			*myast;
+	boolean_t               intr;
+	vm_prot_t		prot;
+        struct recovery		*rp;
+	vm_offset_t		kern_ip;
+	int			fault_in_copy_window = -1;
+	int			is_user = 0;
+#if MACH_KDB	
+	pt_entry_t		*pte;
+#endif /* MACH_KDB */
 
-	type = regs->trapno;
-	code = regs->err;
 	thread = current_thread();
 
+	if (is_saved_state64(state))
+		panic("kernel_trap(%p) with 64-bit state", state);
+	saved_state = saved_state32(state);
+
+	vaddr = (user_addr_t)saved_state->cr2;
+	type  = saved_state->trapno;
+	code  = saved_state->err & 0xffff;
+	intr  = (saved_state->efl & EFL_IF) != 0;	/* state of ints at trap */
+
+	kern_ip = (vm_offset_t)saved_state->eip;
+
+	myast = ast_pending();
+
+	if (perfASTHook) {
+		if (*myast & AST_CHUD_ALL)
+			perfASTHook(type, NULL, 0, 0);
+	} else
+		*myast &= ~AST_CHUD_ALL;
+
+	/*
+	 * Is there a hook?
+	 */
+	if (perfTrapHook) {
+	        if (perfTrapHook(type, NULL, 0, 0) == KERN_SUCCESS) {
+		        /*
+			 * If it succeeds, we are done...
+			 */
+			return;
+		}
+	}
+	/*
+	 * we come here with interrupts off as we don't want to recurse
+	 * on preemption below.  but we do want to re-enable interrupts
+	 * as soon we possibly can to hold latency down
+	 */
+	if (T_PREEMPT == type) {
+
+	        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE,
+				      0, 0, 0, kern_ip, 0);
+
+	        ast_taken(AST_PREEMPTION, FALSE);
+		return;
+	}
+	
+	if (T_PAGE_FAULT == type) {
+		/*
+		 * assume we're faulting in the kernel map
+		 */
+		map = kernel_map;
+
+		if (thread != THREAD_NULL && thread->map != kernel_map) {
+		        vm_offset_t	copy_window_base;
+			vm_offset_t	kvaddr;
+			int		window_index;
+
+			kvaddr = (vm_offset_t)vaddr;
+		        /*
+			 * must determine if fault occurred in
+			 * the copy window while pre-emption is
+			 * disabled for this processor so that
+			 * we only need to look at the window
+			 * associated with this processor
+			 */
+		        copy_window_base = current_cpu_datap()->cpu_copywindow_base;
+
+			if (kvaddr >= copy_window_base && kvaddr < (copy_window_base + (NBPDE * NCOPY_WINDOWS)) ) {
+
+				window_index = (kvaddr - copy_window_base) / NBPDE;
+
+				if (thread->machine.copy_window[window_index].user_base != (user_addr_t)-1) {
+
+				        kvaddr -= (copy_window_base + (NBPDE * window_index));
+				        vaddr = thread->machine.copy_window[window_index].user_base + kvaddr;
+
+					map = thread->map;
+					fault_in_copy_window = window_index;
+				}
+				is_user = -1;
+			}
+		}
+	}
+	KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE,
+			      (int)(vaddr >> 32), (int)vaddr, is_user, kern_ip, 0);
+
+
+	(void) ml_set_interrupts_enabled(intr);
+
 	switch (type) {
-	    case T_PREEMPT:
-		ast_taken(AST_PREEMPTION, FALSE);
-		return (TRUE);
 
 	    case T_NO_FPU:
 		fpnoextflt();
-		return (TRUE);
+		return;
 
 	    case T_FPU_FAULT:
 		fpextovrflt();
-		return (TRUE);
+		return;
 
 	    case T_FLOATING_POINT_ERROR:
 		fpexterrflt();
-		return (TRUE);
+		return;
 
+	    case T_SSE_FLOAT_ERROR:
+	        fpSSEexterrflt();
+		return;
+ 	    case T_DEBUG:
+		    if ((saved_state->efl & EFL_TF) == 0
+		    && !kdp_has_active_watchpoints) {
+			    /* We've somehow encountered a debug
+			     * register match that does not belong
+			     * to the kernel debugger.
+			     * This isn't supposed to happen.
+			     */
+			    reset_dr7();
+			    return;
+			    }
+		    goto debugger_entry;
 	    case T_PAGE_FAULT:
 		/*
 		 * If the current map is a submap of the kernel map,
@@ -270,144 +450,116 @@ kernel_trap(
 		 * (vm_map_lookup), we may deadlock on the kernel map
 		 * lock.
 		 */
-#if	MACH_KDB
-		mp_disable_preemption();
-		if (db_active
-		    && kdb_active[cpu_number()]
-		    && !let_ddb_vm_fault) {
-			/*
-			 * Force kdb to handle this one.
-			 */
-			mp_enable_preemption();
-			return (FALSE);
-		}
-		mp_enable_preemption();
-#endif	/* MACH_KDB */
-		subcode = regs->cr2;	/* get faulting address */
 
-		if (subcode > LINEAR_KERNEL_ADDRESS) {
-		    map = kernel_map;
-		} else if (thread == THREAD_NULL)
-		    map = kernel_map;
-		else {
-		    map = thread->map;
-		}
+		prot = VM_PROT_READ;
+
+		if (code & T_PF_WRITE)
+		        prot |= VM_PROT_WRITE;
+#if     PAE
+		if (code & T_PF_EXECUTE)
+		        prot |= VM_PROT_EXECUTE;
+#endif
+
 #if	MACH_KDB
 		/*
 		 * Check for watchpoint on kernel static data.
 		 * vm_fault would fail in this case 
 		 */
-		if (map == kernel_map && 
-		    db_watchpoint_list &&
-		    db_watchpoints_inserted &&
-		    (code & T_PF_WRITE) &&
-		    (vm_offset_t)subcode < vm_last_phys &&
-		    ((*(pte = pmap_pte(kernel_pmap, (vm_offset_t)subcode))) &
-		     INTEL_PTE_WRITE) == 0) {
-		  *pte = *pte | INTEL_PTE_VALID | INTEL_PTE_WRITE; /* XXX need invltlb here? */
+		if (map == kernel_map && db_watchpoint_list && db_watchpoints_inserted &&
+		    (code & T_PF_WRITE) && vaddr < vm_map_max(map) &&
+		    ((*(pte = pmap_pte(kernel_pmap, (vm_map_offset_t)vaddr))) & INTEL_PTE_WRITE) == 0) {
+			pmap_store_pte(
+				pte,
+				*pte | INTEL_PTE_VALID | INTEL_PTE_WRITE);
+			/* XXX need invltlb here? */
+
 			result = KERN_SUCCESS;
-		} else
-#endif	/* MACH_KDB */
-		{
-		  	/*
-			 * Since the 386 ignores write protection in
-			 * kernel mode, always try for write permission
-			 * first.  If that fails and the fault was a
-			 * read fault, retry with read permission.
-			 */
-			if (map == kernel_map) {
-				register struct recovery *rp;
-
-				interruptible = THREAD_UNINT;
-				for (rp = recover_table; rp < recover_table_end; rp++) {
-					if (regs->eip == rp->fault_addr) {
-						interruptible = THREAD_ABORTSAFE;
-						break;
-					}
-				}
-			}
-		  	result = vm_fault(map,
-					  trunc_page((vm_offset_t)subcode),
-					  VM_PROT_READ|VM_PROT_WRITE,
-					  FALSE, 
-					  (map == kernel_map) ? interruptible : THREAD_ABORTSAFE, NULL, 0);
+			goto look_for_watchpoints;
 		}
+#endif	/* MACH_KDB */
+
+		result = vm_fault(map,
+				  vm_map_trunc_page(vaddr),
+				  prot,
+				  FALSE, 
+				  THREAD_UNINT, NULL, 0);
+
 #if	MACH_KDB
 		if (result == KERN_SUCCESS) {
-		    /* Look for watchpoints */
-		    if (db_watchpoint_list &&
-			db_watchpoints_inserted &&
-			(code & T_PF_WRITE) &&
-			db_find_watchpoint(map,
-				(vm_offset_t)subcode, regs))
-			kdb_trap(T_WATCHPOINT, 0, regs);
+		        /*
+			 * Look for watchpoints
+			 */
+look_for_watchpoints:
+		        if (map == kernel_map && db_watchpoint_list && db_watchpoints_inserted && (code & T_PF_WRITE) &&
+			    db_find_watchpoint(map, vaddr, saved_state))
+			        kdb_trap(T_WATCHPOINT, 0, saved_state);
 		}
-		else
 #endif	/* MACH_KDB */
-		if ((code & T_PF_WRITE) == 0 &&
-		    result == KERN_PROTECTION_FAILURE)
-		{
-		    /*
-		     *	Must expand vm_fault by hand,
-		     *	so that we can ask for read-only access
-		     *	but enter a (kernel)writable mapping.
-		     */
-		    result = intel_read_fault(map,
-					  trunc_page((vm_offset_t)subcode));
-		}
 
 		if (result == KERN_SUCCESS) {
-		    /*
-		     * Certain faults require that we back up
-		     * the EIP.
-		     */
-		    register struct recovery *rp;
-
-		    for (rp = retry_table; rp < retry_table_end; rp++) {
-			if (regs->eip == rp->fault_addr) {
-			    regs->eip = rp->recover_addr;
-			    break;
+
+			if (fault_in_copy_window != -1) {
+			        pt_entry_t	*updp;
+				pt_entry_t	*kpdp;
+
+				/*
+				 * in case there was no page table assigned
+				 * for the user base address and the pmap
+				 * got 'expanded' due to this fault, we'll
+				 * copy in the descriptor 
+				 *
+				 * we're either setting the page table descriptor
+				 * to the same value or it was 0... no need
+				 * for a TLB flush in either case
+				 */
+
+			        ml_set_interrupts_enabled(FALSE);
+			        updp = pmap_pde(map->pmap, thread->machine.copy_window[fault_in_copy_window].user_base);
+				assert(updp);
+				if (0 == updp) panic("trap: updp 0"); /* XXX DEBUG */
+				kpdp = current_cpu_datap()->cpu_copywindow_pdp;
+				kpdp += fault_in_copy_window;
+
+#if JOE_DEBUG
+				if (*kpdp && (*kpdp & PG_FRAME) != (*updp & PG_FRAME))
+				        panic("kernel_fault: user pdp doesn't match - updp = 0x%x, kpdp = 0x%x\n", updp, kpdp);
+#endif
+				pmap_store_pte(kpdp, *updp);
+
+				(void) ml_set_interrupts_enabled(intr);
 			}
-		    }
-		    return (TRUE);
+			return;
 		}
-
-		/* fall through */
+		/*
+		 * fall through
+		 */
 
 	    case T_GENERAL_PROTECTION:
-
 		/*
 		 * If there is a failure recovery address
 		 * for this fault, go there.
 		 */
-		{
-		    register struct recovery *rp;
-
-		    for (rp = recover_table;
-			 rp < recover_table_end;
-			 rp++) {
-			if (regs->eip == rp->fault_addr) {
-			    regs->eip = rp->recover_addr;
-			    return (TRUE);
+	        for (rp = recover_table; rp < recover_table_end; rp++) {
+		        if (kern_ip == rp->fault_addr) {
+			        set_recovery_ip(saved_state, rp->recover_addr);
+				return;
 			}
-		    }
 		}
 
 		/*
-		 * Check thread recovery address also -
-		 * v86 assist uses it.
+		 * Check thread recovery address also.
 		 */
 		if (thread->recover) {
-		    regs->eip = thread->recover;
-		    thread->recover = 0;
-		    return (TRUE);
+		        set_recovery_ip(saved_state, thread->recover);
+			thread->recover = 0;
+			return;
 		}
-
 		/*
 		 * Unanticipated page-fault errors in kernel
 		 * should not happen.
+		 *
+		 * fall through...
 		 */
-		/* fall through... */
 
 	    default:
 		/*
@@ -416,71 +568,345 @@ kernel_trap(
 		 */
 	    	if (type == 15) {
 			kprintf("kernel_trap() ignoring spurious trap 15\n"); 
-			return (TRUE);
+			return;
 		}
-
-		/*
-		 * ...and return failure, so that locore can call into
-		 * debugger.
+debugger_entry:
+		/* Ensure that the i386_kernel_state at the base of the
+		 * current thread's stack (if any) is synchronized with the
+		 * context at the moment of the trap, to facilitate
+		 * access through the debugger.
 		 */
+		sync_iss_to_iks(saved_state);
+#if MACH_KDB
+restart_debugger:
+#endif /* MACH_KDB */		
 #if  MACH_KDP
-		kdp_i386_trap(type, regs, result, regs->cr2);
-#endif
-		return (FALSE);
+                if (current_debugger != KDB_CUR_DB) {
+			if (kdp_i386_trap(type, saved_state, result, vaddr))
+				return;
+		}
+#endif /* MACH_KDP */
+#if MACH_KDB
+		else
+			if (kdb_trap(type, code, saved_state)) {
+				if (switch_debugger) {
+					current_debugger = KDP_CUR_DB;
+					switch_debugger = 0;
+					goto restart_debugger;
+				}
+				return;
+			}
+#endif /* MACH_KDB */
 	}
-	return (TRUE);
+
+	panic_trap(saved_state);
+	/*
+	 * NO RETURN
+	 */
+}
+
+
+static void
+set_recovery_ip(x86_saved_state32_t  *saved_state, vm_offset_t ip)
+{
+        saved_state->eip = ip;
+}
+
+
+static void
+panic_trap(x86_saved_state32_t *regs)
+{
+	const char *trapname = "Unknown";
+	uint32_t	cr0 = get_cr0();
+	uint32_t	cr2 = get_cr2();
+	uint32_t	cr3 = get_cr3();
+	uint32_t	cr4 = get_cr4();
+
+	panic_io_port_read();
+
+	kprintf("panic trap number 0x%x, eip 0x%x\n", regs->trapno, regs->eip);
+	kprintf("cr0 0x%08x cr2 0x%08x cr3 0x%08x cr4 0x%08x\n",
+		cr0, cr2, cr3, cr4);
+
+	if (regs->trapno < TRAP_TYPES)
+	        trapname = trap_type[regs->trapno];
+
+	panic("Unresolved kernel trap (CPU %d, Type %d=%s), registers:\n"
+	      "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
+	      "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
+	      "CR2: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
+	      "EFL: 0x%08x, EIP: 0x%08x, CS:  0x%08x, DS:  0x%08x\n",
+	      cpu_number(), regs->trapno, trapname, cr0, cr2, cr3, cr4,
+	      regs->eax,regs->ebx,regs->ecx,regs->edx,
+	      regs->cr2,regs->ebp,regs->esi,regs->edi,
+	      regs->efl,regs->eip,regs->cs, regs->ds);
+	/*
+	 * This next statement is not executed,
+	 * but it's needed to stop the compiler using tail call optimization
+	 * for the panic call - which confuses the subsequent backtrace.
+	 */
+	cr0 = 0;
 }
 
+extern void     kprintf_break_lock(void);
+
+
 /*
- * Called if both kernel_trap() and kdb_trap() fail.
+ * Called from locore on a special reserved stack after a double-fault
+ * is taken in kernel space.
+ * Kernel stack overflow is one route here.
  */
 void
-panic_trap(
-	register struct i386_saved_state	*regs)
+panic_double_fault(int code)
 {
-	int		code;
-	register int	type;
+	struct i386_tss *my_ktss = current_ktss();
+
+	/* Set postcode (DEBUG only) */
+	postcode(PANIC_DOUBLE_FAULT);
+
+/* Issue an I/O port read if one has been requested - this is an event logic
+ * analyzers can use as a trigger point.
+ */
+	panic_io_port_read();
+
+	/*
+	 * Break kprintf lock in case of recursion,
+	 * and record originally faulted instruction address.
+	 */
+	kprintf_break_lock();
 
-	type = regs->trapno;
-	code = regs->err;
+#if MACH_KDP
+	/*
+	 * Print backtrace leading to first fault:
+	 */
+	panic_i386_backtrace((void *) my_ktss->ebp, 10);
+#endif
 
-	printf("trap type %d, code = %x, pc = %x\n",
-		type, code, regs->eip);
-	panic("trap");
+	panic("Double fault (CPU:%d, thread:%p, code:0x%x),"
+	      "registers:\n"
+	      "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
+	      "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
+	      "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
+	      "EFL: 0x%08x, EIP: 0x%08x\n",
+	      cpu_number(), current_thread(), code,
+	      get_cr0(), get_cr2(), get_cr3(), get_cr4(),
+	      my_ktss->eax, my_ktss->ebx, my_ktss->ecx, my_ktss->edx,
+	      my_ktss->esp, my_ktss->ebp, my_ktss->esi, my_ktss->edi,
+	      my_ktss->eflags, my_ktss->eip);
 }
 
 
+/*
+ * Called from locore on a special reserved stack after a machine-check
+ */
+void
+panic_machine_check(int code)
+{
+	struct i386_tss *my_ktss = current_ktss();
+
+	/* Set postcode (DEBUG only) */
+	postcode(PANIC_MACHINE_CHECK);
+
+	/*
+	 * Break kprintf lock in case of recursion,
+	 * and record originally faulted instruction address.
+	 */
+	kprintf_break_lock();
+
+	/*
+	 * Dump the contents of the machine check MSRs (if any).
+	 */
+	mca_dump();
+
+	/*
+	 * And that's all folks, we don't attempt recovery...
+	 */
+	panic("Machine-check (CPU:%d, thread:%p, code:0x%x),"
+	      "registers:\n"
+	      "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
+	      "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
+	      "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
+	      "EFL: 0x%08x, EIP: 0x%08x\n",
+	      cpu_number(), current_thread(), code,
+	      get_cr0(), get_cr2(), get_cr3(), get_cr4(),
+	      my_ktss->eax, my_ktss->ebx, my_ktss->ecx, my_ktss->edx,
+	      my_ktss->esp, my_ktss->ebp, my_ktss->esi, my_ktss->edi,
+	      my_ktss->eflags, my_ktss->eip);
+}
+
+void
+panic_double_fault64(x86_saved_state_t *esp)
+{
+	/* Set postcode (DEBUG only) */
+	postcode(PANIC_DOUBLE_FAULT);
+
+	/*
+	 * Break kprintf lock in case of recursion,
+	 * and record originally faulted instruction address.
+	 */
+	kprintf_break_lock();
+
+	/*
+	 * Dump the interrupt stack frame at last kernel entry.
+	 */
+	if (is_saved_state64(esp)) {
+		x86_saved_state64_t	*ss64p = saved_state64(esp);
+		panic("Double fault (CPU:%d, thread:%p, trapno:0x%x, err:0x%qx),"
+		      "registers:\n"
+		      "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
+		      "RAX: 0x%016qx, RBX: 0x%016qx, RCX: 0x%016qx, RDX: 0x%016qx\n"
+		      "RSP: 0x%016qx, RBP: 0x%016qx, RSI: 0x%016qx, RDI: 0x%016qx\n"
+		      "R8:  0x%016qx, R9:  0x%016qx, R10: 0x%016qx, R11: 0x%016qx\n"
+		      "R12: 0x%016qx, R13: 0x%016qx, R14: 0x%016qx, R15: 0x%016qx\n"
+		      "RFL: 0x%016qx, RIP: 0x%016qx, CR2: 0x%016qx\n",
+		      cpu_number(), current_thread(), ss64p->isf.trapno, ss64p->isf.err,
+		      get_cr0(), get_cr2(), get_cr3(), get_cr4(),
+		      ss64p->rax, ss64p->rbx, ss64p->rcx, ss64p->rdx,
+		      ss64p->isf.rsp, ss64p->rbp, ss64p->rsi, ss64p->rdi,
+		      ss64p->r8, ss64p->r9, ss64p->r10, ss64p->r11,
+		      ss64p->r12, ss64p->r13, ss64p->r14, ss64p->r15,
+		      ss64p->isf.rflags, ss64p->isf.rip, ss64p->cr2);
+	} else {
+		x86_saved_state32_t	*ss32p = saved_state32(esp);
+		panic("Double fault (CPU:%d, thread:%p, trapno:0x%x, err:0x%x),"
+		      "registers:\n"
+		      "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
+		      "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
+		      "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
+		      "EFL: 0x%08x, EIP: 0x%08x\n",
+		      cpu_number(), current_thread(), ss32p->trapno, ss32p->err,
+		      get_cr0(), get_cr2(), get_cr3(), get_cr4(),
+		      ss32p->eax, ss32p->ebx, ss32p->ecx, ss32p->edx,
+		      ss32p->uesp, ss32p->ebp, ss32p->esi, ss32p->edi,
+		      ss32p->efl, ss32p->eip);
+	}
+}
+
+/*
+ * Machine check handler for 64-bit.
+ */
+void
+panic_machine_check64(x86_saved_state_t *esp)
+{
+	/* Set postcode (DEBUG only) */
+	postcode(PANIC_MACHINE_CHECK);
+
+	/*
+	 * Break kprintf lock in case of recursion,
+	 * and record originally faulted instruction address.
+	 */
+	kprintf_break_lock();
+
+	/*
+	 * Dump the contents of the machine check MSRs (if any).
+	 */
+	mca_dump();
+
+	/*
+	 * And that's all folks, we don't attempt recovery...
+	 */
+	if (is_saved_state64(esp)) {
+		x86_saved_state64_t	*ss64p = saved_state64(esp);
+		panic("Machine Check (CPU:%d, thread:%p, trapno:0x%x, err:0x%qx),"
+		      "registers:\n"
+		      "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
+		      "RAX: 0x%016qx, RBX: 0x%016qx, RCX: 0x%016qx, RDX: 0x%016qx\n"
+		      "RSP: 0x%016qx, RBP: 0x%016qx, RSI: 0x%016qx, RDI: 0x%016qx\n"
+		      "R8:  0x%016qx, R9:  0x%016qx, R10: 0x%016qx, R11: 0x%016qx\n"
+		      "R12: 0x%016qx, R13: 0x%016qx, R14: 0x%016qx, R15: 0x%016qx\n"
+		      "RFL: 0x%016qx, RIP: 0x%016qx\n",
+		      cpu_number(), current_thread(), ss64p->isf.trapno, ss64p->isf.err,
+		      get_cr0(), get_cr2(), get_cr3(), get_cr4(),
+		      ss64p->rax, ss64p->rbx, ss64p->rcx, ss64p->rdx,
+		      ss64p->isf.rsp, ss64p->rbp, ss64p->rsi, ss64p->rdi,
+		      ss64p->r8, ss64p->r9, ss64p->r10, ss64p->r11,
+		      ss64p->r12, ss64p->r13, ss64p->r14, ss64p->r15,
+		      ss64p->isf.rflags, ss64p->isf.rip);
+	} else {
+		x86_saved_state32_t	*ss32p = saved_state32(esp);
+		panic("Machine Check (CPU:%d, thread:%p, trapno:0x%x, err:0x%x),"
+		      "registers:\n"
+		      "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
+		      "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
+		      "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
+		      "EFL: 0x%08x, EIP: 0x%08x\n",
+		      cpu_number(), current_thread(), ss32p->trapno, ss32p->err,
+		      get_cr0(), get_cr2(), get_cr3(), get_cr4(),
+		      ss32p->eax, ss32p->ebx, ss32p->ecx, ss32p->edx,
+		      ss32p->uesp, ss32p->ebp, ss32p->esi, ss32p->edi,
+		      ss32p->efl, ss32p->eip);
+	}
+}
+
 /*
  *	Trap from user mode.
  */
 void
 user_trap(
-	register struct i386_saved_state	*regs)
+	x86_saved_state_t *saved_state)
 {
 	int		exc;
 	int		code;
+	int		err;
 	unsigned int	subcode;
-	register int	type;
-	vm_map_t	map;
+	int		type;
+	user_addr_t	vaddr;
 	vm_prot_t	prot;
-	kern_return_t	result;
 	thread_t	thread = current_thread();
-	boolean_t	kernel_act = FALSE;
-
-	if (regs->efl & EFL_VM) {
-	    /*
-	     * If hardware assist can handle exception,
-	     * continue execution.
-	     */
-	    if (v86_assist(thread, regs))
-		return;
+	ast_t		*myast;
+	boolean_t       intr;
+	kern_return_t	kret;
+	user_addr_t	rip;
+
+	assert((is_saved_state32(saved_state) && !thread_is_64bit(thread)) ||
+	       (is_saved_state64(saved_state) &&  thread_is_64bit(thread)));
+
+	if (is_saved_state64(saved_state)) {
+	        x86_saved_state64_t	*regs;
+
+		regs = saved_state64(saved_state);
+
+		type = regs->isf.trapno;
+		err  = regs->isf.err & 0xffff;
+		vaddr = (user_addr_t)regs->cr2;
+		rip   = (user_addr_t)regs->isf.rip;
+	} else {
+	        x86_saved_state32_t	*regs;
+
+		regs = saved_state32(saved_state);
+
+		type  = regs->trapno;
+		err   = regs->err & 0xffff;
+		vaddr = (user_addr_t)regs->cr2;
+		rip   = (user_addr_t)regs->eip;
 	}
 
-	type = regs->trapno;
+	KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_EXCP_UTRAP_x86, type)) | DBG_FUNC_NONE,
+			      (int)(vaddr>>32), (int)vaddr, (int)(rip>>32), (int)rip, 0);
+
 	code = 0;
 	subcode = 0;
 	exc = 0;
 
+#if DEBUG_TRACE
+	kprintf("user_trap(0x%08x) type=%d vaddr=0x%016llx\n",
+		saved_state, type, vaddr);
+#endif
+	myast = ast_pending();
+	if (perfASTHook) {
+		if (*myast & AST_CHUD_ALL) {
+			perfASTHook(type, saved_state, 0, 0);
+		}
+	} else {
+		*myast &= ~AST_CHUD_ALL;
+	}
+
+	/* Is there a hook? */
+	if (perfTrapHook) {
+		if (perfTrapHook(type, saved_state, 0, 0) == KERN_SUCCESS)
+			return;	/* If it succeeds, we are done... */
+	}
+
 	switch (type) {
 
 	    case T_DIVIDE_ERROR:
@@ -489,10 +915,37 @@ user_trap(
 		break;
 
 	    case T_DEBUG:
-		exc = EXC_BREAKPOINT;
-		code = EXC_I386_SGL;
-		break;
-
+		{
+			pcb_t	pcb;
+			unsigned int clear = 0;
+			/*
+			 * get dr6 and set it in the thread's pcb before
+			 * returning to userland
+			 */
+			pcb = thread->machine.pcb;
+			if (pcb->ids) {
+				/*
+				 * We can get and set the status register
+				 * in 32-bit mode even on a 64-bit thread
+				 * because the high order bits are not
+				 * used on x86_64
+				 */
+				if (thread_is_64bit(thread)) {
+					uint32_t dr6;
+					x86_debug_state64_t *ids = pcb->ids;
+					dr6 = (uint32_t)ids->dr6;
+					__asm__ volatile ("movl %%db6, %0" : "=r" (dr6));
+					ids->dr6 = dr6;
+				} else { /* 32 bit thread */
+					x86_debug_state32_t *ids = pcb->ids;
+					__asm__ volatile ("movl %%db6, %0" : "=r" (ids->dr6));
+				}
+				__asm__ volatile ("movl %0, %%db6" : : "r" (clear));
+			}
+			exc = EXC_BREAKPOINT;
+			code = EXC_I386_SGL;
+			break;
+		}
 	    case T_INT3:
 		exc = EXC_BREAKPOINT;
 		code = EXC_I386_BPT;
@@ -525,67 +978,50 @@ user_trap(
 	    case 10:		/* invalid TSS == iret with NT flag set */
 		exc = EXC_BAD_INSTRUCTION;
 		code = EXC_I386_INVTSSFLT;
-		subcode = regs->err & 0xffff;
+		subcode = err;
 		break;
 
 	    case T_SEGMENT_NOT_PRESENT:
 		exc = EXC_BAD_INSTRUCTION;
 		code = EXC_I386_SEGNPFLT;
-		subcode = regs->err & 0xffff;
+		subcode = err;
 		break;
 
 	    case T_STACK_FAULT:
 		exc = EXC_BAD_INSTRUCTION;
 		code = EXC_I386_STKFLT;
-		subcode = regs->err & 0xffff;
+		subcode = err;
 		break;
 
 	    case T_GENERAL_PROTECTION:
-		if (!(regs->efl & EFL_VM)) {
-		    if (check_io_fault(regs))
-			return;
-		}
 		exc = EXC_BAD_INSTRUCTION;
 		code = EXC_I386_GPFLT;
-		subcode = regs->err & 0xffff;
+		subcode = err;
 		break;
 
 	    case T_PAGE_FAULT:
-		subcode = regs->cr2;
-		prot = VM_PROT_READ|VM_PROT_WRITE;
-		if (kernel_act == FALSE) {
-			if (!(regs->err & T_PF_WRITE))
-				prot = VM_PROT_READ;
-			(void) user_page_fault_continue(vm_fault(thread->map,
-				trunc_page((vm_offset_t)subcode),
-				prot,
-				FALSE,
-				THREAD_ABORTSAFE, NULL, 0));
-			/* NOTREACHED */
-		}
-		else {
-			if (subcode > LINEAR_KERNEL_ADDRESS) {
-			  	map = kernel_map;
-			}
-			result = vm_fault(thread->map,
-				trunc_page((vm_offset_t)subcode),
-				prot,
-				FALSE,
-				(map == kernel_map) ? THREAD_UNINT : THREAD_ABORTSAFE, NULL, 0);
-			if ((result != KERN_SUCCESS) && (result != KERN_ABORTED)) {
-				/*
-				 * Must expand vm_fault by hand,
-				 * so that we can ask for read-only access
-				 * but enter a (kernel) writable mapping.
-				 */
-				result = intel_read_fault(thread->map,
-					trunc_page((vm_offset_t)subcode));
-			}
-			user_page_fault_continue(result);
-			/*NOTREACHED*/
-		}
+		prot = VM_PROT_READ;
+
+		if (err & T_PF_WRITE)
+		        prot |= VM_PROT_WRITE;
+#if     PAE
+		if (err & T_PF_EXECUTE)
+		        prot |= VM_PROT_EXECUTE;
+#endif
+		kret = vm_fault(thread->map, vm_map_trunc_page(vaddr),
+				 prot, FALSE,
+				 THREAD_ABORTSAFE, NULL, 0);
+
+	        user_page_fault_continue(kret);
+	
+		/* NOTREACHED */
 		break;
 
+	    case T_SSE_FLOAT_ERROR:
+	        fpSSEexterrflt();
+		return;
+
+
 	    case T_FLOATING_POINT_ERROR:
 		fpexterrflt();
 		return;
@@ -596,427 +1032,25 @@ user_trap(
 		return;
 #endif  /* MACH_KGDB */
 #if	MACH_KDB
-		if (kdb_trap(type, regs->err, regs))
+		if (kdb_trap(type, err, saved_state))
 		    return;
 #endif	/* MACH_KDB */
-		printf("user trap type %d, code = %x, pc = %x\n",
-		       type, regs->err, regs->eip);
 		panic("user trap");
 		return;
 	}
-
-#if	MACH_KDB
-	if (debug_all_traps_with_kdb &&
-	    kdb_trap(type, regs->err, regs))
-		return;
-#endif	/* MACH_KDB */
+	intr = ml_set_interrupts_enabled(FALSE);
+	myast = ast_pending();
+	while (*myast & AST_ALL) {
+	        ast_taken(AST_ALL, intr);
+		ml_set_interrupts_enabled(FALSE);
+		myast = ast_pending();
+	}
+	ml_set_interrupts_enabled(intr);
 
 	i386_exception(exc, code, subcode);
 	/*NOTREACHED*/
 }
 
-/*
- *	V86 mode assist for interrupt handling.
- */
-boolean_t v86_assist_on = TRUE;
-boolean_t v86_unsafe_ok = FALSE;
-boolean_t v86_do_sti_cli = TRUE;
-boolean_t v86_do_sti_immediate = FALSE;
-
-#define	V86_IRET_PENDING 0x4000
-
-int cli_count = 0;
-int sti_count = 0;
-
-boolean_t
-v86_assist(
-	thread_t				thread,
-	register struct i386_saved_state	*regs)
-{
-	register struct v86_assist_state *v86 = &thread->machine.pcb->ims.v86s;
-
-/*
- * Build an 8086 address.  Use only when off is known to be 16 bits.
- */
-#define	Addr8086(seg,off)	((((seg) & 0xffff) << 4) + (off))
-
-#define	EFL_V86_SAFE		(  EFL_OF | EFL_DF | EFL_TF \
-				 | EFL_SF | EFL_ZF | EFL_AF \
-				 | EFL_PF | EFL_CF )
-	struct iret_32 {
-		int		eip;
-		int		cs;
-		int		eflags;
-	};
-	struct iret_16 {
-		unsigned short	ip;
-		unsigned short	cs;
-		unsigned short	flags;
-	};
-	union iret_struct {
-		struct iret_32	iret_32;
-		struct iret_16	iret_16;
-	};
-
-	struct int_vec {
-		unsigned short	ip;
-		unsigned short	cs;
-	};
-
-	if (!v86_assist_on)
-	    return FALSE;
-
-	/*
-	 * If delayed STI pending, enable interrupts.
-	 * Turn off tracing if on only to delay STI.
-	 */
-	if (v86->flags & V86_IF_PENDING) {
-	    v86->flags &= ~V86_IF_PENDING;
-	    v86->flags |=  EFL_IF;
-	    if ((v86->flags & EFL_TF) == 0)
-		regs->efl &= ~EFL_TF;
-	}
-
-	if (regs->trapno == T_DEBUG) {
-
-	    if (v86->flags & EFL_TF) {
-		/*
-		 * Trace flag was also set - it has priority
-		 */
-		return FALSE;			/* handle as single-step */
-	    }
-	    /*
-	     * Fall through to check for interrupts.
-	     */
-	}
-	else if (regs->trapno == T_GENERAL_PROTECTION) {
-	    /*
-	     * General protection error - must be an 8086 instruction
-	     * to emulate.
-	     */
-	    register int	eip;
-	    boolean_t	addr_32 = FALSE;
-	    boolean_t	data_32 = FALSE;
-	    int		io_port;
-
-	    /*
-	     * Set up error handler for bad instruction/data
-	     * fetches.
-	     */
-	    __asm__("movl $(addr_error), %0" : : "m" (thread->recover));
-
-	    eip = regs->eip;
-	    while (TRUE) {
-		unsigned char	opcode;
-
-		if (eip > 0xFFFF) {
-		    thread->recover = 0;
-		    return FALSE;	/* GP fault: IP out of range */
-		}
-
-		opcode = *(unsigned char *)Addr8086(regs->cs,eip);
-		eip++;
-		switch (opcode) {
-		    case 0xf0:		/* lock */
-		    case 0xf2:		/* repne */
-		    case 0xf3:		/* repe */
-		    case 0x2e:		/* cs */
-		    case 0x36:		/* ss */
-		    case 0x3e:		/* ds */
-		    case 0x26:		/* es */
-		    case 0x64:		/* fs */
-		    case 0x65:		/* gs */
-			/* ignore prefix */
-			continue;
-
-		    case 0x66:		/* data size */
-			data_32 = TRUE;
-			continue;
-
-		    case 0x67:		/* address size */
-			addr_32 = TRUE;
-			continue;
-
-		    case 0xe4:		/* inb imm */
-		    case 0xe5:		/* inw imm */
-		    case 0xe6:		/* outb imm */
-		    case 0xe7:		/* outw imm */
-			io_port = *(unsigned char *)Addr8086(regs->cs, eip);
-			eip++;
-			goto do_in_out;
-
-		    case 0xec:		/* inb dx */
-		    case 0xed:		/* inw dx */
-		    case 0xee:		/* outb dx */
-		    case 0xef:		/* outw dx */
-		    case 0x6c:		/* insb */
-		    case 0x6d:		/* insw */
-		    case 0x6e:		/* outsb */
-		    case 0x6f:		/* outsw */
-			io_port = regs->edx & 0xffff;
-
-		    do_in_out:
-			if (!data_32)
-			    opcode |= 0x6600;	/* word IO */
-
-			switch (emulate_io(regs, opcode, io_port)) {
-			    case EM_IO_DONE:
-				/* instruction executed */
-				break;
-			    case EM_IO_RETRY:
-				/* port mapped, retry instruction */
-				thread->recover = 0;
-				return TRUE;
-			    case EM_IO_ERROR:
-				/* port not mapped */
-				thread->recover = 0;
-				return FALSE;
-			}
-			break;
-
-		    case 0xfa:		/* cli */
-			if (!v86_do_sti_cli) {
-			    thread->recover = 0;
-			    return (FALSE);
-			}
-
-			v86->flags &= ~EFL_IF;
-					/* disable simulated interrupts */
-			cli_count++;
-			break;
-
-		    case 0xfb:		/* sti */
-			if (!v86_do_sti_cli) {
-			    thread->recover = 0;
-			    return (FALSE);
-			}
-
-			if ((v86->flags & EFL_IF) == 0) {
-			    if (v86_do_sti_immediate) {
-				    v86->flags |= EFL_IF;
-			    } else {
-				    v86->flags |= V86_IF_PENDING;
-				    regs->efl |= EFL_TF;
-			    }
-					/* single step to set IF next inst. */
-			}
-			sti_count++;
-			break;
-
-		    case 0x9c:		/* pushf */
-		    {
-			int		flags;
-			vm_offset_t	sp;
-			unsigned int	size;
-
-			flags = regs->efl;
-			if ((v86->flags & EFL_IF) == 0)
-			    flags &= ~EFL_IF;
-
-			if ((v86->flags & EFL_TF) == 0)
-			    flags &= ~EFL_TF;
-			else flags |= EFL_TF;
-
-			sp = regs->uesp;
-			if (!addr_32)
-			    sp &= 0xffff;
-			else if (sp > 0xffff)
-			    goto stack_error;
-			size = (data_32) ? 4 : 2;
-			if (sp < size)
-			    goto stack_error;
-			sp -= size;
-			if (copyout((char *)&flags,
-				    (user_addr_t)Addr8086(regs->ss,sp),
-				    size))
-			    goto addr_error;
-			if (addr_32)
-			    regs->uesp = sp;
-			else
-			    regs->uesp = (regs->uesp & 0xffff0000) | sp;
-			break;
-		    }
-
-		    case 0x9d:		/* popf */
-		    {
-			vm_offset_t sp;
-			int	nflags;
-
-			sp = regs->uesp;
-			if (!addr_32)
-			    sp &= 0xffff;
-			else if (sp > 0xffff)
-			    goto stack_error;
-
-			if (data_32) {
-			    if (sp > 0xffff - sizeof(int))
-				goto stack_error;
-			    nflags = *(int *)Addr8086(regs->ss,sp);
-			    sp += sizeof(int);
-			}
-			else {
-			    if (sp > 0xffff - sizeof(short))
-				goto stack_error;
-			    nflags = *(unsigned short *)
-					Addr8086(regs->ss,sp);
-			    sp += sizeof(short);
-			}
-			if (addr_32)
-			    regs->uesp = sp;
-			else
-			    regs->uesp = (regs->uesp & 0xffff0000) | sp;
-
-			if (v86->flags & V86_IRET_PENDING) {
-				v86->flags = nflags & (EFL_TF | EFL_IF);
-				v86->flags |= V86_IRET_PENDING;
-			} else {
-				v86->flags = nflags & (EFL_TF | EFL_IF);
-			}
-			regs->efl = (regs->efl & ~EFL_V86_SAFE)
-				     | (nflags & EFL_V86_SAFE);
-			break;
-		    }
-		    case 0xcf:		/* iret */
-		    {
-			vm_offset_t sp;
-			int	nflags;
-			union iret_struct iret_struct;
-
-			v86->flags &= ~V86_IRET_PENDING;
-			sp = regs->uesp;
-			if (!addr_32)
-			    sp &= 0xffff;
-			else if (sp > 0xffff)
-			    goto stack_error;
-
-			if (data_32) {
-			    if (sp > 0xffff - sizeof(struct iret_32))
-				goto stack_error;
-			    iret_struct.iret_32 =
-				*(struct iret_32 *) Addr8086(regs->ss,sp);
-			    sp += sizeof(struct iret_32);
-			}
-			else {
-			    if (sp > 0xffff - sizeof(struct iret_16))
-				goto stack_error;
-			    iret_struct.iret_16 =
-				*(struct iret_16 *) Addr8086(regs->ss,sp);
-			    sp += sizeof(struct iret_16);
-			}
-			if (addr_32)
-			    regs->uesp = sp;
-			else
-			    regs->uesp = (regs->uesp & 0xffff0000) | sp;
-
-			if (data_32) {
-			    eip	      = iret_struct.iret_32.eip;
-			    regs->cs  = iret_struct.iret_32.cs & 0xffff;
-			    nflags    = iret_struct.iret_32.eflags;
-			}
-			else {
-			    eip       = iret_struct.iret_16.ip;
-			    regs->cs  = iret_struct.iret_16.cs;
-			    nflags    = iret_struct.iret_16.flags;
-			}
-
-			v86->flags = nflags & (EFL_TF | EFL_IF);
-			regs->efl = (regs->efl & ~EFL_V86_SAFE)
-				     | (nflags & EFL_V86_SAFE);
-			break;
-		    }
-		    default:
-			/*
-			 * Instruction not emulated here.
-			 */
-			thread->recover = 0;
-			return FALSE;
-		}
-		break;	/* exit from 'while TRUE' */
-	    }
-	    regs->eip = (regs->eip & 0xffff0000) | eip;
-	}
-	else {
-	    /*
-	     * Not a trap we handle.
-	     */
-	    thread->recover = 0;
-	    return FALSE;
-	}
-
-	if ((v86->flags & EFL_IF) && ((v86->flags & V86_IRET_PENDING)==0)) {
-
-	    struct v86_interrupt_table *int_table;
-	    int int_count;
-	    int vec;
-	    int i;
-
-	    int_table = (struct v86_interrupt_table *) v86->int_table;
-	    int_count = v86->int_count;
-
-	    vec = 0;
-	    for (i = 0; i < int_count; int_table++, i++) {
-		if (!int_table->mask && int_table->count > 0) {
-		    int_table->count--;
-		    vec = int_table->vec;
-		    break;
-		}
-	    }
-	    if (vec != 0) {
-		/*
-		 * Take this interrupt
-		 */
-		vm_offset_t	sp;
-		struct iret_16 iret_16;
-		struct int_vec int_vec;
-
-		sp = regs->uesp & 0xffff;
-		if (sp < sizeof(struct iret_16))
-		    goto stack_error;
-		sp -= sizeof(struct iret_16);
-		iret_16.ip = regs->eip;
-		iret_16.cs = regs->cs;
-		iret_16.flags = regs->efl & 0xFFFF;
-		if ((v86->flags & EFL_TF) == 0)
-		    iret_16.flags &= ~EFL_TF;
-		else iret_16.flags |= EFL_TF;
-
-		(void) memcpy((char *) &int_vec, 
-			      (char *) (sizeof(struct int_vec) * vec),
-		      	      sizeof (struct int_vec));
-		if (copyout((char *)&iret_16,
-			    (user_addr_t)Addr8086(regs->ss,sp),
-			    sizeof(struct iret_16)))
-		    goto addr_error;
-		regs->uesp = (regs->uesp & 0xFFFF0000) | (sp & 0xffff);
-		regs->eip = int_vec.ip;
-		regs->cs  = int_vec.cs;
-		regs->efl  &= ~EFL_TF;
-		v86->flags &= ~(EFL_IF | EFL_TF);
-		v86->flags |= V86_IRET_PENDING;
-	    }
-	}
-
-	thread->recover = 0;
-	return TRUE;
-
-	/*
-	 *	On address error, report a page fault.
-	 *	XXX report GP fault - we don`t save
-	 *	the faulting address.
-	 */
-    addr_error:
-	__asm__("addr_error:;");
-	thread->recover = 0;
-	return FALSE;
-
-	/*
-	 *	On stack address error, return stack fault (12).
-	 */
-    stack_error:
-	thread->recover = 0;
-	regs->trapno = T_STACK_FAULT;
-	return FALSE;
-}
 
 /*
  * Handle AST traps for i386.
@@ -1029,45 +1063,17 @@ extern void     log_thread_action (thread_t, char *);
 void
 i386_astintr(int preemption)
 {
-	ast_t		*my_ast, mask = AST_ALL;
+	ast_t		mask = AST_ALL;
 	spl_t		s;
 
-	s = splsched();		/* block interrupts to check reasons */
-	mp_disable_preemption();
-	my_ast = ast_pending();
-	if (*my_ast & AST_I386_FP) {
-	    /*
-	     * AST was for delayed floating-point exception -
-	     * FP interrupt occurred while in kernel.
-	     * Turn off this AST reason and handle the FPU error.
-	     */
-
-	    ast_off(AST_I386_FP);
-	    mp_enable_preemption();
-	    splx(s);
-
-	    fpexterrflt();
-	}
-	else {
-	    /*
-	     * Not an FPU trap.  Handle the AST.
-	     * Interrupts are still blocked.
-	     */
-
-#if 1
-	    if (preemption) {
-		mask = AST_PREEMPTION;
-		mp_enable_preemption();
-	    } else {
-		mp_enable_preemption();
-	    }
-#else
-	mp_enable_preemption();
-#endif
+	if (preemption)
+	        mask = AST_PREEMPTION;
+
+	s = splsched();
 
 	ast_taken(mask, s);
 
-	}
+	splx(s);
 }
 
 /*
@@ -1086,133 +1092,68 @@ i386_exception(
 	int	code,
 	int	subcode)
 {
-	spl_t			s;
 	exception_data_type_t   codes[EXCEPTION_CODE_MAX];
 
-	/*
-	 * Turn off delayed FPU error handling.
-	 */
-	s = splsched();
-	mp_disable_preemption();
-	ast_off(AST_I386_FP);
-	mp_enable_preemption();
-	splx(s);
-
 	codes[0] = code;		/* new exception interface */
 	codes[1] = subcode;
 	exception_triage(exc, codes, 2);
 	/*NOTREACHED*/
 }
 
-boolean_t
-check_io_fault(
-	struct i386_saved_state		*regs)
+
+void
+kernel_preempt_check(void)
 {
-	int		eip, opcode, io_port;
-	boolean_t	data_16 = FALSE;
+	ast_t		*myast;
+	boolean_t	intr;
 
 	/*
-	 * Get the instruction.
+	 * disable interrupts to both prevent pre-emption
+	 * and to keep the ast state from changing via
+	 * an interrupt handler making something runnable
 	 */
-	eip = regs->eip;
-
-	for (;;) {
-	    opcode = inst_fetch(eip, regs->cs);
-	    eip++;
-	    switch (opcode) {
-		case 0x66:	/* data-size prefix */
-		    data_16 = TRUE;
-		    continue;
-
-		case 0xf3:	/* rep prefix */
-		case 0x26:	/* es */
-		case 0x2e:	/* cs */
-		case 0x36:	/* ss */
-		case 0x3e:	/* ds */
-		case 0x64:	/* fs */
-		case 0x65:	/* gs */
-		    continue;
-
-		case 0xE4:	/* inb imm */
-		case 0xE5:	/* inl imm */
-		case 0xE6:	/* outb imm */
-		case 0xE7:	/* outl imm */
-		    /* port is immediate byte */
-		    io_port = inst_fetch(eip, regs->cs);
-		    eip++;
-		    break;
-
-		case 0xEC:	/* inb dx */
-		case 0xED:	/* inl dx */
-		case 0xEE:	/* outb dx */
-		case 0xEF:	/* outl dx */
-		case 0x6C:	/* insb */
-		case 0x6D:	/* insl */
-		case 0x6E:	/* outsb */
-		case 0x6F:	/* outsl */
-		    /* port is in DX register */
-		    io_port = regs->edx & 0xFFFF;
-		    break;
-
-		default:
-		    return FALSE;
-	    }
-	    break;
-	}
-
-	if (data_16)
-	    opcode |= 0x6600;		/* word IO */
+	intr = ml_set_interrupts_enabled(FALSE);
 
-	switch (emulate_io(regs, opcode, io_port)) {
-	    case EM_IO_DONE:
-		/* instruction executed */
-		regs->eip = eip;
-		return TRUE;
-
-	    case EM_IO_RETRY:
-		/* port mapped, retry instruction */
-		return TRUE;
-
-	    case EM_IO_ERROR:
-		/* port not mapped */
-		return FALSE;
-	}
-	return FALSE;
-}
+	myast = ast_pending();
 
-void
-kernel_preempt_check (void)
-{
-	ast_t		*myast;
+	if ((*myast & AST_URGENT) && intr == TRUE && get_interrupt_level() == 0) {
+	        /*
+		 * can handle interrupts and preemptions 
+		 * at this point
+		 */
+	        ml_set_interrupts_enabled(intr);
 
-	mp_disable_preemption();
-	myast = ast_pending();
-        if ((*myast & AST_URGENT) &&
-	    get_interrupt_level() == 1
-	    ) {
-		mp_enable_preemption_no_check();
-                __asm__ volatile ("     int     $0xff");
-        } else {
-		mp_enable_preemption_no_check();
+		/*
+		 * now cause the PRE-EMPTION trap
+		 */
+		__asm__ volatile ("     int     $0xff");
+	} else {
+	        /*
+		 * if interrupts were already disabled or
+		 * we're in an interrupt context, we can't
+		 * preempt...  of course if AST_URGENT
+		 * isn't set we also don't want to
+		 */
+	        ml_set_interrupts_enabled(intr);
 	}
 }
 
 #if	MACH_KDB
 
-extern void 	db_i386_state(struct i386_saved_state *regs);
+extern void 	db_i386_state(x86_saved_state32_t *regs);
 
 #include <ddb/db_output.h>
 
 void 
 db_i386_state(
-	struct i386_saved_state *regs)
+	x86_saved_state32_t *regs)
 {
   	db_printf("eip	%8x\n", regs->eip);
   	db_printf("trap	%8x\n", regs->trapno);
   	db_printf("err	%8x\n", regs->err);
   	db_printf("efl	%8x\n", regs->efl);
   	db_printf("ebp	%8x\n", regs->ebp);
-  	db_printf("esp	%8x\n", regs->esp);
+  	db_printf("esp	%8x\n", regs->cr2);
   	db_printf("uesp	%8x\n", regs->uesp);
   	db_printf("cs	%8x\n", regs->cs & 0xff);
   	db_printf("ds	%8x\n", regs->ds & 0xff);
@@ -1229,3 +1170,85 @@ db_i386_state(
 }
 
 #endif	/* MACH_KDB */
+
+/* Synchronize a thread's i386_kernel_state (if any) with the given
+ * i386_saved_state_t obtained from the trap/IPI handler; called in
+ * kernel_trap() prior to entering the debugger, and when receiving
+ * an "MP_KDP" IPI.
+ */
+  
+void
+sync_iss_to_iks(x86_saved_state32_t *saved_state)
+{
+	struct x86_kernel_state32 *iks;
+	vm_offset_t kstack;
+	boolean_t record_active_regs = FALSE;
+
+	if ((kstack = current_thread()->kernel_stack) != 0) {
+	        x86_saved_state32_t	*regs;
+
+		regs = saved_state;
+
+		iks = STACK_IKS(kstack);
+
+		 /*
+		  * Did we take the trap/interrupt in kernel mode?
+		  */
+		if (regs == USER_REGS32(current_thread()))
+		        record_active_regs = TRUE;
+		else {
+		        iks->k_ebx = regs->ebx;
+			iks->k_esp = (int)regs;
+			iks->k_ebp = regs->ebp;
+			iks->k_edi = regs->edi;
+			iks->k_esi = regs->esi;
+			iks->k_eip = regs->eip;
+		}
+	}
+
+	if (record_active_regs == TRUE) {
+		/*
+		 * Show the trap handler path
+		 */
+		__asm__ volatile("movl %%ebx, %0" : "=m" (iks->k_ebx));
+		__asm__ volatile("movl %%esp, %0" : "=m" (iks->k_esp));
+		__asm__ volatile("movl %%ebp, %0" : "=m" (iks->k_ebp));
+		__asm__ volatile("movl %%edi, %0" : "=m" (iks->k_edi));
+		__asm__ volatile("movl %%esi, %0" : "=m" (iks->k_esi));
+		/*
+		 * "Current" instruction pointer
+		 */
+		__asm__ volatile("movl $1f, %0\n1:" : "=m" (iks->k_eip));
+	}
+}
+
+/*
+ * This is used by the NMI interrupt handler (from mp.c) to
+ * uncondtionally sync the trap handler context to the IKS
+ * irrespective of whether the NMI was fielded in kernel
+ * or user space.
+ */
+void
+sync_iss_to_iks_unconditionally(__unused x86_saved_state32_t *saved_state) {
+	struct x86_kernel_state32 *iks;
+	vm_offset_t kstack;
+	boolean_t record_active_regs = FALSE;
+
+	if ((kstack = current_thread()->kernel_stack) != 0) {
+
+		iks = STACK_IKS(kstack);
+				/*
+		 * Show the trap handler path
+		 */
+		__asm__ volatile("movl %%ebx, %0" : "=m" (iks->k_ebx));
+		__asm__ volatile("movl %%esp, %0" : "=m" (iks->k_esp));
+		__asm__ volatile("movl %%ebp, %0" : "=m" (iks->k_ebp));
+		__asm__ volatile("movl %%edi, %0" : "=m" (iks->k_edi));
+		__asm__ volatile("movl %%esi, %0" : "=m" (iks->k_esi));
+		/*
+		 * "Current" instruction pointer
+		 */
+		__asm__ volatile("movl $1f, %0\n1:" : "=m" (iks->k_eip));
+
+	}
+}
diff --git a/osfmk/i386/trap.h b/osfmk/i386/trap.h
index ba6443341..bd0e637e8 100644
--- a/osfmk/i386/trap.h
+++ b/osfmk/i386/trap.h
@@ -80,6 +80,8 @@
 /*				15 */
 #define	T_FLOATING_POINT_ERROR	16
 #define	T_WATCHPOINT		17
+#define T_MACHINE_CHECK		18
+#define T_SSE_FLOAT_ERROR       19
 #define T_PREEMPT		255
 
 #define TRAP_NAMES "divide error", "debug trap", "NMI", "breakpoint", \
@@ -87,7 +89,7 @@
 		   "no coprocessor", "double fault", "coprocessor overrun", \
 		   "invalid TSS", "segment not present", "stack bounds", \
 		   "general protection", "page fault", "(reserved)", \
-		   "coprocessor error", "watchpoint"
+		   "coprocessor error", "watchpoint", "(reserved)", "SSE floating point"
 
 /*
  * Page-fault trap codes.
@@ -96,6 +98,12 @@
 #define	T_PF_WRITE		0x2		/* write access */
 #define	T_PF_USER		0x4		/* from user state */
 
+#ifdef PAE
+#define	T_PF_RSVD		0x8		/* reserved bit set to 1 */
+#define T_PF_EXECUTE		0x10		/* instruction fetch when NX */
+#endif
+
+
 #if !defined(ASSEMBLER) && defined(MACH_KERNEL)
 
 #include <i386/thread.h>
@@ -105,23 +113,42 @@ extern void		i386_exception(
 				int		code,
 				int		subcode);
 
-extern boolean_t	kernel_trap(
-				struct i386_saved_state	*regs);
+extern void		sync_iss_to_iks(x86_saved_state32_t *regs);
+
+extern void		sync_iss_to_iks_unconditionally(x86_saved_state32_t *regs);
+
+extern void		kernel_trap(x86_saved_state_t *regs);
+
+extern void		user_trap(x86_saved_state_t *regs);
 
-extern void		panic_trap(
-				struct i386_saved_state	*regs);
+extern void		panic_double_fault(int code);
 
-extern void		user_trap(
-				struct i386_saved_state	*regs);
+extern void		panic_double_fault64(x86_saved_state_t *regs);
+
+extern void		panic_machine_check(int	code);
+
+extern void		panic_machine_check64(x86_saved_state_t *regs);
 
 extern void		i386_astintr(int preemption);
 
-#if defined(MACH_KDP)
-extern void		kdp_i386_trap(
+
+typedef kern_return_t (*perfCallback)(
+				int			trapno,
+				void			*regs,
+				int			unused1,
+				int			unused2);
+
+extern perfCallback perfTrapHook;
+extern perfCallback perfASTHook;
+extern perfCallback perfIntHook;
+
+#if MACH_KDP
+extern boolean_t	kdp_i386_trap(
 				unsigned int,
-				struct i386_saved_state *,
+				x86_saved_state32_t *,
 				kern_return_t,
 				vm_offset_t);
+extern void		panic_i386_backtrace(void *, int);
 #endif /* MACH_KDP */
 #endif	/* !ASSEMBLER && MACH_KERNEL */
 
diff --git a/osfmk/i386/tsc.c b/osfmk/i386/tsc.c
new file mode 100644
index 000000000..7ef04fb9d
--- /dev/null
+++ b/osfmk/i386/tsc.c
@@ -0,0 +1,221 @@
+/*
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+/*
+ *	File:		i386/tsc.c
+ *	Purpose:	Initializes the TSC and the various conversion
+ *			factors needed by other parts of the system.
+ */
+
+#include <platforms.h>
+#include <mach_kdb.h>
+
+#include <mach/mach_types.h>
+
+#include <kern/cpu_data.h>
+#include <kern/cpu_number.h>
+#include <kern/clock.h>
+#include <kern/host_notify.h>
+#include <kern/macro_help.h>
+#include <kern/misc_protos.h>
+#include <kern/spl.h>
+#include <kern/assert.h>
+#include <mach/vm_prot.h>
+#include <vm/pmap.h>
+#include <vm/vm_kern.h>		/* for kernel_map */
+#include <i386/ipl.h>
+#include <i386/pit.h>
+#include <architecture/i386/pio.h>
+#include <i386/misc_protos.h>
+#include <i386/proc_reg.h>
+#include <i386/machine_cpu.h>
+#include <i386/mp.h>
+#include <i386/cpuid.h>
+#include <i386/cpu_data.h>
+#include <i386/cpu_threads.h>
+#include <i386/perfmon.h>
+#include <i386/machine_routines.h>
+#include <pexpert/pexpert.h>
+#include <machine/limits.h>
+#include <machine/commpage.h>
+#include <sys/kdebug.h>
+#include <pexpert/device_tree.h>
+#include <i386/tsc.h>
+
+uint64_t	busFCvtt2n = 0;
+uint64_t	busFCvtn2t = 0;
+uint64_t	tscFreq = 0;
+uint64_t	tscFCvtt2n = 0;
+uint64_t	tscFCvtn2t = 0;
+uint64_t	tscGranularity = 0;
+uint64_t	bus2tsc = 0;
+
+/* Decimal powers: */
+#define kilo (1000ULL)
+#define Mega (kilo * kilo)
+#define Giga (kilo * Mega)
+#define Tera (kilo * Giga)
+#define Peta (kilo * Tera)
+
+static const char	FSB_Frequency_prop[] = "FSBFrequency";
+/*
+ * This routine extracts the front-side bus frequency in Hz from
+ * the device tree.
+ */
+static uint64_t
+EFI_FSB_frequency(void)
+{
+	uint64_t	frequency = 0;
+	DTEntry		entry;
+	void		*value;
+	int		size;
+
+	if (DTLookupEntry(0, "/efi/platform", &entry) != kSuccess) {
+		kprintf("EFI_FSB_frequency: didn't find /efi/platform\n");
+		return 0;
+	}
+	if (DTGetProperty(entry,FSB_Frequency_prop,&value,&size) != kSuccess) {
+		kprintf("EFI_FSB_frequency: property %s not found\n");
+		return 0;
+	}
+	if (size == sizeof(uint64_t)) {
+		frequency = *(uint64_t *) value;
+		kprintf("EFI_FSB_frequency: read %s value: %llu\n",
+			FSB_Frequency_prop, frequency);
+		if (!(90*Mega < frequency && frequency < 10*Giga)) {
+			kprintf("EFI_FSB_frequency: value out of range\n");
+			frequency = 0;
+		}
+	} else {
+		kprintf("EFI_FSB_frequency: unexpected size %d\n", size);
+	}
+	return frequency;
+}
+
+/*
+ * Initialize the various conversion factors needed by code referencing
+ * the TSC.
+ */
+void
+tsc_init(void)
+{
+    uint64_t	busFreq;
+    uint64_t	busFCvtInt;
+    uint32_t	cpuModel;
+    uint32_t	cpuFamily;
+    uint32_t	xcpuid[4];
+
+    /*
+     * Get the FSB frequency and conversion factors.
+     */
+    busFreq = EFI_FSB_frequency();
+    if (busFreq != 0) {
+	busFCvtt2n = ((1 * Giga) << 32) / busFreq;
+	busFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / busFCvtt2n;
+	busFCvtInt = tmrCvt(1 * Peta, 0xFFFFFFFFFFFFFFFFULL / busFreq); 
+    } else {
+	panic("rtclock_init: EFI not supported!\n");
+    }
+	
+    kprintf(" BUS: Frequency = %6d.%04dMHz, "
+	    "cvtt2n = %08X.%08X, cvtn2t = %08X.%08X, "
+	    "cvtInt = %08X.%08X\n",
+	    (uint32_t)(busFreq / Mega),
+	    (uint32_t)(busFreq % Mega), 
+	    (uint32_t)(busFCvtt2n >> 32), (uint32_t)busFCvtt2n,
+	    (uint32_t)(busFCvtn2t >> 32), (uint32_t)busFCvtn2t,
+	    (uint32_t)(busFCvtInt >> 32), (uint32_t)busFCvtInt);
+
+    do_cpuid(1, xcpuid);
+    cpuFamily = ( xcpuid[eax] >> 8 ) & 0xf;
+    /*
+     * Get the extended family if necessary.
+     */
+    if (cpuFamily == 0x0f)
+	cpuFamily += (xcpuid[eax] >> 20) & 0x00ff;
+
+    cpuModel = ( xcpuid[eax] >> 4 ) & 0xf;
+    /*
+     * Get the extended model if necessary.
+     */
+    if (cpuFamily == CPUID_FAMILY_686
+	|| cpuFamily == CPUID_FAMILY_EXTENDED)
+	cpuModel += ((xcpuid[eax] >> 16) & 0xf) << 4;
+
+    /*
+     * Get the TSC increment.  The TSC is incremented by this
+     * on every bus tick.  Calculate the TSC conversion factors
+     * to and from nano-seconds.
+     */
+    if (cpuFamily == CPUID_FAMILY_686) {
+	if (cpuModel == CPUID_MODEL_CORE || cpuModel == CPUID_MODEL_CORE2) {
+	uint64_t	prfsts;
+	
+	prfsts = rdmsr64(IA32_PERF_STS);
+	tscGranularity = (uint32_t)(prfsts >> BusRatioShift) & BusRatioMask;
+	} else {
+	    panic("rtclock_init: unknown CPU model: 0x%X\n",
+	      cpuModel);
+	}
+    } else {
+	panic("rtclock_init: unknown CPU family: 0x%X\n",
+	      cpuFamily);
+    }
+	
+    tscFCvtt2n = busFCvtt2n / (uint64_t)tscGranularity;
+    tscFreq = ((1 * Giga)  << 32) / tscFCvtt2n;
+    tscFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / tscFCvtt2n;
+
+    kprintf(" TSC: Frequency = %6d.%04dMHz, "
+	    "cvtt2n = %08X.%08X, cvtn2t = %08X.%08X, gran = %d\n",
+	    (uint32_t)(tscFreq / Mega),
+	    (uint32_t)(tscFreq % Mega), 
+	    (uint32_t)(tscFCvtt2n >> 32), (uint32_t)tscFCvtt2n,
+	    (uint32_t)(tscFCvtn2t >> 32), (uint32_t)tscFCvtn2t,
+	    tscGranularity);
+
+    /*
+     * Calculate conversion from BUS to TSC
+     */
+    bus2tsc = tmrCvt(busFCvtt2n, tscFCvtn2t);
+}
+
+void
+tsc_get_info(tscInfo_t *info)
+{
+    info->busFCvtt2n     = busFCvtt2n;
+    info->busFCvtn2t     = busFCvtn2t;
+    info->tscFreq        = tscFreq;
+    info->tscFCvtt2n     = tscFCvtt2n;
+    info->tscFCvtn2t     = tscFCvtn2t;
+    info->tscGranularity = tscGranularity;
+    info->bus2tsc        = bus2tsc;
+}
diff --git a/osfmk/i386/tsc.h b/osfmk/i386/tsc.h
new file mode 100644
index 000000000..df66a83c0
--- /dev/null
+++ b/osfmk/i386/tsc.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+/*
+ * @APPLE_FREE_COPYRIGHT@
+ */
+/*
+ *	File:		tsc.h
+ *	Purpose:	Contains the TSC initialization and conversion
+ *			factors.
+ */
+#ifdef KERNEL_PRIVATE
+#ifndef _I386_TSC_H_
+#define _I386_TSC_H_
+
+#define BusRatioShift 40
+#define BusRatioMask 0x1F
+#define IA32_PERF_STS 0x198
+
+extern uint64_t	busFCvtt2n;
+extern uint64_t	busFCvtn2t;
+extern uint64_t tscFreq;
+extern uint64_t tscFCvtt2n;
+extern uint64_t tscFCvtn2t;
+extern uint64_t tscGranularity;
+extern uint64_t bus2tsc;
+
+struct tscInfo
+{
+uint64_t	busFCvtt2n;
+uint64_t	busFCvtn2t;
+uint64_t	tscFreq;
+uint64_t	tscFCvtt2n;
+uint64_t	tscFCvtn2t;
+uint64_t	tscGranularity;
+uint64_t	bus2tsc;
+};
+typedef struct tscInfo tscInfo_t;
+
+extern void tsc_get_info(tscInfo_t *info);
+
+extern void tsc_init(void);
+
+#endif /* _I386_TSC_H_ */
+#endif /* KERNEL_PRIVATE */
diff --git a/osfmk/i386/tss.h b/osfmk/i386/tss.h
index 2ea97230d..f421b27a1 100644
--- a/osfmk/i386/tss.h
+++ b/osfmk/i386/tss.h
@@ -59,41 +59,75 @@
 #ifndef	_I386_TSS_H_
 #define	_I386_TSS_H_
 
+#include <stdint.h>
+
 /*
  *	i386 Task State Segment
  */
 struct i386_tss {
-	int		back_link;	/* segment number of previous task,
+	uint32_t	back_link;	/* segment number of previous task,
 					   if nested */
-	int		esp0;		/* initial stack pointer ... */
-	int		ss0;		/* and segment for ring 0 */
-	int		esp1;		/* initial stack pointer ... */
-	int		ss1;		/* and segment for ring 1 */
-	int		esp2;		/* initial stack pointer ... */
-	int		ss2;		/* and segment for ring 2 */
-	int		cr3;		/* CR3 - page table directory
+	uint32_t	esp0;		/* initial stack pointer ... */
+	uint32_t	ss0;		/* and segment for ring 0 */
+	uint32_t	esp1;		/* initial stack pointer ... */
+	uint32_t	ss1;		/* and segment for ring 1 */
+	uint32_t	esp2;		/* initial stack pointer ... */
+	uint32_t	ss2;		/* and segment for ring 2 */
+	uint32_t	cr3;		/* CR3 - page table directory
 						 physical address */
-	int		eip;
-	int		eflags;
-	int		eax;
-	int		ecx;
-	int		edx;
-	int		ebx;
-	int		esp;		/* current stack pointer */
-	int		ebp;
-	int		esi;
-	int		edi;
-	int		es;
-	int		cs;
-	int		ss;		/* current stack segment */
-	int		ds;
-	int		fs;
-	int		gs;
-	int		ldt;		/* local descriptor table segment */
-	unsigned short	trace_trap;	/* trap on switch to this task */
-	unsigned short	io_bit_map_offset;
+	uint32_t	eip;
+	uint32_t	eflags;
+	uint32_t	eax;
+	uint32_t	ecx;
+	uint32_t	edx;
+	uint32_t	ebx;
+	uint32_t	esp;		/* current stack pointer */
+	uint32_t	ebp;
+	uint32_t	esi;
+	uint32_t	edi;
+	uint32_t	es;
+	uint32_t	cs;
+	uint32_t	ss;		/* current stack segment */
+	uint32_t	ds;
+	uint32_t	fs;
+	uint32_t	gs;
+	uint32_t	ldt;		/* local descriptor table segment */
+	uint16_t	trace_trap;	/* trap on switch to this task */
+	uint16_t	io_bit_map_offset;
 					/* offset to start of IO permission
 					   bit map */
 };
 
+/*
+ * Temporary stack used on kernel entry via the sysenter instruction.
+ * Its top points on to the PCB save area. It must contain space for
+ * a single interrupt stack frame in case of single-stepping over the sysenter.
+ * Although this is defined as a 64-bit stack, the space is also used in
+ * 32-bit legacy mode. For 64-bit the stack is 16-byte aligned.
+ */
+struct sysenter_stack {
+	uint64_t	stack[16];	/* Space for a 64-bit frame and some */
+	uint64_t	top;		/* Top and pointer to ISS in PCS */
+};
+
+struct x86_64_tss {
+	uint32_t	reserved1;
+	uint64_t	rsp0;		/* stack pointer for CPL0 */
+	uint64_t	rsp1;		/* stack pointer for CPL1 */
+	uint64_t	rsp2;		/* stack pointer for CPL2 */
+	uint32_t	reserved2;
+	uint32_t	reserved3;
+	uint64_t	ist1;		/* interrupt stack table 1 */
+	uint64_t	ist2;		/* interrupt stack table 2 */
+	uint64_t	ist3;		/* interrupt stack table 3 */
+	uint64_t	ist4;		/* interrupt stack table 4 */
+	uint64_t	ist5;		/* interrupt stack table 5 */
+	uint64_t	ist6;		/* interrupt stack table 6 */
+	uint64_t	ist7;		/* interrupt stack table 7 */
+	uint32_t	reserved4;
+	uint32_t	reserved5;
+	uint16_t	reserved6;
+	uint16_t	io_bit_map_offset;
+					/* offset to IO permission bit map */
+};
 #endif	/* _I386_TSS_H_ */
diff --git a/osfmk/i386/user_ldt.c b/osfmk/i386/user_ldt.c
index a21105f9b..afb9795a3 100644
--- a/osfmk/i386/user_ldt.c
+++ b/osfmk/i386/user_ldt.c
@@ -59,7 +59,7 @@
 
 /*
  * User LDT management.
- * Each thread in a task may have its own LDT.
+ * Each task may have its own LDT.
  */
 
 #include <kern/kalloc.h>
@@ -71,366 +71,328 @@
 #include <i386/seg.h>
 #include <i386/thread.h>
 #include <i386/user_ldt.h>
+#include <i386/mp_desc.h>
+#include <i386/proc_reg.h>
+#include <i386/machdep_call.h>
+#include <i386/mp.h>
+#include <i386/machine_routines.h>
 
-char	acc_type[8][3] = {
-    /*	code	stack	data */
-    {	0,	0,	1	},	/* data */
-    {	0,	1,	1	},	/* data, writable */
-    {	0,	0,	1	},	/* data, expand-down */
-    {	0,	1,	1	},	/* data, writable, expand-down */
-    {	1,	0,	0	},	/* code */
-    {	1,	0,	1	},	/* code, readable */
-    {	1,	0,	0	},	/* code, conforming */
-    {	1,	0,	1	},	/* code, readable, conforming */
-};
-
-#if 0
-/* Forward */
-
-extern boolean_t	selector_check(
-				thread_t		thread,
-				int			sel,
-				int			type);
-
-boolean_t
-selector_check(
-	thread_t		thread,
-	int			sel,
-	int			type)
-{
-	struct user_ldt	*ldt;
-	int	access;
-
-	ldt = thread->machine.pcb->ims.ldt;
-	if (ldt == 0) {
-	    switch (type) {
-		case S_CODE:
-		    return sel == USER_CS;
-		case S_STACK:
-		    return sel == USER_DS;
-		case S_DATA:
-		    return sel == 0 ||
-			   sel == USER_CS ||
-			   sel == USER_DS;
-	    }
-	}
+#include <sys/errno.h>
 
-	if (type != S_DATA && sel == 0)
-	    return FALSE;
-	if ((sel & (SEL_LDTS|SEL_PL)) != (SEL_LDTS|SEL_PL_U)
-	  || sel > ldt->desc.limit_low)
-		return FALSE;
-
-	access = ldt->ldt[sel_idx(sel)].access;
-	
-	if ((access & (ACC_P|ACC_PL|ACC_TYPE_USER))
-		!= (ACC_P|ACC_PL_U|ACC_TYPE_USER))
-	    return FALSE;
-		/* present, pl == pl.user, not system */
-
-	return acc_type[(access & 0xe)>>1][type];
-}
+static void user_ldt_set_action(void *);
 
 /*
  * Add the descriptors to the LDT, starting with
  * the descriptor for 'first_selector'.
  */
 
-kern_return_t
+int
 i386_set_ldt(
-	thread_t		thr_act,
-	int			first_selector,
-	descriptor_list_t	desc_list,
-	mach_msg_type_number_t	count)
+	int			*retval,
+	uint32_t		start_sel,
+	uint32_t		descs,	/* out */
+	uint32_t		num_sels)
 {
-	user_ldt_t	new_ldt, old_ldt, temp;
+	user_ldt_t	new_ldt, old_ldt;
 	struct real_descriptor *dp;
-	int		i;
-	int		min_selector = 0;
-	pcb_t		pcb;
-	vm_size_t	ldt_size_needed;
-	int		first_desc = sel_idx(first_selector);
-	vm_map_copy_t	old_copy_object;
-	thread_t		thread;
-
-	if (first_desc < min_selector || first_desc > 8191)
-	    return KERN_INVALID_ARGUMENT;
-	if (first_desc + count >= 8192)
-	    return KERN_INVALID_ARGUMENT;
-	if (thr_act == THREAD_NULL)
-	    return KERN_INVALID_ARGUMENT;
-	if ((thread = act_lock_thread(thr_act)) == THREAD_NULL) {
-		act_unlock_thread(thr_act);
-	    return KERN_INVALID_ARGUMENT;
-	}
-	if (thread == current_thread())
-		min_selector = LDTSZ;
-	act_unlock_thread(thr_act);
+	unsigned int	i;
+	unsigned int	min_selector = LDTSZ_MIN;	/* do not allow the system selectors to be changed */
+	task_t		task = current_task();
+	unsigned int	ldt_count;
+	kern_return_t err;
 
-	/*
-	 * We must copy out desc_list to the kernel map, and wire
-	 * it down (we touch it while the PCB is locked).
-	 *
-	 * We make a copy of the copyin object, and clear
-	 * out the old one, so that the MIG stub will have a
-	 * a empty (but valid) copyin object to discard.
-	 */
-	{
-	    kern_return_t	kr;
-	    vm_map_offset_t	dst_addr;
-
-	    old_copy_object = (vm_map_copy_t) desc_list;
-
-	    kr = vm_map_copyout(ipc_kernel_map, &dst_addr,
-				vm_map_copy_copy(old_copy_object));
-	    if (kr != KERN_SUCCESS)
-		return kr;
-
-	    (void) vm_map_wire(ipc_kernel_map,
-			vm_map_trunc_page(dst_addr),
-			vm_map_round_page(dst_addr + 
-				count * sizeof(struct real_descriptor)),
-			VM_PROT_READ|VM_PROT_WRITE, FALSE);
-	    desc_list = CAST_DOWN(descriptor_list_t, dst_addr);
-	}
+	if (start_sel != LDT_AUTO_ALLOC
+	    && (start_sel != 0 || num_sels != 0)
+	    && (start_sel < min_selector || start_sel >= LDTSZ))
+	    return EINVAL;
+	if (start_sel != LDT_AUTO_ALLOC
+	    && start_sel + num_sels > LDTSZ)
+	    return EINVAL;
 
-	for (i = 0, dp = (struct real_descriptor *) desc_list;
-	     i < count;
-	     i++, dp++)
-	{
-	    switch (dp->access & ~ACC_A) {
-		case 0:
-		case ACC_P:
-		    /* valid empty descriptor */
-		    break;
-		case ACC_P | ACC_CALL_GATE:
-		    /* Mach kernel call */
-		    *dp = *(struct real_descriptor *)
-				&ldt[sel_idx(USER_SCALL)];
-		    break;
-		case ACC_P | ACC_PL_U | ACC_DATA:
-		case ACC_P | ACC_PL_U | ACC_DATA_W:
-		case ACC_P | ACC_PL_U | ACC_DATA_E:
-		case ACC_P | ACC_PL_U | ACC_DATA_EW:
-		case ACC_P | ACC_PL_U | ACC_CODE:
-		case ACC_P | ACC_PL_U | ACC_CODE_R:
-		case ACC_P | ACC_PL_U | ACC_CODE_C:
-		case ACC_P | ACC_PL_U | ACC_CODE_CR:
-		case ACC_P | ACC_PL_U | ACC_CALL_GATE_16:
-		case ACC_P | ACC_PL_U | ACC_CALL_GATE:
-		    break;
-		default:
-		    (void) vm_map_remove(ipc_kernel_map, 
-					 vm_map_trunc_page(desc_list),
-					 vm_map_round_page(&desc_list[count]),
-					 VM_MAP_REMOVE_KUNWIRE);
-		    return KERN_INVALID_ARGUMENT;
+	task_lock(task);
+	
+	old_ldt = task->i386_ldt;
+
+	if (start_sel == LDT_AUTO_ALLOC) {
+	    if (old_ldt) {
+		unsigned int null_count;
+		struct real_descriptor null_ldt;
+		
+		bzero(&null_ldt, sizeof(null_ldt));
+
+		/*
+		 * Look for null selectors among the already-allocated
+		 * entries.
+		 */
+		null_count = 0;
+		i = 0;
+		while (i < old_ldt->count)
+		{
+		    if (!memcmp(&old_ldt->ldt[i++], &null_ldt, sizeof(null_ldt))) {
+			null_count++;
+			if (null_count == num_sels)
+			    break;  /* break out of while loop */
+		    } else {
+			null_count = 0;
+		    }
+		}
+
+		/*
+		 * If we broke out of the while loop, i points to the selector
+		 * after num_sels null selectors.  Otherwise it points to the end
+		 * of the old LDTs, and null_count is the number of null selectors
+		 * at the end. 
+		 *
+		 * Either way, there are null_count null selectors just prior to
+		 * the i-indexed selector, and either null_count >= num_sels,
+		 * or we're at the end, so we can extend.
+		 */
+		start_sel = old_ldt->start + i - null_count;
+	    } else {
+		start_sel = LDTSZ_MIN;
+	    }
+		
+	    if (start_sel + num_sels > LDTSZ) {
+		task_unlock(task);
+		return ENOMEM;
 	    }
 	}
-	ldt_size_needed = sizeof(struct real_descriptor)
-			* (first_desc + count);
-
-	pcb = thr_act->machine.pcb;
-	new_ldt = 0;
-    Retry:
-	simple_lock(&pcb->lock);
-	old_ldt = pcb->ims.ldt;
-	if (old_ldt == 0 ||
-	    old_ldt->desc.limit_low + 1 < ldt_size_needed)
-	{
+
+	if (start_sel == 0 && num_sels == 0) {
+	    new_ldt = NULL;
+	} else {
 	    /*
-	     * No old LDT, or not big enough
+	     * Allocate new LDT
 	     */
-	    if (new_ldt == 0) {
-		simple_unlock(&pcb->lock);
-
-		new_ldt = (user_ldt_t) kalloc(ldt_size_needed
-					      + sizeof(struct real_descriptor));
-		new_ldt->desc.limit_low   = ldt_size_needed - 1;
-		new_ldt->desc.limit_high  = 0;
-		new_ldt->desc.base_low    = 
-				((vm_offset_t)&new_ldt->ldt[0]) & 0xffff;
-		new_ldt->desc.base_med    = 
-				(((vm_offset_t)&new_ldt->ldt[0]) >> 16)
-						 & 0xff;
-		new_ldt->desc.base_high   = 
-				((vm_offset_t)&new_ldt->ldt[0]) >> 24;
-		new_ldt->desc.access      = ACC_P | ACC_LDT;
-		new_ldt->desc.granularity = 0;
-
-		goto Retry;
+
+	    unsigned int    begin_sel = start_sel;
+	    unsigned int    end_sel = begin_sel + num_sels;
+	    
+	    if (old_ldt != NULL) {
+		if (old_ldt->start < begin_sel)
+		    begin_sel = old_ldt->start;
+		if (old_ldt->start + old_ldt->count > end_sel)
+		    end_sel = old_ldt->start + old_ldt->count;
+	    }
+
+	    ldt_count = end_sel - begin_sel;
+
+	    new_ldt = (user_ldt_t)kalloc(sizeof(struct user_ldt) + (ldt_count * sizeof(struct real_descriptor)));
+	    if (new_ldt == NULL) {
+		task_unlock(task);
+		return ENOMEM;
 	    }
 
+	    new_ldt->start = begin_sel;
+	    new_ldt->count = ldt_count;
+
 	    /*
 	     * Have new LDT.  If there was a an old ldt, copy descriptors
-	     * from old to new.  Otherwise copy the default ldt.
+	     * from old to new.
 	     */
 	    if (old_ldt) {
-		bcopy((char *)&old_ldt->ldt[0],
-		      (char *)&new_ldt->ldt[0],
-		      old_ldt->desc.limit_low + 1);
+		bcopy(&old_ldt->ldt[0],
+		      &new_ldt->ldt[old_ldt->start - begin_sel],
+		      old_ldt->count * sizeof(struct real_descriptor));
+
+		/*
+		 * If the old and new LDTs are non-overlapping, fill the 
+		 * center in with null selectors.
+		 */
+		 		 
+		if (old_ldt->start + old_ldt->count < start_sel)
+		    bzero(&new_ldt->ldt[old_ldt->count],
+			  (start_sel - (old_ldt->start + old_ldt->count)) * sizeof(struct real_descriptor));
+		else if (old_ldt->start > start_sel + num_sels)
+		    bzero(&new_ldt->ldt[num_sels],
+			  (old_ldt->start - (start_sel + num_sels)) * sizeof(struct real_descriptor));
 	    }
-	    else if (thr_act == current_thread()) {
-		struct real_descriptor template = {0, 0, 0, ACC_P, 0, 0 ,0};
-
-		for (dp = &new_ldt->ldt[0], i = 0; i < first_desc; i++, dp++) {
-		    if (i < LDTSZ)
-		    	*dp = *(struct real_descriptor *) &ldt[i];
-		    else
-			*dp = template;
+
+	    /*
+	     * Install new descriptors.
+	     */
+	    if (descs != 0) {
+		err = copyin(descs, (char *)&new_ldt->ldt[start_sel - begin_sel],
+			     num_sels * sizeof(struct real_descriptor));
+		if (err != 0)
+		{
+		    task_unlock(task);
+		    user_ldt_free(new_ldt);
+		    return err;
 		}
+	    } else {
+		bzero(&new_ldt->ldt[start_sel - begin_sel], num_sels * sizeof(struct real_descriptor));
 	    }
 
-	    temp = old_ldt;
-	    old_ldt = new_ldt;	/* use new LDT from now on */
-	    new_ldt = temp;	/* discard old LDT */
-
-	    pcb->ims.ldt = old_ldt;	/* new LDT for thread */
+	    /*
+	     * Validate descriptors.
+	     * Only allow descriptors with user priviledges.
+	     */
+	    for (i = 0, dp = (struct real_descriptor *) &new_ldt->ldt[start_sel - begin_sel];
+		 i < num_sels;
+		 i++, dp++)
+	    {
+		switch (dp->access & ~ACC_A) {
+		    case 0:
+		    case ACC_P:
+			/* valid empty descriptor */
+			break;
+		    case ACC_P | ACC_PL_U | ACC_DATA:
+		    case ACC_P | ACC_PL_U | ACC_DATA_W:
+		    case ACC_P | ACC_PL_U | ACC_DATA_E:
+		    case ACC_P | ACC_PL_U | ACC_DATA_EW:
+		    case ACC_P | ACC_PL_U | ACC_CODE:
+		    case ACC_P | ACC_PL_U | ACC_CODE_R:
+		    case ACC_P | ACC_PL_U | ACC_CODE_C:
+		    case ACC_P | ACC_PL_U | ACC_CODE_CR:
+		    case ACC_P | ACC_PL_U | ACC_CALL_GATE_16:
+		    case ACC_P | ACC_PL_U | ACC_CALL_GATE:
+			break;
+		    default:
+			task_unlock(task);
+			user_ldt_free(new_ldt);
+			return EACCES;
+		}
+	    }
 	}
 
+	task->i386_ldt = new_ldt; /* new LDT for task */
+
 	/*
-	 * Install new descriptors.
+	 * Switch to new LDT.  We need to do this on all CPUs, since
+	 * another thread in this same task may be currently running,
+	 * and we need to make sure the new LDT is in place
+	 * throughout the task before returning to the user.
 	 */
-	bcopy((char *)desc_list,
-	      (char *)&old_ldt->ldt[first_desc],
-	      count * sizeof(struct real_descriptor));
+	mp_rendezvous_no_intrs(user_ldt_set_action, task);
 
-	simple_unlock(&pcb->lock);
+	task_unlock(task);
 
-	if (new_ldt)
-	    kfree((vm_offset_t)new_ldt,
-		  new_ldt->desc.limit_low+1+sizeof(struct real_descriptor));
-
-	/*
-	 * Free the descriptor list.
+	/* free old LDT.  We can't do this until after we've
+	 * rendezvoused with all CPUs, in case another thread
+	 * in this task was in the process of context switching.
 	 */
-	(void) vm_map_remove(ipc_kernel_map, vm_map_trunc_page(desc_list),
-			     vm_map_round_page(&desc_list[count]),
-			     VM_MAP_REMOVE_KUNWIRE);
-	return KERN_SUCCESS;
+	if (old_ldt)
+	    user_ldt_free(old_ldt);
+
+	*retval = start_sel;
+
+	return 0;
 }
 
-kern_return_t
+int
 i386_get_ldt(
-	thread_t		thr_act,
-	int			first_selector,
-	int			selector_count,	/* number wanted */
-	descriptor_list_t	*desc_list,	/* in/out */
-	mach_msg_type_number_t	*count)		/* in/out */
+	int			*retval,
+	uint32_t		start_sel,
+	uint32_t		descs,	/* out */
+	uint32_t		num_sels)
 {
-	struct user_ldt *user_ldt;
-	pcb_t		pcb = thr_act->machine.pcb;
-	int		first_desc = sel_idx(first_selector);
+	user_ldt_t	user_ldt;
+	task_t		task = current_task();
 	unsigned int	ldt_count;
-	vm_size_t	ldt_size;
-	vm_size_t	size, size_needed;
-	vm_offset_t	addr;
-	thread_t		thread;
-
-	if (thr_act == THREAD_NULL)
-	    return KERN_INVALID_ARGUMENT;
-
-	if (first_desc < 0 || first_desc > 8191)
-	    return KERN_INVALID_ARGUMENT;
-	if (first_desc + selector_count >= 8192)
-	    return KERN_INVALID_ARGUMENT;
-
-	addr = 0;
-	size = 0;
-
-	for (;;) {
-	    simple_lock(&pcb->lock);
-	    user_ldt = pcb->ims.ldt;
-	    if (user_ldt == 0) {
-		simple_unlock(&pcb->lock);
-		if (addr)
-		    kmem_free(ipc_kernel_map, addr, size);
-		*count = 0;
-		return KERN_SUCCESS;
-	    }
+	kern_return_t	err;
 
-	    /*
-	     * Find how many descriptors we should return.
-	     */
-	    ldt_count = (user_ldt->desc.limit_low + 1) /
-			sizeof (struct real_descriptor);
-	    ldt_count -= first_desc;
-	    if (ldt_count > selector_count)
-		ldt_count = selector_count;
+	if (start_sel >= 8192)
+	    return EINVAL;
+	if (start_sel + num_sels > 8192)
+	    return EINVAL;
+	if (descs == 0)
+	    return EINVAL;
 
-	    ldt_size = ldt_count * sizeof(struct real_descriptor);
+	task_lock(task);
 
-	    /*
-	     * Do we have the memory we need?
-	     */
-	    if (ldt_count <= *count)
-		break;		/* fits in-line */
+	user_ldt = task->i386_ldt;
+	err = 0;
 
-	    size_needed = round_page(ldt_size);
-	    if (size_needed <= size)
-		break;
+	/*
+	 * copy out the descriptors
+	 */
+
+	if (user_ldt != 0)
+	    ldt_count = user_ldt->start + user_ldt->count;
+	else
+	    ldt_count = LDTSZ_MIN;
 
-	    /*
-	     * Unlock the pcb and allocate more memory
-	     */
-	    simple_unlock(&pcb->lock);
 
-	    if (size != 0)
-		kmem_free(ipc_kernel_map, addr, size);
+	if (start_sel < ldt_count)
+	{
+	    unsigned int copy_sels = num_sels;
 
-	    size = size_needed;
+	    if (start_sel + num_sels > ldt_count)
+		copy_sels = ldt_count - start_sel;
 
-	    if (kmem_alloc(ipc_kernel_map, &addr, size)
-			!= KERN_SUCCESS)
-		return KERN_RESOURCE_SHORTAGE;
+	    err = copyout((char *)(current_ldt() + start_sel),
+			  descs, copy_sels * sizeof(struct real_descriptor));
 	}
 
-	/*
-	 * copy out the descriptors
-	 */
-	bcopy((char *)&user_ldt->ldt[first_desc],
-	      (char *)addr,
-	      ldt_size);
-	*count = ldt_count;
-	simple_unlock(&pcb->lock);
+	task_unlock(task);
 
-	if (addr) {
-	    vm_size_t		size_used, size_left;
-	    vm_map_copy_t	memory;
+	*retval = ldt_count;
 
-	    /*
-	     * Free any unused memory beyond the end of the last page used
-	     */
-	    size_used = round_page(ldt_size);
-	    if (size_used != size)
-		kmem_free(ipc_kernel_map,
-			addr + size_used, size - size_used);
+	return err;
+}
 
-	    /*
-	     * Zero the remainder of the page being returned.
-	     */
-	    size_left = size_used - ldt_size;
-	    if (size_left > 0)
-		bzero((char *)addr + ldt_size, size_left);
+void
+user_ldt_free(
+	user_ldt_t	user_ldt)
+{
+	kfree(user_ldt, sizeof(struct user_ldt) + (user_ldt->count * sizeof(struct real_descriptor)));
+}
 
-	    /*
-	     * Unwire the memory and make it into copyin form.
-	     */
-	    (void) vm_map_unwire(ipc_kernel_map, vm_map_trunc_page(addr),
-				 vm_map_round_page(addr + size_used), FALSE);
-	    (void) vm_map_copyin(ipc_kernel_map, (vm_map_address_t)addr, 
-				 (vm_map_size_t)size_used, TRUE, &memory);
-	    *desc_list = (descriptor_list_t) memory;
+user_ldt_t
+user_ldt_copy(
+	user_ldt_t	user_ldt)
+{
+	if (user_ldt != NULL) {
+	    size_t	size = sizeof(struct user_ldt) + (user_ldt->count * sizeof(struct real_descriptor));
+	    user_ldt_t	new_ldt = (user_ldt_t)kalloc(size);
+	    if (new_ldt != NULL)
+		bcopy(user_ldt, new_ldt, size);
+	    return new_ldt;
 	}
+	
+	return 0;
+}
 
-	return KERN_SUCCESS;
+void
+user_ldt_set_action(
+	void *arg)
+{
+	task_t		arg_task = (task_t)arg;
+
+	if (arg_task == current_task())	{
+	    user_ldt_set(current_thread());
+	}
 }
 
-#endif 
+/*
+ * Set the LDT for the given thread on the current CPU.  Should be invoked
+ * with interrupts disabled.
+ */
 void
-user_ldt_free(
-	user_ldt_t	user_ldt)
+user_ldt_set(
+	thread_t thread)
 {
-	kfree(user_ldt,
-		user_ldt->desc.limit_low+1+sizeof(struct real_descriptor));
+        task_t		task = thread->task;
+	user_ldt_t	user_ldt;
+
+	user_ldt = task->i386_ldt;
+
+	if (user_ldt != 0) {
+	    struct real_descriptor *ldtp = (struct real_descriptor *)current_ldt();
+
+	    if (user_ldt->start > LDTSZ_MIN) {
+		bzero(&ldtp[LDTSZ_MIN],
+		      sizeof(struct real_descriptor) * (user_ldt->start - LDTSZ_MIN));
+	    }
+	    
+	    bcopy(user_ldt->ldt, &ldtp[user_ldt->start],
+		  sizeof(struct real_descriptor) * (user_ldt->count));
+
+	    gdt_desc_p(USER_LDT)->limit_low = (sizeof(struct real_descriptor) * (user_ldt->start + user_ldt->count)) - 1;
+
+	    ml_cpu_set_ldt(USER_LDT);
+	} else {
+	    ml_cpu_set_ldt(KERNEL_LDT);
+	}
 }
diff --git a/osfmk/i386/user_ldt.h b/osfmk/i386/user_ldt.h
index 4295d939e..e83c16dca 100644
--- a/osfmk/i386/user_ldt.h
+++ b/osfmk/i386/user_ldt.h
@@ -63,29 +63,36 @@
 /*
  * User LDT management.
  *
- * Each thread in a task may have its own LDT.
+ * Each task may have its own LDT.
  */
 
+#define	LDT_AUTO_ALLOC	0xffffffff
+
+#ifdef KERNEL
 #include <i386/seg.h>
 
 struct user_ldt {
-	struct real_descriptor	desc;	/* descriptor for self */
-	struct real_descriptor	ldt[1];	/* descriptor table (variable) */
+	unsigned int start;		/* first descriptor in table */
+	unsigned int count;		/* how many descriptors in table */
+	struct real_descriptor	ldt[0];	/* descriptor table (variable) */
 };
 typedef struct user_ldt *	user_ldt_t;
 
-/*
- * Check code/stack/data selector values against LDT if present.
- */
-#define	S_CODE	0		/* code segment */
-#define	S_STACK	1		/* stack segment */
-#define	S_DATA	2		/* data segment */
-
-extern boolean_t selector_check(
-			thread_t	thread,
-			int		sel,
-			int		type);
+extern user_ldt_t	user_ldt_copy(
+			user_ldt_t	uldt);
 extern void	user_ldt_free(
 			user_ldt_t	uldt);
+extern void	user_ldt_set(
+			thread_t	thread);
+#else /* !KERNEL */
+#include <sys/cdefs.h>
+
+union ldt_entry;
+
+__BEGIN_DECLS
+int i386_get_ldt(int, union ldt_entry *, int);
+int i386_set_ldt(int, const union ldt_entry *, int);
+__END_DECLS
+#endif /* KERNEL */
 
 #endif	/* _I386_USER_LDT_H_ */
diff --git a/osfmk/ipc/ipc_object.c b/osfmk/ipc/ipc_object.c
index 6abf8eecf..424a17a7f 100644
--- a/osfmk/ipc/ipc_object.c
+++ b/osfmk/ipc/ipc_object.c
@@ -1047,10 +1047,13 @@ const char *ikot_print_array[IKOT_MAX_TYPE] = {
 	"(IOKIT_CONNECT)    ",
 	"(IOKIT_OBJECT)     ",	/* 30 */
 	"(UPL)              ",
-						/* << new entries here	*/
+	"(MEM_OBJ_CONTROL)  ",
+/*
+ * Add new entries here.
+ * Please keep in sync with kern/ipc_kobject.h
+ */
 	"(UNKNOWN)          "	/* magic catchall	*/
 };
-/* Please keep in sync with kern/ipc_kobject.h	*/
 
 void
 ipc_object_print(
diff --git a/osfmk/ipc/ipc_port.c b/osfmk/ipc/ipc_port.c
index 93d0b9bbd..e3f4267d4 100644
--- a/osfmk/ipc/ipc_port.c
+++ b/osfmk/ipc/ipc_port.c
@@ -1506,15 +1506,32 @@ print_ports(void)
 	PRINT_ONE_PORT_TYPE(PROCESSOR);
 	PRINT_ONE_PORT_TYPE(PSET);
 	PRINT_ONE_PORT_TYPE(PSET_NAME);
+	PRINT_ONE_PORT_TYPE(TIMER);
 	PRINT_ONE_PORT_TYPE(PAGING_REQUEST);
-	PRINT_ONE_PORT_TYPE(MEMORY_OBJECT);
 	PRINT_ONE_PORT_TYPE(MIG);
+	PRINT_ONE_PORT_TYPE(MEMORY_OBJECT);
 	PRINT_ONE_PORT_TYPE(XMM_PAGER);
 	PRINT_ONE_PORT_TYPE(XMM_KERNEL);
 	PRINT_ONE_PORT_TYPE(XMM_REPLY);
+	PRINT_ONE_PORT_TYPE(UND_REPLY);
+	PRINT_ONE_PORT_TYPE(HOST_NOTIFY);
+	PRINT_ONE_PORT_TYPE(HOST_SECURITY);
+	PRINT_ONE_PORT_TYPE(LEDGER);
+	PRINT_ONE_PORT_TYPE(MASTER_DEVICE);
+	PRINT_ONE_PORT_TYPE(TASK_NAME);
+	PRINT_ONE_PORT_TYPE(SUBSYSTEM);
+	PRINT_ONE_PORT_TYPE(IO_DONE_QUEUE);
+	PRINT_ONE_PORT_TYPE(SEMAPHORE);
+	PRINT_ONE_PORT_TYPE(LOCK_SET);
 	PRINT_ONE_PORT_TYPE(CLOCK);
 	PRINT_ONE_PORT_TYPE(CLOCK_CTRL);
-	PRINT_ONE_PORT_TYPE(MASTER_DEVICE);
+	PRINT_ONE_PORT_TYPE(IOKIT_SPARE);
+	PRINT_ONE_PORT_TYPE(NAMED_ENTRY);
+	PRINT_ONE_PORT_TYPE(IOKIT_CONNECT);
+	PRINT_ONE_PORT_TYPE(IOKIT_OBJECT);
+	PRINT_ONE_PORT_TYPE(UPL);
+	PRINT_ONE_PORT_TYPE(MEM_OBJ_CONTROL);
+
 	PRINT_ONE_PORT_TYPE(UNKNOWN);
 	printf("\nipc_space:\n\n");
 	printf("NULL	KERNEL	REPLY	PAGER	OTHER\n");
diff --git a/osfmk/ipc/mach_msg.c b/osfmk/ipc/mach_msg.c
index 13481685d..516c7a15a 100644
--- a/osfmk/ipc/mach_msg.c
+++ b/osfmk/ipc/mach_msg.c
@@ -591,6 +591,7 @@ mach_msg_overwrite_trap(
 	mach_port_name_t	notify = args->notify;
 	mach_vm_address_t	rcv_msg_addr = args->rcv_msg;
         mach_msg_size_t		scatter_list_size = 0; /* NOT INITIALIZED - but not used in pactice */
+	mach_port_seqno_t temp_seqno = 0;
 
 	register mach_msg_header_t *hdr;
 	mach_msg_return_t  mr = MACH_MSG_SUCCESS;
@@ -1497,7 +1498,6 @@ mach_msg_overwrite_trap(
 
 	    slow_copyin:
 	    {
-		mach_port_seqno_t temp_seqno = 0;
 		register mach_port_name_t reply_name =
 			        (mach_port_name_t)hdr->msgh_local_port;
 
@@ -1590,7 +1590,6 @@ mach_msg_overwrite_trap(
 
 	    {
 		register ipc_port_t	reply_port;
-		mach_port_seqno_t	local_seqno;
 		spl_t s;
 
 		/*
@@ -1667,7 +1666,7 @@ mach_msg_overwrite_trap(
 		 * no threads blocked waiting to send.
 		 */
 		dest_port = reply_port;
-		local_seqno = rcv_mqueue->imq_seqno++;
+		temp_seqno = rcv_mqueue->imq_seqno++;
 		imq_unlock(rcv_mqueue);
 		splx(s);
 
@@ -1679,7 +1678,7 @@ mach_msg_overwrite_trap(
 		ip_check_unlock(reply_port);
 
 		if (option & MACH_RCV_TRAILER_MASK) {
-			trailer->msgh_seqno = local_seqno;	
+			trailer->msgh_seqno = temp_seqno;	
 			trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
 		}
 		/* copy out the kernel reply */
@@ -1764,6 +1763,7 @@ mach_msg_overwrite_trap(
 		/* LP64support - have to compute real size as it would be received */
 		reply_size = ipc_kmsg_copyout_size(kmsg, current_map()) +
 		             REQUESTED_TRAILER_SIZE(option);
+		temp_seqno = trailer->msgh_seqno;
 		if (rcv_size < reply_size) {
 			if (msg_receive_error(kmsg, msg_addr, option, temp_seqno,
 				        space) == MACH_RCV_INVALID_DATA) {
diff --git a/osfmk/kdp/kdp.c b/osfmk/kdp/kdp.c
index a52b72bc4..00f50e82a 100644
--- a/osfmk/kdp/kdp.c
+++ b/osfmk/kdp/kdp.c
@@ -36,6 +36,11 @@
 
 #include <string.h> /* bcopy */
 
+#include <kern/processor.h>
+#include <kern/thread.h>
+#include <vm/vm_map.h>
+#include <vm/vm_kern.h>
+
 int kdp_vm_read( caddr_t, caddr_t, unsigned int);
 int kdp_vm_write( caddr_t, caddr_t, unsigned int);
 
@@ -95,6 +100,45 @@ static unsigned int breakpoints_initialized = 0;
 int reattach_wait = 0;
 int noresume_on_disconnect = 0;
 
+#define MAXCOMLEN 16
+
+struct thread_snapshot {
+	uint32_t snapshot_magic;
+	thread_t thread_id;
+	int32_t state;
+	wait_queue_t wait_queue;
+	event64_t wait_event;
+	vm_offset_t kernel_stack;
+	vm_offset_t reserved_stack;
+	thread_continue_t continuation;
+	uint32_t nkern_frames;
+	char user64_p;
+	uint32_t nuser_frames;
+	int32_t pid;
+	char p_comm[MAXCOMLEN + 1];
+};
+
+typedef struct thread_snapshot *thread_snapshot_t;
+
+extern int
+machine_trace_thread(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p);
+extern int
+machine_trace_thread64(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p);
+extern int
+proc_pid(void *p);
+extern void
+proc_name(int pid, char *buf, int size);
+extern void
+kdp_snapshot_postflight(void);
+
+static int
+pid_from_task(task_t task);
+
+int
+kdp_stackshot(int pid, uint32_t tracebuf, uint32_t tracebuf_size, unsigned trace_options, uint32_t *pbytesTraced);
+
+extern char version[];
+
 boolean_t
 kdp_packet(
     unsigned char	*pkt,
@@ -380,7 +424,9 @@ kdp_readmem(
     int			plen = *len;
     kdp_readmem_reply_t *rp = &pkt->readmem_reply;
     int			cnt;
-
+#if __i386__
+    void		*pversion = &version;
+#endif
     if (plen < sizeof (*rq))
 	return (FALSE);
 
@@ -393,7 +439,17 @@ kdp_readmem(
 	unsigned int	n = rq->nbytes;
 
 	dprintf(("kdp_readmem addr %x size %d\n", rq->address, rq->nbytes));
-
+#if __i386__
+	/* XXX This is a hack to facilitate the "showversion" macro
+	 * on i386, which is used to obtain the kernel version without
+	 * symbols - a pointer to the version string should eventually
+	 * be pinned at a fixed address when an equivalent of the
+	 * VECTORS segment (loaded at a fixed load address, and contains
+	 * a table) is implemented on x86, as with PPC.
+	 */
+	if (rq->address == (void *)0x501C)
+		rq->address = &pversion;
+#endif
 	cnt = kdp_vm_read((caddr_t)rq->address, (caddr_t)rp->data, rq->nbytes);
 	rp->error = KDPERR_NO_ERROR;
 
@@ -682,3 +738,97 @@ kdp_remove_all_breakpoints()
     }
   return breakpoint_found;
 }
+
+
+#define MAX_FRAMES 1000
+
+static int pid_from_task(task_t task)
+{
+	int pid = -1;
+
+	if (task->bsd_info)
+		pid = proc_pid(task->bsd_info);
+
+	return pid;
+}
+
+int
+kdp_stackshot(int pid, uint32_t tracebuf, uint32_t tracebuf_size, unsigned trace_options, uint32_t *pbytesTraced)
+{
+	uint32_t tracepos = (uint32_t) tracebuf;
+	uint32_t tracebound = tracepos + tracebuf_size;
+	uint32_t tracebytes = 0;
+	int error = 0;
+	
+	processor_set_t pset = &default_pset;
+	task_t task = TASK_NULL;
+	thread_t thread = THREAD_NULL;
+	int nframes = trace_options;
+	thread_snapshot_t tsnap = NULL;
+	unsigned framesize = 2 * sizeof(vm_offset_t);
+
+	if ((nframes <= 0) || nframes > MAX_FRAMES)
+		nframes = MAX_FRAMES;
+
+	queue_iterate(&pset->tasks, task, task_t, pset_tasks) {
+		/* Trace everything, unless a process was specified */
+		if ((pid == -1) || (pid == pid_from_task(task)))
+			queue_iterate(&task->threads, thread, thread_t, task_threads){
+				if ((tracepos + 4 * sizeof(struct thread_snapshot)) > tracebound) {
+					error = -1;
+					goto error_exit;
+				}
+/* Populate the thread snapshot header */
+				tsnap = (thread_snapshot_t) tracepos;
+				tsnap->thread_id = thread;
+				tsnap->state = thread->state;
+				tsnap->wait_queue = thread->wait_queue;
+				tsnap->wait_event = thread->wait_event;
+				tsnap->kernel_stack = thread->kernel_stack;
+				tsnap->reserved_stack = thread->reserved_stack;
+				tsnap->continuation = thread->continuation;
+/* Add the BSD process identifiers */
+				if ((tsnap->pid = pid_from_task(task)) != -1)
+					proc_name(tsnap->pid, tsnap->p_comm, MAXCOMLEN + 1);
+				else
+					tsnap->p_comm[0] = '\0';
+
+				tsnap->snapshot_magic = 0xfeedface;
+				tracepos += sizeof(struct thread_snapshot);
+
+/* Call through to the machine specific trace routines
+ * Frames are added past the snapshot header.
+ */
+				if (tsnap->kernel_stack != 0)
+					tracebytes = machine_trace_thread(thread, tracepos, tracebound, nframes, FALSE);
+				tsnap->nkern_frames = tracebytes/(2 * sizeof(vm_offset_t));
+				tracepos += tracebytes;
+				tracebytes = 0;
+				tsnap->user64_p = 0;
+/* Trace user stack, if any */
+				if (thread->task->map != kernel_map) {
+/* 64-bit task? */
+					if (task_has_64BitAddr(thread->task)) {
+						tracebytes = machine_trace_thread64(thread, tracepos, tracebound, nframes, TRUE);
+						tsnap->user64_p = 1;
+						framesize = 2 * sizeof(addr64_t);
+					}
+					else {
+						tracebytes = machine_trace_thread(thread, tracepos, tracebound, nframes, TRUE);
+						framesize = 2 * sizeof(vm_offset_t);
+					}
+				}
+				tsnap->nuser_frames = tracebytes/framesize;
+				tracepos += tracebytes;
+				tracebytes = 0;
+			}
+	}
+
+error_exit:
+	/* Release stack snapshot wait indicator */
+	kdp_snapshot_postflight();
+
+	*pbytesTraced = tracepos - tracebuf;
+
+	return error;
+}
diff --git a/osfmk/kdp/kdp_core.h b/osfmk/kdp/kdp_core.h
index b43b5b77d..aa54f350d 100644
--- a/osfmk/kdp/kdp_core.h
+++ b/osfmk/kdp/kdp_core.h
@@ -79,4 +79,8 @@ void abort_panic_transfer (void);
 
 struct corehdr *create_panic_header(unsigned int request, const char *corename, unsigned length, unsigned block);
 
-int kdp_send_panic_pkt (unsigned int request, char *corename, unsigned int length, void *panic_data);
+int 	kdp_send_crashdump_pkt(unsigned int request, char *corename,
+				unsigned int length, void *panic_data);
+
+int	kdp_send_crashdump_data(unsigned int request, char *corename,
+				unsigned int length, caddr_t txstart);
diff --git a/osfmk/kdp/kdp_internal.h b/osfmk/kdp/kdp_internal.h
index c98c08937..968a38887 100644
--- a/osfmk/kdp/kdp_internal.h
+++ b/osfmk/kdp/kdp_internal.h
@@ -46,7 +46,8 @@ typedef struct {
 
 extern kdp_glob_t	kdp;
 
-extern int		kdp_flag;
+extern volatile int	kdp_flag;
+
 #define KDP_READY       0x1
 #define KDP_ARP         0x2
 #define KDP_BP_DIS      0x4
diff --git a/osfmk/kdp/kdp_udp.c b/osfmk/kdp/kdp_udp.c
index 872e93638..34b854fec 100644
--- a/osfmk/kdp/kdp_udp.c
+++ b/osfmk/kdp/kdp_udp.c
@@ -41,14 +41,14 @@
 #include <kern/cpu_data.h>
 #include <kern/debug.h>
 
+#include <kdp/kdp_core.h>
 #include <kdp/kdp_internal.h>
 #include <kdp/kdp_en_debugger.h>
 #include <kdp/kdp_udp.h>
 
-#include <kdp/kdp_core.h>
-
 #include <vm/vm_map.h>
 #include <vm/vm_protos.h>
+#include <vm/vm_kern.h> /* kernel_map */
 
 #include <mach/memory_object_types.h>
 
@@ -88,7 +88,7 @@ struct {
     } rmt;
 } adr;
 
-static char
+static const char
 *exception_message[] = {
     "Unknown",
     "Memory access",		/* EXC_BAD_ACCESS */
@@ -99,7 +99,7 @@ static char
     "Breakpoint"		/* EXC_BREAKPOINT */
 };
 
-int kdp_flag = 0;
+volatile int kdp_flag = 0;
 
 static kdp_send_t kdp_en_send_pkt = 0;
 static kdp_receive_t kdp_en_recv_pkt = 0;
@@ -111,23 +111,34 @@ static void *kdp_current_ifp = 0;
 
 static void kdp_handler( void *);
 
-static unsigned int panic_server_ip = 0; 
-static unsigned int parsed_router_ip = 0;
-static unsigned int router_ip = 0;
-static unsigned int panicd_specified = 0;
-static unsigned int router_specified = 0;
+static uint32_t panic_server_ip = 0; 
+static uint32_t parsed_router_ip = 0;
+static uint32_t router_ip = 0;
+static uint32_t target_ip = 0;
+
+static volatile boolean_t panicd_specified = FALSE;
+static boolean_t router_specified = FALSE;
+static unsigned int panicd_port = CORE_REMOTE_PORT;
+
+/* As in bsd/net/ether_if_module.c */
+static struct ether_addr etherbroadcastaddr = {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}};
 
 static struct ether_addr router_mac = {{0, 0, 0 , 0, 0, 0}};
+static struct ether_addr destination_mac = {{0, 0, 0 , 0, 0, 0}};
+static struct ether_addr temp_mac = {{0, 0, 0 , 0, 0, 0}};
+static struct ether_addr current_resolved_MAC = {{0, 0, 0 , 0, 0, 0}};
 
-static u_char flag_panic_dump_in_progress = 0;
-static u_char flag_router_mac_initialized = 0;
+static boolean_t flag_panic_dump_in_progress = FALSE;
+static boolean_t flag_router_mac_initialized = FALSE;
+
+static boolean_t flag_arp_resolved = FALSE;
 
 static unsigned int panic_timeout = 100000;
 static unsigned int last_panic_port = CORE_REMOTE_PORT;
 
 unsigned int SEGSIZE = 512;
 
-static unsigned int PANIC_PKTSIZE = 518;
+__unused static unsigned int PANIC_PKTSIZE = 518;
 static char panicd_ip_str[20];
 static char router_ip_str[20];
 
@@ -137,7 +148,52 @@ static volatile unsigned int flag_kdp_trigger_reboot = 0;
 
 extern unsigned int not_in_kdp;
 
-extern int kdp_vm_read( caddr_t, caddr_t, unsigned int);
+extern unsigned long panic_caller;
+extern unsigned int disableConsoleOutput;
+
+extern int 		kdp_vm_read( caddr_t, caddr_t, unsigned int);
+extern void 		kdp_call(void);
+extern boolean_t 	kdp_call_kdb(void);
+extern int 		kern_dump(void);
+
+void *	kdp_get_interface(void);
+void 	kdp_set_gateway_mac(void *);
+void 	kdp_set_ip_and_mac_addresses(struct in_addr *, struct ether_addr *);
+void 	kdp_set_interface(void *);
+
+void 			kdp_disable_arp(void);
+static void 		kdp_arp_reply(struct ether_arp *);
+static void 		kdp_process_arp_reply(struct ether_arp *);
+static boolean_t 	kdp_arp_resolve(uint32_t, struct ether_addr *);
+
+static boolean_t	gKDPDebug = FALSE;
+#define KDP_DEBUG(...) if (gKDPDebug) printf(__VA_ARGS__);
+
+int kdp_snapshot = 0;
+static int stack_snapshot_ret = 0;
+static unsigned stack_snapshot_bytes_traced = 0;
+
+static void *stack_snapshot_buf;
+static uint32_t stack_snapshot_bufsize;
+static int stack_snapshot_pid;
+static uint32_t stack_snapshot_options;
+
+void
+kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size,
+    uint32_t options);
+
+void
+kdp_snapshot_postflight(void);
+
+extern int
+kdp_stackshot(int pid, uint32_t tracebuf, uint32_t tracebuf_size,
+    unsigned trace_options, uint32_t *pbytesTraced);
+
+int
+kdp_stack_snapshot_geterror(void);
+
+int
+kdp_stack_snapshot_bytes_traced(void);
 
 void
 kdp_register_send_receive(
@@ -172,10 +228,13 @@ kdp_register_send_receive(
 	  kdp_flag |= PANIC_LOG_DUMP;
 	
 	if (PE_parse_boot_arg ("_panicd_ip", panicd_ip_str))
-	  panicd_specified = 1;
-	/* For the future, currently non-functional */
+	  panicd_specified = TRUE;
+
 	if (PE_parse_boot_arg ("_router_ip", router_ip_str))
-	  router_specified = 1;
+	  router_specified = TRUE;
+
+	if (!PE_parse_boot_arg ("panicd_port", &panicd_port))
+		panicd_port = CORE_REMOTE_PORT;
 
 	kdp_flag |= KDP_READY;
 	if (current_debugger == NO_CUR_DB)
@@ -188,8 +247,8 @@ kdp_register_send_receive(
 
 void
 kdp_unregister_send_receive(
-	kdp_send_t		send, 
-	kdp_receive_t	receive)
+	__unused kdp_send_t	send, 
+	__unused kdp_receive_t	receive)
 {
 	if (current_debugger == KDP_CUR_DB)
 		current_debugger = NO_CUR_DB;
@@ -198,6 +257,35 @@ kdp_unregister_send_receive(
 	kdp_en_recv_pkt = NULL;
 }
 
+/* Cache stack snapshot parameters in preparation for a trace */
+void
+kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_t options)
+{
+	stack_snapshot_pid = pid;
+	stack_snapshot_buf = tracebuf;
+	stack_snapshot_bufsize = tracebuf_size;
+	stack_snapshot_options = options;
+	kdp_snapshot++;
+}
+
+void
+kdp_snapshot_postflight(void)
+{
+	kdp_snapshot--;
+}
+
+int
+kdp_stack_snapshot_geterror(void)
+{
+	return stack_snapshot_ret;
+}
+
+int
+kdp_stack_snapshot_bytes_traced(void)
+{
+	return stack_snapshot_bytes_traced;
+}
+
 static void
 enaddr_copy(
 	void	*src,
@@ -211,92 +299,92 @@ static unsigned short
 ip_sum(
 	unsigned char	*c,
 	unsigned int	hlen
-)
+	)
 {
-    unsigned int	high, low, sum;
+	unsigned int	high, low, sum;
     
-    high = low = 0;
-    while (hlen-- > 0) {
-	low += c[1] + c[3];
-	high += c[0] + c[2];
+	high = low = 0;
+	while (hlen-- > 0) {
+		low += c[1] + c[3];
+		high += c[0] + c[2];
 	
-	c += sizeof (int);
-    }
+		c += sizeof (int);
+	}
     
-    sum = (high << 8) + low;
-    sum = (sum >> 16) + (sum & 65535);
+	sum = (high << 8) + low;
+	sum = (sum >> 16) + (sum & 65535);
     
-    return (sum > 65535 ? sum - 65535 : sum);
+	return (sum > 65535 ? sum - 65535 : sum);
 }
 
 static void
 kdp_reply(
 	unsigned short		reply_port
-)
+	)
 {
-    struct udpiphdr		aligned_ui, *ui = &aligned_ui;
-    struct ip			aligned_ip, *ip = &aligned_ip;
-    struct in_addr		tmp_ipaddr;
-    struct ether_addr		tmp_enaddr;
-    struct ether_header		*eh;
+	struct udpiphdr		aligned_ui, *ui = &aligned_ui;
+	struct ip		aligned_ip, *ip = &aligned_ip;
+	struct in_addr		tmp_ipaddr;
+	struct ether_addr	tmp_enaddr;
+	struct ether_header	*eh = NULL;
     
-    if (!pkt.input)
-	kdp_panic("kdp_reply");
+	if (!pkt.input)
+		kdp_panic("kdp_reply");
 	
-    pkt.off -= sizeof (struct udpiphdr);
+	pkt.off -= sizeof (struct udpiphdr);
 
 #if DO_ALIGN    
-    bcopy((char *)&pkt.data[pkt.off], (char *)ui, sizeof(*ui));
+	bcopy((char *)&pkt.data[pkt.off], (char *)ui, sizeof(*ui));
 #else
-    ui = (struct udpiphdr *)&pkt.data[pkt.off];
+	ui = (struct udpiphdr *)&pkt.data[pkt.off];
 #endif
-    ui->ui_next = ui->ui_prev = 0;
-    ui->ui_x1 = 0;
-    ui->ui_pr = IPPROTO_UDP;
-    ui->ui_len = htons((u_short)pkt.len + sizeof (struct udphdr));
-    tmp_ipaddr = ui->ui_src;
-    ui->ui_src = ui->ui_dst;
-    ui->ui_dst = tmp_ipaddr;
-    ui->ui_sport = htons(KDP_REMOTE_PORT);
-    ui->ui_dport = reply_port;
-    ui->ui_ulen = ui->ui_len;
-    ui->ui_sum = 0;
+	ui->ui_next = ui->ui_prev = 0;
+	ui->ui_x1 = 0;
+	ui->ui_pr = IPPROTO_UDP;
+	ui->ui_len = htons((u_short)pkt.len + sizeof (struct udphdr));
+	tmp_ipaddr = ui->ui_src;
+	ui->ui_src = ui->ui_dst;
+	ui->ui_dst = tmp_ipaddr;
+	ui->ui_sport = htons(KDP_REMOTE_PORT);
+	ui->ui_dport = reply_port;
+	ui->ui_ulen = ui->ui_len;
+	ui->ui_sum = 0;
 #if DO_ALIGN
-    bcopy((char *)ui, (char *)&pkt.data[pkt.off], sizeof(*ui));
-    bcopy((char *)&pkt.data[pkt.off], (char *)ip, sizeof(*ip));
+	bcopy((char *)ui, (char *)&pkt.data[pkt.off], sizeof(*ui));
+	bcopy((char *)&pkt.data[pkt.off], (char *)ip, sizeof(*ip));
 #else
-    ip = (struct ip *)&pkt.data[pkt.off];
+	ip = (struct ip *)&pkt.data[pkt.off];
 #endif
-    ip->ip_len = htons(sizeof (struct udpiphdr) + pkt.len);
-    ip->ip_v = IPVERSION;
-    ip->ip_id = htons(ip_id++);
-    ip->ip_hl = sizeof (struct ip) >> 2;
-    ip->ip_ttl = udp_ttl;
-    ip->ip_sum = 0;
-    ip->ip_sum = htons(~ip_sum((unsigned char *)ip, ip->ip_hl));
+	ip->ip_len = htons(sizeof (struct udpiphdr) + pkt.len);
+	ip->ip_v = IPVERSION;
+	ip->ip_id = htons(ip_id++);
+	ip->ip_hl = sizeof (struct ip) >> 2;
+	ip->ip_ttl = udp_ttl;
+	ip->ip_sum = 0;
+	ip->ip_sum = htons(~ip_sum((unsigned char *)ip, ip->ip_hl));
 #if DO_ALIGN
-    bcopy((char *)ip, (char *)&pkt.data[pkt.off], sizeof(*ip));
+	bcopy((char *)ip, (char *)&pkt.data[pkt.off], sizeof(*ip));
 #endif
     
-    pkt.len += sizeof (struct udpiphdr);
+	pkt.len += sizeof (struct udpiphdr);
     
-    pkt.off -= sizeof (struct ether_header);
+	pkt.off -= sizeof (struct ether_header);
     
-    eh = (struct ether_header *)&pkt.data[pkt.off];
-    enaddr_copy(eh->ether_shost, &tmp_enaddr);
-    enaddr_copy(eh->ether_dhost, eh->ether_shost);
-    enaddr_copy(&tmp_enaddr, eh->ether_dhost);
-    eh->ether_type = htons(ETHERTYPE_IP);
+	eh = (struct ether_header *)&pkt.data[pkt.off];
+	enaddr_copy(eh->ether_shost, &tmp_enaddr);
+	enaddr_copy(eh->ether_dhost, eh->ether_shost);
+	enaddr_copy(&tmp_enaddr, eh->ether_dhost);
+	eh->ether_type = htons(ETHERTYPE_IP);
     
-    pkt.len += sizeof (struct ether_header);
+	pkt.len += sizeof (struct ether_header);
     
-    // save reply for possible retransmission
-    bcopy((char *)&pkt, (char *)&saved_reply, sizeof(pkt));
+	// save reply for possible retransmission
+	bcopy((char *)&pkt, (char *)&saved_reply, sizeof(pkt));
 
-    (*kdp_en_send_pkt)(&pkt.data[pkt.off], pkt.len);
+	(*kdp_en_send_pkt)(&pkt.data[pkt.off], pkt.len);
 
-    // increment expected sequence number
-    exception_seq++;
+	// increment expected sequence number
+	exception_seq++;
 }
 
 static void
@@ -385,7 +473,7 @@ void
 kdp_set_gateway_mac(void *gatewaymac)
 {
   router_mac = *(struct ether_addr *)gatewaymac;
-  flag_router_mac_initialized = 1;
+  flag_router_mac_initialized = TRUE;
 } 
 
 struct ether_addr 
@@ -400,16 +488,58 @@ kdp_get_ip_address(void)
   return kdp_current_ip_address;
 }
 
+void
+kdp_disable_arp(void)
+{
+	kdp_flag &= ~(DB_ARP);
+}
+
+static void
+kdp_arp_dispatch(void)
+{
+	struct ether_arp	aligned_ea, *ea = &aligned_ea;
+	unsigned		arp_header_offset;
+
+	arp_header_offset = sizeof(struct ether_header) + pkt.off;
+	memcpy((void *)ea, (void *)&pkt.data[arp_header_offset], sizeof(*ea));
+
+	switch(ntohs(ea->arp_op)) {
+	case ARPOP_REQUEST:
+		kdp_arp_reply(ea);
+		break;
+	case ARPOP_REPLY:
+		kdp_process_arp_reply(ea);
+		break;
+	default:
+		return;
+	}
+}
+
+static void
+kdp_process_arp_reply(struct ether_arp *ea)
+{
+	/* Are we interested in ARP replies? */
+	if (flag_arp_resolved == TRUE)
+		return;
+
+	/* Did we receive a reply from the right source? */
+	if (((struct in_addr *)(ea->arp_spa))->s_addr != target_ip)
+	  return;
+
+	flag_arp_resolved = TRUE;
+	current_resolved_MAC = *(struct ether_addr *) (ea->arp_sha);
+
+	return;
+}
+
 /* ARP responses are enabled when the DB_ARP bit of the debug boot arg
-   is set. A workaround if you don't want to reboot is to set 
-   kdpDEBUGFlag &= DB_ARP when connected (but that certainly isn't a published
-   interface!)
-*/
+ * is set.
+ */
+
 static void 
-kdp_arp_reply(void)
+kdp_arp_reply(struct ether_arp *ea)
 {
 	struct ether_header	*eh;
-	struct ether_arp	aligned_ea, *ea = &aligned_ea;
 
 	struct in_addr 		isaddr, itaddr, myaddr;
 	struct ether_addr	my_enaddr;
@@ -417,20 +547,25 @@ kdp_arp_reply(void)
 	eh = (struct ether_header *)&pkt.data[pkt.off];
 	pkt.off += sizeof(struct ether_header);
 
-	memcpy((void *)ea, (void *)&pkt.data[pkt.off],sizeof(*ea));
-  
 	if(ntohs(ea->arp_op) != ARPOP_REQUEST)
 	  return;
 
 	myaddr.s_addr = kdp_get_ip_address();
 	my_enaddr = kdp_get_mac_addr();
 
-	if (!(myaddr.s_addr) || !(my_enaddr.ether_addr_octet[1]))
+	if ((ntohl(myaddr.s_addr) == 0) ||
+	    ((my_enaddr.ether_addr_octet[0] & 0xff) == 0
+		 && (my_enaddr.ether_addr_octet[1] & 0xff) == 0
+		 && (my_enaddr.ether_addr_octet[2] & 0xff) == 0
+		 && (my_enaddr.ether_addr_octet[3] & 0xff) == 0
+		 && (my_enaddr.ether_addr_octet[4] & 0xff) == 0
+		 && (my_enaddr.ether_addr_octet[5] & 0xff) == 0
+		 ))
 		return;
 
 	(void)memcpy((void *)&isaddr, (void *)ea->arp_spa, sizeof (isaddr));
 	(void)memcpy((void *)&itaddr, (void *)ea->arp_tpa, sizeof (itaddr));
-  
+
 	if (itaddr.s_addr == myaddr.s_addr) {
 		(void)memcpy((void *)ea->arp_tha, (void *)ea->arp_sha, sizeof(ea->arp_sha));
 		(void)memcpy((void *)ea->arp_sha, (void *)&my_enaddr, sizeof(ea->arp_sha));
@@ -453,100 +588,185 @@ kdp_arp_reply(void)
 static void
 kdp_poll(void)
 {
-  struct ether_header	*eh;
-  struct udpiphdr	aligned_ui, *ui = &aligned_ui;
-  struct ip		aligned_ip, *ip = &aligned_ip;
-  static int		msg_printed;
-
+	struct ether_header	*eh = NULL;
+	struct udpiphdr		aligned_ui, *ui = &aligned_ui;
+	struct ip		aligned_ip, *ip = &aligned_ip;
+	static int		msg_printed;
 
-    if (pkt.input)
-	kdp_panic("kdp_poll");
+	if (pkt.input)
+		kdp_panic("kdp_poll");
  
-    if (!kdp_en_recv_pkt || !kdp_en_send_pkt) {
-	if( msg_printed == 0) {
-	    msg_printed = 1;
-            printf("kdp_poll: no debugger device\n");
+	if (!kdp_en_recv_pkt || !kdp_en_send_pkt) {
+		if( msg_printed == 0) {
+			msg_printed = 1;
+			printf("kdp_poll: no debugger device\n");
+		}
+		return;
 	}
-	return;
-    }
 
-    pkt.off = pkt.len = 0;
-    (*kdp_en_recv_pkt)(pkt.data, &pkt.len, 3/* ms */);
-  
-    if (pkt.len == 0)
-	return;
+	pkt.off = pkt.len = 0;
+	(*kdp_en_recv_pkt)(pkt.data, &pkt.len, 3/* ms */);
 
-    if (pkt.len >= sizeof(struct ether_header))
-      {
-	eh = (struct ether_header *)&pkt.data[pkt.off];  
-	
-	if (kdp_flag & KDP_ARP)
-	  {
-	    if (ntohs(eh->ether_type) == ETHERTYPE_ARP)
-	      {
-		kdp_arp_reply();
+	if (pkt.len == 0)
 		return;
-	      }
-	  }
-      }
 
-    if (pkt.len < (sizeof (struct ether_header) + sizeof (struct udpiphdr)))
-    	return;
+	if (pkt.len >= sizeof(struct ether_header))
+	{
+		eh = (struct ether_header *)&pkt.data[pkt.off];  
+	
+		if (kdp_flag & KDP_ARP)
+		{
+			if (ntohs(eh->ether_type) == ETHERTYPE_ARP)
+			{
+				kdp_arp_dispatch();
+				return;
+			}
+		}
+	}
+
+	if (pkt.len < (sizeof (struct ether_header) + sizeof (struct udpiphdr)))
+		return;
 
-    pkt.off += sizeof (struct ether_header);
-    if (ntohs(eh->ether_type) != ETHERTYPE_IP) {
-	return;
-    }
+	pkt.off += sizeof (struct ether_header);
+	if (ntohs(eh->ether_type) != ETHERTYPE_IP) {
+		return;
+	}
 
 #if DO_ALIGN
-    bcopy((char *)&pkt.data[pkt.off], (char *)ui, sizeof(*ui));
-    bcopy((char *)&pkt.data[pkt.off], (char *)ip, sizeof(*ip));
+	bcopy((char *)&pkt.data[pkt.off], (char *)ui, sizeof(*ui));
+	bcopy((char *)&pkt.data[pkt.off], (char *)ip, sizeof(*ip));
 #else
-    ui = (struct udpiphdr *)&pkt.data[pkt.off];
-    ip = (struct ip *)&pkt.data[pkt.off];
+	ui = (struct udpiphdr *)&pkt.data[pkt.off];
+	ip = (struct ip *)&pkt.data[pkt.off];
 #endif
 
-    pkt.off += sizeof (struct udpiphdr);
-    if (ui->ui_pr != IPPROTO_UDP) {
-	return;
-    }
+	pkt.off += sizeof (struct udpiphdr);
+	if (ui->ui_pr != IPPROTO_UDP) {
+		return;
+	}
  
-    if (ip->ip_hl > (sizeof (struct ip) >> 2)) {
-	return;
-    }
+	if (ip->ip_hl > (sizeof (struct ip) >> 2)) {
+		return;
+	}
 
-    if (ntohs(ui->ui_dport) != KDP_REMOTE_PORT) {
-      if (CORE_REMOTE_PORT == (ntohs(ui->ui_dport)) && 
-	  flag_panic_dump_in_progress) {
-	  last_panic_port = ui->ui_sport;
+	if (ntohs(ui->ui_dport) != KDP_REMOTE_PORT) {
+		if (panicd_port == (ntohs(ui->ui_dport)) && 
+		    flag_panic_dump_in_progress) {
+			last_panic_port = ui->ui_sport;
+		}
+		else
+			return;
 	}
-      else
-	return;
-    }
-    /* If we receive a kernel debugging packet whilst a 
-     * core dump is in progress, abort the transfer and 
-     * enter the debugger.
-     */
-    else
-      if (flag_panic_dump_in_progress)
-	{
-	  abort_panic_transfer();
-	  return;
+	/* If we receive a kernel debugging packet whilst a 
+	 * core dump is in progress, abort the transfer and 
+	 * enter the debugger.
+	 */
+	else
+		if (flag_panic_dump_in_progress)
+		{
+			abort_panic_transfer();
+			return;
+		}
+
+	if (!kdp.is_conn && !flag_panic_dump_in_progress) {
+		enaddr_copy(eh->ether_dhost, &adr.loc.ea);
+		adr.loc.in = ui->ui_dst;
+
+		enaddr_copy(eh->ether_shost, &adr.rmt.ea);
+		adr.rmt.in = ui->ui_src;
 	}
 
-    if (!kdp.is_conn && !flag_panic_dump_in_progress) {
-	enaddr_copy(eh->ether_dhost, &adr.loc.ea);
-	adr.loc.in = ui->ui_dst;
+	/*
+	 * Calculate kdp packet length.
+	 */
+	pkt.len = ntohs((u_short)ui->ui_ulen) - sizeof (struct udphdr);
+	pkt.input = TRUE;
+}
 
-	enaddr_copy(eh->ether_shost, &adr.rmt.ea);
-	adr.rmt.in = ui->ui_src;
-    }
+/* Create and transmit an ARP resolution request for the target IP address.
+ * This is modeled on ether_inet_arp()/RFC 826.
+ */
 
-    /*
-     * Calculate kdp packet length.
-     */
-    pkt.len = ntohs((u_short)ui->ui_ulen) - sizeof (struct udphdr);
-    pkt.input = TRUE;
+static void
+transmit_ARP_request(uint32_t ip_addr)
+{
+	struct ether_header	*eh = (struct ether_header *) &pkt.data[0];
+	struct ether_arp	*ea = (struct ether_arp *) &pkt.data[sizeof(struct ether_header)];
+
+ 	KDP_DEBUG("Transmitting ARP request\n");
+	/* Populate the ether_header */
+	eh->ether_type = htons(ETHERTYPE_ARP);
+	enaddr_copy(&kdp_current_mac_address, eh->ether_shost);
+	enaddr_copy(&etherbroadcastaddr, eh->ether_dhost);
+
+	/* Populate the ARP header */
+	ea->arp_pro = htons(ETHERTYPE_IP);
+	ea->arp_hln = sizeof(ea->arp_sha);
+	ea->arp_pln = sizeof(ea->arp_spa);
+	ea->arp_hrd = htons(ARPHRD_ETHER);
+	ea->arp_op = htons(ARPOP_REQUEST);
+
+	/* Target fields */
+	enaddr_copy(&etherbroadcastaddr, ea->arp_tha);
+	memcpy(ea->arp_tpa, (void *) &ip_addr, sizeof(ip_addr));
+
+	/* Source fields */
+	enaddr_copy(&kdp_current_mac_address, ea->arp_sha);
+	memcpy(ea->arp_spa, (void *) &kdp_current_ip_address, sizeof(kdp_current_ip_address));
+
+	pkt.off = 0;
+	pkt.len = sizeof(struct ether_header) + sizeof(struct ether_arp);
+	/* Transmit */
+	(*kdp_en_send_pkt)(&pkt.data[pkt.off], pkt.len);
+}
+
+static boolean_t
+kdp_arp_resolve(uint32_t arp_target_ip, struct ether_addr *resolved_MAC)
+{
+	int poll_count = 256; /* ~770 ms modulo broadcast/delayed traffic? */
+	char tretries = 0;
+
+#define NUM_ARP_TX_RETRIES 5
+
+	target_ip = arp_target_ip;
+	flag_arp_resolved = FALSE;
+
+TRANSMIT_RETRY:
+	pkt.off = pkt.len = 0;
+
+	tretries++;
+
+	if (tretries >= NUM_ARP_TX_RETRIES) {
+		return FALSE;
+	}
+
+	KDP_DEBUG("ARP TX attempt #%d \n", tretries);
+
+	transmit_ARP_request(arp_target_ip);
+
+	while (!pkt.input && !flag_arp_resolved && flag_panic_dump_in_progress && --poll_count) {
+		kdp_poll();
+	}
+
+	if (flag_arp_resolved) {
+		*resolved_MAC = current_resolved_MAC;
+		return TRUE;
+	}
+	
+	if (!flag_panic_dump_in_progress || pkt.input) /* we received a debugging packet, bail*/
+	{
+		printf("Received a debugger packet,transferring control to debugger\n");
+		/* Indicate that we should wait in the debugger when we return */
+		kdp_flag |= DBG_POST_CORE;
+		pkt.input = FALSE;
+		return FALSE;
+	}
+	else /* We timed out */
+		if (0 == poll_count) {
+			poll_count = 256;
+			goto TRANSMIT_RETRY;
+		}
+	return FALSE;
 }
 
 static void
@@ -557,7 +777,6 @@ kdp_handler(
     unsigned short		reply_port;
     kdp_hdr_t			aligned_hdr, *hdr = &aligned_hdr;
 
-
     kdp.saved_state = saved_state;  // see comment in kdp_raise_exception
 
     do {
@@ -605,10 +824,15 @@ static void
 kdp_connection_wait(void)
 {
 	unsigned short		reply_port;
-	boolean_t		kdp_call_kdb();
 	struct ether_addr	kdp_mac_addr = kdp_get_mac_addr();
 	unsigned int		ip_addr = ntohl(kdp_get_ip_address());
 
+	/*
+	 * Do both a printf() and a kprintf() of the MAC and IP so that
+	 * they will print out on headless machines but not be added to
+	 * the panic.log
+	 */
+
 	printf( "ethernet MAC address: %02x:%02x:%02x:%02x:%02x:%02x\n",
             kdp_mac_addr.ether_addr_octet[0] & 0xff,
             kdp_mac_addr.ether_addr_octet[1] & 0xff,
@@ -617,12 +841,26 @@ kdp_connection_wait(void)
             kdp_mac_addr.ether_addr_octet[4] & 0xff,
             kdp_mac_addr.ether_addr_octet[5] & 0xff);
 		
+	kprintf( "ethernet MAC address: %02x:%02x:%02x:%02x:%02x:%02x\n",
+			kdp_mac_addr.ether_addr_octet[0] & 0xff,
+			kdp_mac_addr.ether_addr_octet[1] & 0xff,
+			kdp_mac_addr.ether_addr_octet[2] & 0xff,
+			kdp_mac_addr.ether_addr_octet[3] & 0xff,
+			kdp_mac_addr.ether_addr_octet[4] & 0xff,
+			kdp_mac_addr.ether_addr_octet[5] & 0xff);
+		
 	printf( "ip address: %d.%d.%d.%d\n",
             (ip_addr & 0xff000000) >> 24,
             (ip_addr & 0xff0000) >> 16,
             (ip_addr & 0xff00) >> 8,
             (ip_addr & 0xff));
             
+	kprintf( "ip address: %d.%d.%d.%d\n",
+			(ip_addr & 0xff000000) >> 24,
+			(ip_addr & 0xff0000) >> 16,
+			(ip_addr & 0xff00) >> 8,
+			(ip_addr & 0xff));
+            
 	printf("\nWaiting for remote debugger connection.\n");
 
 	if (reattach_wait == 0) {
@@ -760,8 +998,6 @@ kdp_raise_exception(
 {
     int			index;
 
-    extern unsigned int	disableConsoleOutput;
-
     disable_preemption();
 
     if (saved_state == 0) 
@@ -787,11 +1023,19 @@ kdp_raise_exception(
     if (pkt.input)
 	kdp_panic("kdp_raise_exception");
 
+    /* Was a system trace requested ? */
+    if (kdp_snapshot && (panicstr == ((char *) 0)) && (panic_caller == 0) && !kdp.is_conn) {
+	    /* XXX This should be reworked to take a pointer to the buffer */
+	    stack_snapshot_ret = kdp_stackshot(stack_snapshot_pid,
+	    (uint32_t) stack_snapshot_buf, stack_snapshot_bufsize,
+	    stack_snapshot_options, &stack_snapshot_bytes_traced);
+	    goto exit_raise_exception;
+    }
+	    
     if (((kdp_flag & KDP_PANIC_DUMP_ENABLED) || (kdp_flag & PANIC_LOG_DUMP))
 	&& (panicstr != (char *) 0)) {
 
-	kdp_panic_dump();
-
+	    kdp_panic_dump();
       }
     else
       if ((kdp_flag & PANIC_CORE_ON_NMI) && (panicstr == (char *) 0) &&
@@ -866,218 +1110,214 @@ kdp_reset(void)
 
 struct corehdr *
 create_panic_header(unsigned int request, const char *corename, 
-		    unsigned length, unsigned int block)
+    unsigned length, unsigned int block)
 {
-  struct udpiphdr	aligned_ui, *ui = &aligned_ui;
-  struct ip		aligned_ip, *ip = &aligned_ip;
-  struct ether_header	*eh;
-  struct corehdr        *coreh;
-  const char            *mode = "octet";
-  char                  modelen  = strlen(mode);
+	struct udpiphdr		aligned_ui, *ui = &aligned_ui;
+	struct ip		aligned_ip, *ip = &aligned_ip;
+	struct ether_header	*eh;
+	struct corehdr		*coreh;
+	const char		*mode = "octet";
+	char			modelen  = strlen(mode);
 
-  pkt.off = sizeof (struct ether_header);
-  pkt.len = length + ((request == KDP_WRQ) ? modelen : 0) + 
-    (corename ? strlen(corename): 0) + sizeof(struct corehdr);
+	pkt.off = sizeof (struct ether_header);
+	pkt.len = length + ((request == KDP_WRQ) ? modelen : 0) + 
+	    (corename ? strlen(corename): 0) + sizeof(struct corehdr);
 
 #if DO_ALIGN
-  bcopy((char *)&pkt.data[pkt.off], (char *)ui, sizeof(*ui));
+	bcopy((char *)&pkt.data[pkt.off], (char *)ui, sizeof(*ui));
 #else
-  ui = (struct udpiphdr *)&pkt.data[pkt.off];
+	ui = (struct udpiphdr *)&pkt.data[pkt.off];
 #endif
-  ui->ui_next = ui->ui_prev = 0;
-  ui->ui_x1 = 0;
-  ui->ui_pr = IPPROTO_UDP;
-  ui->ui_len = htons((u_short)pkt.len + sizeof (struct udphdr));
-  ui->ui_src.s_addr = htonl(kdp_current_ip_address);
-  ui->ui_dst.s_addr = panic_server_ip;
-  ui->ui_sport = htons(CORE_REMOTE_PORT);
-  ui->ui_dport = ((request == KDP_WRQ) ? htons(CORE_REMOTE_PORT) : last_panic_port);
-  ui->ui_ulen = ui->ui_len;
-  ui->ui_sum = 0;
+	ui->ui_next = ui->ui_prev = 0;
+	ui->ui_x1 = 0;
+	ui->ui_pr = IPPROTO_UDP;
+	ui->ui_len = htons((u_short)pkt.len + sizeof (struct udphdr));
+	ui->ui_src.s_addr = kdp_current_ip_address;
+	/* Already in network byte order via inet_aton() */
+	ui->ui_dst.s_addr = panic_server_ip;
+	ui->ui_sport = htons(panicd_port);
+	ui->ui_dport = ((request == KDP_WRQ) ? htons(panicd_port) : last_panic_port);
+	ui->ui_ulen = ui->ui_len;
+	ui->ui_sum = 0;
 #if DO_ALIGN
-  bcopy((char *)ui, (char *)&pkt.data[pkt.off], sizeof(*ui));
-  bcopy((char *)&pkt.data[pkt.off], (char *)ip, sizeof(*ip));
+	bcopy((char *)ui, (char *)&pkt.data[pkt.off], sizeof(*ui));
+	bcopy((char *)&pkt.data[pkt.off], (char *)ip, sizeof(*ip));
 #else
-  ip = (struct ip *)&pkt.data[pkt.off];
+	ip = (struct ip *)&pkt.data[pkt.off];
 #endif
-  ip->ip_len = htons(sizeof (struct udpiphdr) + pkt.len);
-  ip->ip_v = IPVERSION;
-  ip->ip_id = htons(ip_id++);
-  ip->ip_hl = sizeof (struct ip) >> 2;
-  ip->ip_ttl = udp_ttl;
-  ip->ip_sum = 0;
-  ip->ip_sum = htons(~ip_sum((unsigned char *)ip, ip->ip_hl));
+	ip->ip_len = htons(sizeof (struct udpiphdr) + pkt.len);
+	ip->ip_v = IPVERSION;
+	ip->ip_id = htons(ip_id++);
+	ip->ip_hl = sizeof (struct ip) >> 2;
+	ip->ip_ttl = udp_ttl;
+	ip->ip_sum = 0;
+	ip->ip_sum = htons(~ip_sum((unsigned char *)ip, ip->ip_hl));
 #if DO_ALIGN
-  bcopy((char *)ip, (char *)&pkt.data[pkt.off], sizeof(*ip));
+	bcopy((char *)ip, (char *)&pkt.data[pkt.off], sizeof(*ip));
 #endif
     
-  pkt.len += sizeof (struct udpiphdr);
+	pkt.len += sizeof (struct udpiphdr);
 
-  pkt.off += sizeof (struct udpiphdr);
+	pkt.off += sizeof (struct udpiphdr);
   
-  coreh = (struct corehdr *) &pkt.data[pkt.off];
-  coreh->th_opcode = htons((u_short)request);
+	coreh = (struct corehdr *) &pkt.data[pkt.off];
+	coreh->th_opcode = htons((u_short)request);
   
-  if (request == KDP_WRQ)
-    {
-      register char *cp;
-
-      cp = coreh->th_u.tu_rpl;
-      strcpy (cp, corename);
-      cp += strlen(corename);
-      *cp++ = '\0';
-      strcpy (cp, mode);
-      cp+= modelen;
-      *cp++ = '\0';
-    }
-  else
-    {
-      coreh->th_block = htonl((unsigned int) block);
-    }
+	if (request == KDP_WRQ)
+	{
+		register char *cp;
+
+		cp = coreh->th_u.tu_rpl;
+		strcpy (cp, corename);
+		cp += strlen(corename);
+		*cp++ = '\0';
+		strcpy (cp, mode);
+		cp+= modelen;
+		*cp++ = '\0';
+	}
+	else
+	{
+		coreh->th_block = htonl((unsigned int) block);
+	}
 
-  pkt.off -= sizeof (struct udpiphdr);
-  pkt.off -= sizeof (struct ether_header);
+	pkt.off -= sizeof (struct udpiphdr);
+	pkt.off -= sizeof (struct ether_header);
 
-  eh = (struct ether_header *)&pkt.data[pkt.off];
-  enaddr_copy(&kdp_current_mac_address, eh->ether_shost);
-  enaddr_copy(&router_mac, eh->ether_dhost);
-  eh->ether_type = htons(ETHERTYPE_IP);
+	eh = (struct ether_header *)&pkt.data[pkt.off];
+	enaddr_copy(&kdp_current_mac_address, eh->ether_shost);
+	enaddr_copy(&destination_mac, eh->ether_dhost);
+	eh->ether_type = htons(ETHERTYPE_IP);
     
-  pkt.len += sizeof (struct ether_header);
-  return coreh;
+	pkt.len += sizeof (struct ether_header);
+	return coreh;
 }
 
-int kdp_send_panic_packets (unsigned int request, char *corename, 
-			    unsigned int length, unsigned int txstart)
+int kdp_send_crashdump_data(unsigned int request, char *corename,
+    unsigned int length, caddr_t txstart)
 {
-  unsigned int txend = txstart + length;
-  int panic_error = 0;
+	caddr_t txend = txstart + length;
+	int panic_error = 0;
 
-  if (length <= SEGSIZE) {
-    if ((panic_error = kdp_send_panic_pkt (request, corename, length, (caddr_t) txstart)) < 0) {
-      printf ("kdp_send_panic_pkt failed with error %d\n", panic_error);
-      return panic_error ;
-    }
-  }
-  else
-    {
-      while (txstart <= (txend - SEGSIZE))  {
-	if ((panic_error = kdp_send_panic_pkt (KDP_DATA, NULL, SEGSIZE, (caddr_t) txstart)) < 0) {
-	  printf ("kdp_send_panic_pkt failed with error %d\n", panic_error);
-	  return panic_error;
+	if (length <= SEGSIZE) {
+		if ((panic_error = kdp_send_crashdump_pkt(request, corename, length, (caddr_t) txstart)) < 0) {
+			printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error);
+			return panic_error ;
+		}
 	}
-	txstart += SEGSIZE;
-	if (!(panic_block % 2000))
-	  printf(".");
-      }
-      if (txstart < txend) {
-	kdp_send_panic_pkt(request, corename, (txend - txstart), (caddr_t) txstart);
-      }
-    }
-  return 0;
+	else
+	{
+		while (txstart <= (txend - SEGSIZE))  {
+			if ((panic_error = kdp_send_crashdump_pkt(KDP_DATA, NULL, SEGSIZE, txstart)) < 0) {
+				printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error);
+				return panic_error;
+			}
+			txstart += SEGSIZE;
+			if (!(panic_block % 2000))
+				printf(".");
+		}
+		if (txstart < txend) {
+			kdp_send_crashdump_pkt(request, corename, (txend - txstart), txstart);
+		}
+	}
+	return 0;
 }
 
-int 
-kdp_send_panic_pkt (unsigned int request, char *corename, 
-		    unsigned int length, void *panic_data)
+int
+kdp_send_crashdump_pkt(unsigned int request, char *corename, 
+    unsigned int length, void *panic_data)
 {
-  struct corehdr *th = NULL;
-  int poll_count = 2500;
+	struct corehdr *th = NULL;
+	int poll_count = 2500;
   
-  char rretries = 0, tretries = 0;
-  /*
-  extern signed long gIODebuggerSemaphore;
-  */
-  pkt.off = pkt.len = 0;
+	char rretries = 0, tretries = 0;
+
+	pkt.off = pkt.len = 0;
   
-  if (request == KDP_WRQ) /* longer timeout for initial request */
-    poll_count += 1000;
+	if (request == KDP_WRQ) /* longer timeout for initial request */
+		poll_count += 1000;
 
 TRANSMIT_RETRY:
-  tretries++;
-
-  if (tretries > 2)
-    printf("TX retry #%d ", tretries );
-  
-  if (tretries >=15) {
-    /* This iokit layer issue can potentially 
-     *cause a hang, uncomment to check if it's happening.
-     */
-    /*
-      if (gIODebuggerSemaphore)
-      printf("The gIODebuggerSemaphore is raised, preventing packet transmission (2760413)\n");
-    */
-    
-    printf ("Cannot contact panic server, timing out.\n");
-    return (-3);
-  }
+	tretries++;
 
-  th = create_panic_header(request, corename, length, panic_block);
+	if (tretries >=15) {
+/* The crashdump server is unreachable for some reason. This could be a network
+ * issue or, if we've been especially unfortunate, we've hit Radar 2760413,
+ * which is a long standing problem with the IOKit polled mode network driver
+ * shim which can prevent transmits/receives completely.
+ */
+		printf ("Cannot contact panic server, timing out.\n");
+		return (-3);
+	}
 
-  if (request == KDP_DATA || request == KDP_SEEK) {
-      if (!kdp_vm_read ((caddr_t) panic_data, (caddr_t) th->th_data, length)) {
-	memset ((caddr_t) th->th_data, 'X', length);
-      }
-    }
+	if (tretries > 2)
+		printf("TX retry #%d ", tretries );
+  
+	th = create_panic_header(request, corename, length, panic_block);
 
-  (*kdp_en_send_pkt)(&pkt.data[pkt.off], pkt.len);
+	if (request == KDP_DATA) {
+		if (!kdp_vm_read((caddr_t) panic_data, (caddr_t) th->th_data, length)) {
+			memset ((caddr_t) th->th_data, 'X', length);
+		}
+	}
+	else if (request == KDP_SEEK) {
+		*(unsigned int *) th->th_data = htonl(*(unsigned int *) panic_data);
+	}
 
-  /* Now we have to listen for the ACK */
- RECEIVE_RETRY:
+	(*kdp_en_send_pkt)(&pkt.data[pkt.off], pkt.len);
 
-  while (!pkt.input && flag_panic_dump_in_progress && poll_count) {
-      kdp_poll();
-      poll_count--;
-    }
+	/* Listen for the ACK */
+RECEIVE_RETRY:
+	while (!pkt.input && flag_panic_dump_in_progress && poll_count) {
+		kdp_poll();
+		poll_count--;
+	}
 
-  if (pkt.input) {
+	if (pkt.input) {
     
-    pkt.input = FALSE;
+		pkt.input = FALSE;
     
-    th = (struct corehdr *) &pkt.data[pkt.off];
-    /* These will eventually have to be ntoh[ls]'ed as appropriate */
+		th = (struct corehdr *) &pkt.data[pkt.off];
     
-    if (th->th_opcode == KDP_ACK && th->th_block == panic_block) {
-    }
-    else
-      if (th->th_opcode == KDP_ERROR) {
-	printf("Panic server returned error %d, retrying\n", th->th_code);
-	poll_count = 1000;
-	goto TRANSMIT_RETRY;
-      }
-      else 
-	if (th->th_block == (panic_block -1)) {
-	  printf("RX retry ");
-	  if (++rretries > 1)
-	    goto TRANSMIT_RETRY;
-	  else
-	    goto RECEIVE_RETRY;
+		if (ntohs(th->th_opcode) == KDP_ACK && ntohl(th->th_block) == panic_block) {
+		}
+		else
+			if (ntohs(th->th_opcode) == KDP_ERROR) {
+				printf("Panic server returned error %d, retrying\n", ntohl(th->th_code));
+				poll_count = 1000;
+				goto TRANSMIT_RETRY;
+			}
+			else 
+				if (ntohl(th->th_block) == (panic_block - 1)) {
+					printf("RX retry ");
+					if (++rretries > 1)
+						goto TRANSMIT_RETRY;
+					else
+						goto RECEIVE_RETRY;
+				}
 	}
-  }
-  else
-    if (!flag_panic_dump_in_progress) /* we received a debugging packet, bail*/
-      {
-	printf("Received a debugger packet,transferring control to debugger\n");
-	/* Configure that if not set ..*/
-	kdp_flag |= DBG_POST_CORE;
-	return (-2);
-      }
-    else /* We timed out */
-      if (0 == poll_count) {
-	poll_count = 1000;
-	kdp_us_spin ((tretries%4) * panic_timeout); /* capped linear backoff */
-	goto TRANSMIT_RETRY;
-      }
+	else
+		if (!flag_panic_dump_in_progress) /* we received a debugging packet, bail*/
+		{
+			printf("Received a debugger packet,transferring control to debugger\n");
+			/* Configure that if not set ..*/
+			kdp_flag |= DBG_POST_CORE;
+			return (-2);
+		}
+		else /* We timed out */
+			if (0 == poll_count) {
+				poll_count = 1000;
+				kdp_us_spin ((tretries%4) * panic_timeout); /* capped linear backoff */
+				goto TRANSMIT_RETRY;
+			}
   
-  panic_block++;
+	panic_block++;
   
-  if (request == KDP_EOF)
-    printf ("\nTotal number of packets transmitted: %d\n", panic_block);
+	if (request == KDP_EOF)
+		printf("\nTotal number of packets transmitted: %d\n", panic_block);
   
-  return 1;
+	return 1;
 }
 
-/* Since we don't seem to have an isdigit() .. */
 static int 
 isdigit (char c)
 {
@@ -1085,10 +1325,7 @@ isdigit (char c)
 }
 /* From user mode Libc - this ought to be in a library */
 static char *
-strnstr(s, find, slen)
-        const char *s;
-        const char *find;
-        size_t slen;
+strnstr(char *s, const char *find, size_t slen)
 {
   char c, sc;
   size_t len;
@@ -1108,148 +1345,167 @@ strnstr(s, find, slen)
   return ((char *)s);
 }
 
+extern char version[];
+
 /* Horrid hack to extract xnu version if possible - a much cleaner approach
  * would be to have the integrator run a script which would copy the
  * xnu version into a string or an int somewhere at project submission
  * time - makes assumptions about sizeof(version), but will not fail if
  * it changes, but may be incorrect.
  */
- 
+/* 2006: Incorporated a change from Darwin user P. Lovell to extract
+ * the minor kernel version numbers from the version string.
+ */
 static int 
 kdp_get_xnu_version(char *versionbuf)
 {
-  extern const char version[];
-  char *versionpos;
-  char vstr[10];
-  int retval = -1;
-
-  strcpy(vstr, "custom");
-   if (version) { 
-       if (kdp_vm_read(version, versionbuf, 90)) {
-
- 	   versionbuf[89] = '\0'; 
-	   
-	   versionpos = strnstr(versionbuf, "xnu-", 80);
-
-	   if (versionpos) {
-	     strncpy (vstr, versionpos, (isdigit (versionpos[7]) ? 8 : 7));
-	     vstr[(isdigit (versionpos[7]) ? 8 : 7)] = '\0';
-	     retval = 0;
-	   }
-       }
-   }
-   strcpy(versionbuf, vstr);
-   return retval;
+
+	char *versionpos;
+	char vstr[20];
+	int retval = -1;
+	char *vptr;
+
+	strcpy(vstr, "custom");
+	if (version) {
+		if (kdp_vm_read(version, versionbuf, 95)) {
+			versionbuf[94] = '\0';
+			versionpos = strnstr(versionbuf, "xnu-", 90);
+			if (versionpos) {
+				strncpy(vstr, versionpos, sizeof(vstr));
+				vstr[sizeof(vstr)-1] = '\0';
+				vptr = vstr + 4; /* Begin after "xnu-" */
+				while (*vptr && (isdigit(*vptr) || *vptr == '.'))
+					vptr++;
+				*vptr = '\0';
+				/* Remove trailing period, if any */
+				if (*(--vptr) == '.')
+					*vptr = '\0';
+				retval = 0;
+			}
+		}
+	}
+	strcpy(versionbuf, vstr);
+	return retval;
 }
 
+extern char *inet_aton(const char *cp, struct in_addr *pin);
+extern int snprintf(char *str, size_t size, const char *format, ...);
+
 /* Primary dispatch routine for the system dump */
 void 
 kdp_panic_dump()
 {
-  char corename[50];
-  char coreprefix[10];
-  int panic_error;
+	char corename[50];
+	char coreprefix[10];
+	int panic_error;
 
-  extern vm_map_t kernel_map;
+	uint64_t 	abstime;
+	uint32_t	current_ip = ntohl(kdp_current_ip_address);
 
-  extern char *inet_aton(const char *cp, struct in_addr *pin);
-
-  uint64_t abstime;
-
-  printf ("Entering system dump routine\n");
+	if (flag_panic_dump_in_progress) {
+		printf("System dump aborted.\n");
+		goto panic_dump_exit;
+	}
+		
+	printf("Entering system dump routine\n");
   
-  if (!panicd_specified) {
-      printf ("A panic server was not specified in the boot-args, terminating kernel core dump.\n");
-      goto panic_dump_exit;
-    }
+	if (!panicd_specified) {
+		printf("A dump server was not specified in the boot-args, terminating kernel core dump.\n");
+		goto panic_dump_exit;
+	}
 
-  flag_panic_dump_in_progress = 1;
-  not_in_kdp = 0;
+	flag_panic_dump_in_progress = TRUE;
+	not_in_kdp = 0;
 
-  if (pkt.input)
-    kdp_panic("kdp_panic_dump");
+	if (pkt.input)
+		kdp_panic("kdp_panic_dump: unexpected pending input packet");
 
-  kdp_get_xnu_version((char *) &pkt.data[0]);
+	kdp_get_xnu_version((char *) &pkt.data[0]);
 
-  /* Panic log bit takes precedence over core dump bit */
-  if ((panicstr != (char *) 0) && (kdp_flag & PANIC_LOG_DUMP))
-    strncpy(coreprefix, "paniclog", sizeof(coreprefix));
-  else
-    strncpy(coreprefix, "core", sizeof(coreprefix));
+	/* Panic log bit takes precedence over core dump bit */
+	if ((panicstr != (char *) 0) && (kdp_flag & PANIC_LOG_DUMP))
+		strncpy(coreprefix, "paniclog", sizeof(coreprefix));
+	else
+		strncpy(coreprefix, "core", sizeof(coreprefix));
   
-  abstime = mach_absolute_time();
-  pkt.data[10] = '\0';
-  snprintf (corename, sizeof(corename), "%s-%s-%d.%d.%d.%d-%x", 
+	abstime = mach_absolute_time();
+	pkt.data[20] = '\0';
+	snprintf (corename, sizeof(corename), "%s-%s-%d.%d.%d.%d-%x", 
 	    coreprefix, &pkt.data[0],
-	   (kdp_current_ip_address & 0xff000000) >> 24,
-	   (kdp_current_ip_address & 0xff0000) >> 16,
-	   (kdp_current_ip_address & 0xff00) >> 8,
-	   (kdp_current_ip_address & 0xff),
-	   (unsigned int) (abstime & 0xffffffff));
-
-  if (0 == inet_aton(panicd_ip_str, (struct in_addr *) &panic_server_ip)) {
-      printf("inet_aton() failed interpreting %s as a panic server IP\n", 
-	     panicd_ip_str);
-    }
-  else
-    printf("Attempting connection to panic server configured at IP %s\n", 
-	   panicd_ip_str);
+	    (current_ip & 0xff000000) >> 24,
+	    (current_ip & 0xff0000) >> 16,
+	    (current_ip & 0xff00) >> 8,
+	    (current_ip & 0xff),
+	    (unsigned int) (abstime & 0xffffffff));
+
+	if (0 == inet_aton(panicd_ip_str, (struct in_addr *) &panic_server_ip)) {
+		printf("inet_aton() failed interpreting %s as a panic server IP\n", panicd_ip_str);
+	}
+	else
+		printf("Attempting connection to panic server configured at IP %s, port %d\n", panicd_ip_str, panicd_port);
 
-  if (router_specified) {
-      if (0 == inet_aton(router_ip_str, (struct in_addr *) &parsed_router_ip)){
-	  printf("inet_aton() failed interpreting %s as an IP\n", router_ip);
+	destination_mac = router_mac;
+
+	if (kdp_arp_resolve(panic_server_ip, &temp_mac)) {
+		printf("Resolved %s's (or proxy's) link level address\n", panicd_ip_str);
+		destination_mac = temp_mac;
 	}
-      else {
-	  router_ip = parsed_router_ip;
-	  printf("Routing through specified router IP %s (%d)\n", router_ip_str, router_ip);
-	  /* We will eventually need to resolve the router's MAC ourselves,
-	   * if one is specified,rather than being set through the BSD callback
-	   * but the _router_ip option does not function currently
-	   */
+	else {
+		if (!flag_panic_dump_in_progress) goto panic_dump_exit;
+		if (router_specified) {
+			if (0 == inet_aton(router_ip_str, (struct in_addr *) &parsed_router_ip))
+				printf("inet_aton() failed interpreting %s as an IP\n", router_ip_str);
+			else {
+				router_ip = parsed_router_ip;
+				if (kdp_arp_resolve(router_ip, &temp_mac)) {
+					destination_mac = temp_mac;
+					printf("Routing through specified router IP %s (%d)\n", router_ip_str, router_ip);
+				}
+			}
+		}
 	}
-    }
 
-  printf("Routing via router MAC address: %02x:%02x:%02x:%02x:%02x:%02x\n",
-	  router_mac.ether_addr_octet[0] & 0xff,
-	  router_mac.ether_addr_octet[1] & 0xff,
-	  router_mac.ether_addr_octet[2] & 0xff,
-	  router_mac.ether_addr_octet[3] & 0xff,
-	  router_mac.ether_addr_octet[4] & 0xff,
-	  router_mac.ether_addr_octet[5] & 0xff);
+	if (!flag_panic_dump_in_progress) goto panic_dump_exit;
 
-  printf("Kernel map size is %llu\n", (unsigned long long) get_vmmap_size(kernel_map));
-  printf ("Sending write request for %s\n", corename);  
+	printf("Transmitting packets to link level address: %02x:%02x:%02x:%02x:%02x:%02x\n",
+	    destination_mac.ether_addr_octet[0] & 0xff,
+	    destination_mac.ether_addr_octet[1] & 0xff,
+	    destination_mac.ether_addr_octet[2] & 0xff,
+	    destination_mac.ether_addr_octet[3] & 0xff,
+	    destination_mac.ether_addr_octet[4] & 0xff,
+	    destination_mac.ether_addr_octet[5] & 0xff);
 
-  if ((panic_error = kdp_send_panic_pkt (KDP_WRQ, corename, 0 , NULL)) < 0) {
-      printf ("kdp_send_panic_pkt failed with error %d\n", panic_error);
-      goto panic_dump_exit;
-    }
+	printf("Kernel map size is %llu\n", (unsigned long long) get_vmmap_size(kernel_map));
+	printf("Sending write request for %s\n", corename);  
 
-  /* Just the panic log requested */
-  if ((panicstr != (char *) 0) && (kdp_flag & PANIC_LOG_DUMP)) {
-    printf("Transmitting panic log, please wait: ");
-    kdp_send_panic_packets (KDP_DATA, corename, (debug_buf_ptr - debug_buf), (unsigned int) debug_buf);
-    kdp_send_panic_pkt (KDP_EOF, NULL, 0, ((void *) 0));
-    printf("Please file a bug report on this panic, if possible.\n");
-    goto panic_dump_exit;
-    }
+	if ((panic_error = kdp_send_crashdump_pkt(KDP_WRQ, corename, 0 , NULL)) < 0) {
+		printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error);
+		goto panic_dump_exit;
+	}
+
+	/* Just the panic log requested */
+	if ((panicstr != (char *) 0) && (kdp_flag & PANIC_LOG_DUMP)) {
+		printf("Transmitting panic log, please wait: ");
+		kdp_send_crashdump_data(KDP_DATA, corename, (debug_buf_ptr - debug_buf), debug_buf);
+		kdp_send_crashdump_pkt (KDP_EOF, NULL, 0, ((void *) 0));
+		printf("Please file a bug report on this panic, if possible.\n");
+		goto panic_dump_exit;
+	}
   
-  /* We want a core dump if we're here */
-  kern_dump();
+	/* We want a core dump if we're here */
+	kern_dump();
 panic_dump_exit:
-  not_in_kdp = 1;
-  flag_panic_dump_in_progress = 0;
-  panic_block = 0;
-  pkt.input = FALSE;
-  pkt.len = 0;
-  kdp_reset();
-  return;
+	abort_panic_transfer();
+	pkt.input = FALSE;
+	pkt.len = 0;
+	kdp_reset();
+	return;
 }
 
 void 
-abort_panic_transfer()
+abort_panic_transfer(void)
 {
-  flag_panic_dump_in_progress = 0;
-  not_in_kdp = 1;
-  panic_block = 0;
+	flag_panic_dump_in_progress = FALSE;
+	not_in_kdp = 1;
+	panic_block = 0;
 }
diff --git a/osfmk/kdp/ml/i386/kdp_machdep.c b/osfmk/kdp/ml/i386/kdp_machdep.c
index 2eb2d4c3a..8d9194119 100644
--- a/osfmk/kdp/ml/i386/kdp_machdep.c
+++ b/osfmk/kdp/ml/i386/kdp_machdep.c
@@ -26,6 +26,7 @@
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
+#include <mach_kdp.h>
 #include <mach/mach_types.h>
 #include <mach/machine.h>
 #include <mach/exception_types.h>
@@ -33,6 +34,15 @@
 #include <i386/trap.h>
 #include <i386/mp.h>
 #include <kdp/kdp_internal.h>
+#include <mach-o/loader.h>
+#include <mach-o/nlist.h>
+#include <IOKit/IOPlatformExpert.h> /* for PE_halt_restart */
+#include <kern/machine.h> /* for halt_all_cpus */
+
+#include <kern/thread.h>
+#include <i386/thread.h>
+#include <vm/vm_map.h>
+#include <i386/pmap.h>
 
 #define KDP_TEST_HARNESS 0
 #if KDP_TEST_HARNESS
@@ -41,7 +51,8 @@
 #define dprintf(x)
 #endif
 
-extern void	kdreboot(void);
+extern cpu_type_t cpuid_cputype(void);
+extern cpu_subtype_t cpuid_cpusubtype(void);
 
 void		print_saved_state(void *);
 void		kdp_call(void);
@@ -50,12 +61,14 @@ boolean_t	kdp_call_kdb(void);
 void		kdp_getstate(i386_thread_state_t *);
 void		kdp_setstate(i386_thread_state_t *);
 void		kdp_print_phys(int);
-void		kdp_i386_backtrace(void *, int);
-void		kdp_i386_trap(
-			unsigned int,
-			struct i386_saved_state	*,
-			kern_return_t,
-			vm_offset_t);
+
+int
+machine_trace_thread(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p);
+
+int
+machine_trace_thread64(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p);
+
+extern unsigned kdp_vm_read(caddr_t src, caddr_t dst, unsigned len);
 
 void
 kdp_exception(
@@ -116,13 +129,13 @@ kdp_exception_ack(
 
 void
 kdp_getstate(
-    i386_thread_state_t		*state
+    x86_thread_state32_t	*state
 )
 {
-    static i386_thread_state_t	null_state;
-    struct i386_saved_state	*saved_state;
+    static x86_thread_state32_t	null_state;
+    x86_saved_state32_t	*saved_state;
     
-    saved_state = (struct i386_saved_state *)kdp.saved_state;
+    saved_state = (x86_saved_state32_t *)kdp.saved_state;
     
     *state = null_state;	
     state->eax = saved_state->eax;
@@ -153,12 +166,12 @@ kdp_getstate(
 
 void
 kdp_setstate(
-    i386_thread_state_t		*state
+    x86_thread_state32_t	*state
 )
 {
-    struct i386_saved_state	*saved_state;
+    x86_saved_state32_t		*saved_state;
     
-    saved_state = (struct i386_saved_state *)kdp.saved_state;
+    saved_state = (x86_saved_state32_t *)kdp.saved_state;
 
     saved_state->eax = state->eax;
     saved_state->ebx = state->ebx;
@@ -186,20 +199,21 @@ kdp_machine_read_regs(
     __unused int *size
 )
 {
-    static i386_thread_fpstate_t null_fpstate;
+    static struct i386_float_state  null_fpstate;
 
     switch (flavor) {
 
-    case i386_THREAD_STATE:
+    case OLD_i386_THREAD_STATE:
+    case x86_THREAD_STATE32:
 	dprintf(("kdp_readregs THREAD_STATE\n"));
-	kdp_getstate((i386_thread_state_t *)data);
-	*size = sizeof (i386_thread_state_t);
+	kdp_getstate((x86_thread_state32_t *)data);
+	*size = sizeof (x86_thread_state32_t);
 	return KDPERR_NO_ERROR;
 	
-    case i386_THREAD_FPSTATE:
+    case x86_FLOAT_STATE32:
 	dprintf(("kdp_readregs THREAD_FPSTATE\n"));
-	*(i386_thread_fpstate_t *)data = null_fpstate;
-	*size = sizeof (i386_thread_fpstate_t);
+	*(x86_float_state32_t *)data = null_fpstate;
+	*size = sizeof (x86_float_state32_t);
 	return KDPERR_NO_ERROR;
 	
     default:
@@ -219,12 +233,13 @@ kdp_machine_write_regs(
 {
     switch (flavor) {
 
-    case i386_THREAD_STATE:
+    case OLD_i386_THREAD_STATE:
+    case x86_THREAD_STATE32:
 	dprintf(("kdp_writeregs THREAD_STATE\n"));
-	kdp_setstate((i386_thread_state_t *)data);
+	kdp_setstate((x86_thread_state32_t *)data);
 	return KDPERR_NO_ERROR;
 	
-    case i386_THREAD_FPSTATE:
+    case x86_FLOAT_STATE32:
 	dprintf(("kdp_writeregs THREAD_FPSTATE\n"));
 	return KDPERR_NO_ERROR;
 	
@@ -252,9 +267,8 @@ kdp_machine_hostinfo(
         hostinfo->cpus_mask |= (1 << i);
     }
 
-   /* FIXME?? */
-    hostinfo->cpu_type = CPU_TYPE_I386;
-    hostinfo->cpu_subtype = CPU_SUBTYPE_486;
+    hostinfo->cpu_type = cpuid_cputype();
+    hostinfo->cpu_subtype = cpuid_cpusubtype();
 }
 
 void
@@ -270,7 +284,12 @@ kdp_panic(
 void
 kdp_reboot(void)
 {
-    kdreboot();
+	printf("Attempting system restart...");
+	/* Call the platform specific restart*/
+	if (PE_halt_restart)
+		(*PE_halt_restart)(kPERestartCPU);
+	/* If we do reach this, give up */
+	halt_all_cpus(TRUE);
 }
 
 int
@@ -299,14 +318,14 @@ kdp_us_spin(int usec)
 
 void print_saved_state(void *state)
 {
-    struct i386_saved_state	*saved_state;
+    x86_saved_state32_t		*saved_state;
 
     saved_state = state;
 
 	kprintf("pc = 0x%x\n", saved_state->eip);
-	kprintf("cr3= 0x%x\n", saved_state->cr2);
+	kprintf("cr2= 0x%x\n", saved_state->cr2);
 	kprintf("rp = TODO FIXME\n");
-	kprintf("sp = 0x%x\n", saved_state->esp);
+	kprintf("sp = 0x%x\n", saved_state);
 
 }
 
@@ -352,51 +371,24 @@ kdp_print_phys(int src)
 
 }
 
-
-#define MAX_FRAME_DELTA		65536
-
-void
-kdp_i386_backtrace(void	*_frame, int nframes)
-{
-	cframe_t	*frame = (cframe_t *)_frame;
-	int i;
-
-	for (i=0; i<nframes; i++) {
-	    if ((vm_offset_t)frame < VM_MIN_KERNEL_ADDRESS ||
-	        (vm_offset_t)frame > VM_MAX_KERNEL_ADDRESS) {
-		goto invalid;
-	    }
-	    kprintf("frame 0x%x called by 0x%x ",
-		frame, frame->caller);
-	    kprintf("args 0x%x 0x%x 0x%x 0x%x\n",
-		frame->args[0], frame->args[1],
-		frame->args[2], frame->args[3]);
-	    if ((frame->prev < frame) ||	/* wrong direction */
-	    	((frame->prev - frame) > MAX_FRAME_DELTA)) {
-		goto invalid;
-	    }
-	    frame = frame->prev;
-	}
-	return;
-invalid:
-	kprintf("invalid frame pointer 0x%x\n",frame);
-}
-
-void
+boolean_t
 kdp_i386_trap(
-    unsigned int		trapno,
-    struct i386_saved_state	*saved_state,
+    unsigned int	trapno,
+    x86_saved_state32_t	*saved_state,
     kern_return_t	result,
     vm_offset_t		va
 )
 {
     unsigned int exception, subcode = 0, code;
 
-    mp_kdp_enter();
-
-    if (trapno != T_INT3 && trapno != T_DEBUG)
+    if (trapno != T_INT3 && trapno != T_DEBUG) {
     	kprintf("unexpected kernel trap 0x%x eip 0x%x cr2 0x%x \n",
-		trapno, saved_state->eip, saved_state->esp);
+		trapno, saved_state->eip, saved_state->cr2);
+	if (!kdp.is_conn)
+	    return FALSE;
+    }	
+
+    mp_kdp_enter();
 
     switch (trapno) {
     
@@ -461,11 +453,11 @@ kdp_i386_trap(
 	break;
     }
 
-    kdp_i386_backtrace((void *) saved_state->ebp, 10);
-
     kdp_raise_exception(exception, code, subcode, saved_state);
 
     mp_kdp_exit();
+
+    return TRUE;
 }
 
 boolean_t 
@@ -480,3 +472,84 @@ kdp_ml_get_breakinsn(void)
 {
   return 0xcc;
 }
+extern pmap_t kdp_pmap;
+
+#define RETURN_OFFSET 4
+int
+machine_trace_thread(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p)
+{
+	uint32_t *tracebuf = (uint32_t *)tracepos;
+	uint32_t fence = 0;
+	uint32_t stackptr = 0;
+	uint32_t stacklimit = 0xfc000000;
+	int framecount = 0;
+	uint32_t init_eip = 0;
+	uint32_t prevsp = 0;
+	uint32_t framesize = 2 * sizeof(vm_offset_t);
+	
+	if (user_p) {
+	        x86_saved_state32_t	*iss32;
+		
+		iss32 = USER_REGS32(thread);
+
+		init_eip = iss32->eip;
+		stackptr = iss32->ebp;
+
+		/* This bound isn't useful, but it doesn't hinder us*/
+		stacklimit = 0xffffffff;
+		kdp_pmap = thread->task->map->pmap;
+	}
+	else {
+		/*Examine the i386_saved_state at the base of the kernel stack*/
+		stackptr = STACK_IKS(thread->kernel_stack)->k_ebp;
+		init_eip = STACK_IKS(thread->kernel_stack)->k_eip;
+	}
+
+	*tracebuf++ = init_eip;
+
+	for (framecount = 0; framecount < nframes; framecount++) {
+
+		if ((tracebound - ((uint32_t) tracebuf)) < (4 * framesize)) {
+			tracebuf--;
+			break;
+		}
+
+		*tracebuf++ = stackptr;
+/* Invalid frame, or hit fence */
+		if (!stackptr || (stackptr == fence)) {
+			break;
+		}
+		/* Stack grows downward */
+		if (stackptr < prevsp) {
+			break;
+		}
+		/* Unaligned frame */
+		if (stackptr & 0x0000003) {
+			break;
+		}
+		if (stackptr > stacklimit) {
+			break;
+		}
+
+		if (kdp_vm_read((caddr_t) (stackptr + RETURN_OFFSET), (caddr_t) tracebuf, sizeof(caddr_t)) != sizeof(caddr_t)) {
+			break;
+		}
+		tracebuf++;
+		
+		prevsp = stackptr;
+		if (kdp_vm_read((caddr_t) stackptr, (caddr_t) &stackptr, sizeof(caddr_t)) != sizeof(caddr_t)) {
+			*tracebuf++ = 0;
+			break;
+		}
+	}
+
+	kdp_pmap = 0;
+
+	return ((uint32_t) tracebuf - tracepos);
+}
+
+/* This is a stub until the x86 64-bit model becomes clear */
+int
+machine_trace_thread64(__unused thread_t thread, __unused uint32_t tracepos, __unused uint32_t tracebound, __unused int nframes, __unused boolean_t user_p) {
+	return 0;
+}
diff --git a/osfmk/kdp/ml/i386/kdp_vm.c b/osfmk/kdp/ml/i386/kdp_vm.c
index 474d30c88..bc23863c1 100644
--- a/osfmk/kdp/ml/i386/kdp_vm.c
+++ b/osfmk/kdp/ml/i386/kdp_vm.c
@@ -29,14 +29,78 @@
 #include <mach/vm_attributes.h>
 #include <mach/vm_param.h>
 #include <libsa/types.h>
- 
+
+#include <vm/vm_map.h>
+#include <i386/pmap.h>
+
+#include <kdp/kdp_core.h>
+#include <kdp/kdp_internal.h>
+#include <mach-o/loader.h>
+#include <mach/vm_map.h>
+#include <mach/vm_statistics.h>
+#include <mach/thread_status.h>
+#include <i386/thread.h>
+
+#include <vm/vm_protos.h>
+#include <vm/vm_kern.h>
+
 unsigned kdp_vm_read( caddr_t, caddr_t, unsigned);
 unsigned kdp_vm_write( caddr_t, caddr_t, unsigned);
-unsigned kdp_copy_kmem( caddr_t, caddr_t, unsigned);
-int	 kern_dump(void);
+
+boolean_t kdp_trans_off = 0;
+uint32_t kdp_src_high32 = 0;
+extern pmap_paddr_t avail_start, avail_end;
+
+extern void bcopy_phys(addr64_t from, addr64_t to, int size);
+static addr64_t kdp_vtophys(pmap_t pmap, addr64_t va);
+
+pmap_t kdp_pmap = 0;
 
 unsigned int not_in_kdp = 1; /* Cleared when we begin to access vm functions in kdp */
 
+extern vm_offset_t sectTEXTB, sectDATAB, sectLINKB, sectPRELINKB;
+extern int sectSizeTEXT, sectSizeDATA, sectSizeLINK, sectSizePRELINK;
+
+int	kern_dump(void);
+int	kdp_dump_trap(int type, x86_saved_state32_t *regs);
+
+typedef struct {
+	int	flavor;			/* the number for this flavor */
+	mach_msg_type_number_t	count;	/* count of ints in this flavor */
+} mythread_state_flavor_t;
+
+static mythread_state_flavor_t thread_flavor_array [] = { 
+	{x86_THREAD_STATE32, x86_THREAD_STATE32_COUNT}
+};
+
+static int kdp_mynum_flavors = 1;
+static int MAX_TSTATE_FLAVORS = 1;
+
+typedef struct {
+	vm_offset_t header; 
+	int  hoffset;
+	mythread_state_flavor_t *flavors;
+	int tstate_size;
+} tir_t;
+
+char command_buffer[512];
+
+static addr64_t
+kdp_vtophys(
+	pmap_t pmap,
+	addr64_t va)
+{
+	addr64_t    pa;
+	ppnum_t pp;
+/* Clear high 32 - pmap_find_phys() may panic() otherwise */
+	va &= 0xFFFFFFFFULL;
+	pp = pmap_find_phys(pmap, va);
+	if(!pp) return 0;
+	
+	pa = ((addr64_t)pp << 12) | (va & 0x0000000000000FFFULL);
+	return(pa);
+}
+
 /*
  *
  */
@@ -45,7 +109,55 @@ unsigned kdp_vm_read(
 	caddr_t dst, 
 	unsigned len)
 {
-	return kdp_copy_kmem(src, dst, len);
+	addr64_t cur_virt_src = (addr64_t)((unsigned int)src | (((uint64_t)kdp_src_high32) << 32));
+	addr64_t cur_virt_dst = (addr64_t)((unsigned int)dst);
+	addr64_t cur_phys_dst, cur_phys_src;
+	unsigned resid = len;
+	unsigned cnt = 0;
+	pmap_t src_pmap = kernel_pmap;
+
+/* If a different pmap has been specified with kdp_pmap, use it to translate the
+ * source (cur_virt_src); otherwise, the source is translated using the
+ * kernel_pmap.
+ */
+	if (kdp_pmap)
+		src_pmap = kdp_pmap;
+
+	while (resid != 0) {
+/* Translate, unless kdp_trans_off is set */
+		if (!kdp_trans_off) {
+			if (!(cur_phys_src = kdp_vtophys(src_pmap,
+				    cur_virt_src)))
+				goto exit;
+		}
+		else
+			cur_phys_src = cur_virt_src;
+
+/* Always translate the destination buffer using the kernel_pmap */
+		if(!(cur_phys_dst = kdp_vtophys(kernel_pmap, cur_virt_dst)))
+			goto exit;
+
+		/* Validate physical page numbers when performing a crashdump */
+		if (not_in_kdp == 0)
+			if (!pmap_valid_page(i386_btop(cur_phys_dst)) || !pmap_valid_page(i386_btop(cur_phys_src)))
+				goto exit;
+
+/* Get length left on page */
+		cnt = PAGE_SIZE - (cur_phys_src & PAGE_MASK);
+		if (cnt > (PAGE_SIZE - (cur_phys_dst & PAGE_MASK)))
+			cnt = PAGE_SIZE - (cur_phys_dst & PAGE_MASK);
+		if (cnt > resid)
+			cnt = resid;
+
+/* Do a physical copy */
+		bcopy_phys(cur_phys_src, cur_phys_dst, cnt);
+
+		cur_virt_src += cnt;
+		cur_virt_dst += cnt;
+		resid -= cnt;
+	}
+exit:
+	return (len - resid);
 }
 
 /*
@@ -56,11 +168,375 @@ unsigned kdp_vm_write(
         caddr_t dst,
         unsigned len)
 {       
-	return kdp_copy_kmem(src, dst, len);
+	addr64_t cur_virt_src, cur_virt_dst;
+	addr64_t cur_phys_src, cur_phys_dst;
+	unsigned resid, cnt, cnt_src, cnt_dst;
+
+#ifdef KDP_VM_WRITE_DEBUG
+	printf("kdp_vm_write: src %x dst %x len %x - %08X %08X\n", src, dst, len, ((unsigned long *)src)[0], ((unsigned long *)src)[1]);
+#endif
+
+	cur_virt_src = (addr64_t)((unsigned int)src);
+	cur_virt_dst = (addr64_t)((unsigned int)dst);
+
+	resid = len;
+
+	while (resid != 0) {
+		if ((cur_phys_dst = kdp_vtophys(kernel_pmap, cur_virt_dst)) == 0) 
+			goto exit;
+
+		if ((cur_phys_src = kdp_vtophys(kernel_pmap, cur_virt_src)) == 0) 
+			goto exit;
+
+		cnt_src = ((cur_phys_src + PAGE_SIZE) & (PAGE_MASK)) - cur_phys_src;
+		cnt_dst = ((cur_phys_dst + PAGE_SIZE) & (PAGE_MASK)) - cur_phys_dst;
+
+		if (cnt_src > cnt_dst)
+			cnt = cnt_dst;
+		else
+			cnt = cnt_src;
+		if (cnt > resid) 
+			cnt = resid;
+
+		bcopy_phys(cur_phys_src, cur_phys_dst, cnt);		/* Copy stuff over */
+
+		cur_virt_src +=cnt;
+		cur_virt_dst +=cnt;
+		resid -= cnt;
+	}
+exit:
+	return (len - resid);
 }
 
-/* A stub until i386 support is added for remote kernel core dumps */
-int kern_dump(void)
+static void
+kern_collectth_state(thread_t thread, tir_t *t)
 {
-  return 0;
+	vm_offset_t	header;
+	int  hoffset, i ;
+	mythread_state_flavor_t *flavors;
+	struct thread_command	*tc;
+	/*
+	 *	Fill in thread command structure.
+	 */
+	header = t->header;
+	hoffset = t->hoffset;
+	flavors = t->flavors;
+	
+	tc = (struct thread_command *) (header + hoffset);
+	tc->cmd = LC_THREAD;
+	tc->cmdsize = sizeof(struct thread_command) + t->tstate_size;
+	hoffset += sizeof(struct thread_command);
+	/*
+	 * Follow with a struct thread_state_flavor and
+	 * the appropriate thread state struct for each
+	 * thread state flavor.
+	 */
+	for (i = 0; i < kdp_mynum_flavors; i++) {
+		*(mythread_state_flavor_t *)(header+hoffset) =
+		    flavors[i];
+		hoffset += sizeof(mythread_state_flavor_t);
+		/* Locate and obtain the non-volatile register context
+		 * for this kernel thread. This should ideally be
+		 * encapsulated in machine_thread_get_kern_state()
+		 * but that routine appears to have been co-opted
+		 * by CHUD to obtain pre-interrupt state.
+		 */
+		if (flavors[i].flavor == x86_THREAD_STATE32) {
+			x86_thread_state32_t *tstate = (x86_thread_state32_t *) (header + hoffset);
+			vm_offset_t kstack;
+			bzero(tstate, x86_THREAD_STATE32_COUNT * sizeof(int));
+			if ((kstack = thread->kernel_stack) != 0){
+				struct x86_kernel_state32 *iks = STACK_IKS(kstack);
+				tstate->ebx = iks->k_ebx;
+				tstate->esp = iks->k_esp;
+				tstate->ebp = iks->k_ebp;
+				tstate->edi = iks->k_edi;
+				tstate->esi = iks->k_esi;
+				tstate->eip = iks->k_eip;
+		}
+		}
+		else if (machine_thread_get_kern_state(thread,
+			flavors[i].flavor, (thread_state_t) (header+hoffset),
+			&flavors[i].count) != KERN_SUCCESS)
+			printf ("Failure in machine_thread_get_kern_state()\n");
+		hoffset += flavors[i].count*sizeof(int);
+	}
+
+	t->hoffset = hoffset;
+}
+
+/* Intended to be called from the kernel trap handler if an unrecoverable fault
+ * occurs during a crashdump (which shouldn't happen since we validate mappings
+ * and so on). This should be reworked to attempt some form of recovery.
+ */
+int
+kdp_dump_trap(
+	int type,
+	__unused x86_saved_state32_t	*saved_state)
+{
+	printf ("An unexpected trap (type %d) occurred during the system dump, terminating.\n", type);
+	kdp_send_crashdump_pkt (KDP_EOF, NULL, 0, ((void *) 0));
+	abort_panic_transfer();
+	kdp_flag &= ~KDP_PANIC_DUMP_ENABLED;
+	kdp_flag &= ~PANIC_CORE_ON_NMI;
+	kdp_flag &= ~PANIC_LOG_DUMP;
+
+	kdp_reset();
+
+	kdp_raise_exception(EXC_BAD_ACCESS, 0, 0, kdp.saved_state);
+	return( 0 );
+}
+
+int
+kern_dump(void)
+{
+	vm_map_t	map;
+	unsigned int	thread_count, segment_count;
+	unsigned int	command_size = 0, header_size = 0, tstate_size = 0;
+	unsigned int	hoffset = 0, foffset = 0, nfoffset = 0,  vmoffset = 0;
+	unsigned int	max_header_size = 0;
+	vm_offset_t	header;
+	struct mach_header	*mh;
+	struct segment_command	*sc;
+	vm_size_t	size;
+	vm_prot_t	prot = 0;
+	vm_prot_t	maxprot = 0;
+	vm_inherit_t	inherit = 0;
+	mythread_state_flavor_t flavors[MAX_TSTATE_FLAVORS];
+	vm_size_t	nflavors;
+	vm_size_t	i;
+	uint32_t	nesting_depth = 0;
+	kern_return_t	kret = 0;
+	struct vm_region_submap_info_64	vbr;
+	mach_msg_type_number_t	vbrcount  = 0;
+	tir_t tir1;
+
+	int error = 0;
+	int panic_error = 0;
+	unsigned int txstart = 0;
+	unsigned int mach_section_count = 4;
+	unsigned int num_sects_txed = 0;
+
+	map = kernel_map;
+
+	not_in_kdp = 0; /* Signal vm functions not to acquire locks */
+
+	thread_count = 1;
+	segment_count = get_vmmap_entries(map); 
+  
+	printf("Kernel map has %d entries\n", segment_count);
+
+	nflavors = kdp_mynum_flavors;
+	bcopy((char *)thread_flavor_array,(char *) flavors,sizeof(thread_flavor_array));
+
+	for (i = 0; i < nflavors; i++)
+		tstate_size += sizeof(mythread_state_flavor_t) +
+		    (flavors[i].count * sizeof(int));
+
+	command_size = (segment_count + mach_section_count) *
+	    sizeof(struct segment_command) +
+	    thread_count * sizeof(struct thread_command) +
+	    tstate_size * thread_count;
+
+	header_size = command_size + sizeof(struct mach_header);
+	header = (vm_offset_t) command_buffer;
+	
+	/*
+	 *	Set up Mach-O header for currently executing 32 bit kernel.
+	 */
+	printf ("Generated Mach-O header size was %d\n", header_size);
+
+	mh = (struct mach_header *) header;
+	mh->magic = MH_MAGIC;
+	mh->cputype = cpu_type();
+	mh->cpusubtype = cpu_subtype();
+	mh->filetype = MH_CORE;
+	mh->ncmds = segment_count + thread_count + mach_section_count;
+	mh->sizeofcmds = command_size;
+	mh->flags = 0;
+
+	hoffset = sizeof(struct mach_header);	/* offset into header */
+	foffset = round_page_32(header_size);	/* offset into file */
+	/* Padding */
+	if ((foffset - header_size) < (4*sizeof(struct segment_command))) {
+		foffset += ((4*sizeof(struct segment_command)) - (foffset-header_size)); 
+	}
+
+	max_header_size = foffset;
+
+	vmoffset = VM_MIN_ADDRESS;		/* offset into VM */
+
+	/* Transmit the Mach-O MH_CORE header, and seek forward past the 
+	 * area reserved for the segment and thread commands 
+	 * to begin data transmission 
+	 */
+
+	if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(nfoffset) , &nfoffset)) < 0) { 
+		printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error);
+		error = panic_error;
+		goto out;
+	} 
+
+	if ((panic_error = kdp_send_crashdump_data (KDP_DATA, NULL, sizeof(struct mach_header), (caddr_t) mh) < 0)) {
+		printf ("kdp_send_crashdump_data failed with error %d\n", panic_error);
+		error = panic_error;
+		goto out;
+	}
+
+	if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(foffset) , &foffset) < 0)) {
+		printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error);
+		error = panic_error;
+		goto out;
+	}
+	printf ("Transmitting kernel state, please wait: ");
+
+	while ((segment_count > 0) || (kret == KERN_SUCCESS)){
+		/* Check if we've transmitted all the kernel sections */
+		if (num_sects_txed == mach_section_count) {
+
+			while (1) {
+
+				/*
+				 *	Get region information for next region.
+				 */
+
+				vbrcount = VM_REGION_SUBMAP_INFO_COUNT_64;
+				if((kret = vm_region_recurse_64(map, 
+					    &vmoffset, &size, &nesting_depth, 
+					    (vm_region_recurse_info_t)&vbr,
+					    &vbrcount)) != KERN_SUCCESS) {
+					break;
+				}
+
+				if(vbr.is_submap) {
+					nesting_depth++;
+					continue;
+				} else {
+					break;
+				}
+			}
+
+			if(kret != KERN_SUCCESS)
+				break;
+
+			prot = vbr.protection;
+			maxprot = vbr.max_protection;
+			inherit = vbr.inheritance;
+		}
+		else
+		{
+			switch (num_sects_txed) {
+			case 0:
+				/* Transmit the kernel text section */
+				vmoffset = sectTEXTB;
+				size = sectSizeTEXT;
+				break;
+			case 1:
+				vmoffset = sectDATAB;
+				size = sectSizeDATA;
+				break;
+			case 2:
+				vmoffset = sectPRELINKB;
+				size = sectSizePRELINK;
+				break;
+			case 3:
+				vmoffset = sectLINKB;
+				size = sectSizeLINK;
+				break;
+			}
+			num_sects_txed++;
+		}
+		/*
+		 *	Fill in segment command structure.
+		 */
+    
+		if (hoffset > max_header_size)
+			break;
+		sc = (struct segment_command *) (header);
+		sc->cmd = LC_SEGMENT;
+		sc->cmdsize = sizeof(struct segment_command);
+		sc->segname[0] = 0;
+		sc->vmaddr = vmoffset;
+		sc->vmsize = size;
+		sc->fileoff = foffset;
+		sc->filesize = size;
+		sc->maxprot = maxprot;
+		sc->initprot = prot;
+		sc->nsects = 0;
+
+		if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(hoffset) , &hoffset)) < 0) { 
+			printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error);
+			error = panic_error;
+			goto out;
+		} 
+    
+		if ((panic_error = kdp_send_crashdump_data (KDP_DATA, NULL, sizeof(struct segment_command) , (caddr_t) sc)) < 0) {
+			printf ("kdp_send_crashdump_data failed with error %d\n", panic_error);
+			error = panic_error;
+			goto out;
+		}
+
+		/* Do not transmit memory tagged VM_MEMORY_IOKIT - instead,
+		 * seek past that region on the server - this creates a
+		 * hole in the file.
+		 */
+
+		if ((vbr.user_tag != VM_MEMORY_IOKIT)) {
+      
+			if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(foffset) , &foffset)) < 0) {
+				printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error);
+				error = panic_error;
+				goto out;
+			}
+
+			txstart = vmoffset;
+
+			if ((panic_error = kdp_send_crashdump_data (KDP_DATA, NULL, size, (caddr_t) txstart)) < 0)	{
+				printf ("kdp_send_crashdump_data failed with error %d\n", panic_error);
+				error = panic_error;
+				goto out;
+			}
+		}
+
+		hoffset += sizeof(struct segment_command);
+		foffset += size;
+		vmoffset += size;
+		segment_count--;
+	}
+	tir1.header = header;
+	tir1.hoffset = 0;
+	tir1.flavors = flavors;
+	tir1.tstate_size = tstate_size;
+
+	/* Now send out the LC_THREAD load command, with the thread information
+	 * for the current activation.
+	 * Note that the corefile can contain LC_SEGMENT commands with file
+	 * offsets that point past the edge of the corefile, in the event that
+	 * the last N VM regions were all I/O mapped or otherwise
+	 * non-transferable memory,  not followed by a normal VM region;
+	 * i.e. there will be no hole that reaches to the end of the core file.
+	 */
+	kern_collectth_state (current_thread(), &tir1);
+
+	if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(hoffset) , &hoffset)) < 0) { 
+		printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error);
+		error = panic_error;
+		goto out;
+	}
+  
+	if ((panic_error = kdp_send_crashdump_data (KDP_DATA, NULL, tir1.hoffset , (caddr_t) header)) < 0) {
+		printf ("kdp_send_crashdump_data failed with error %d\n", panic_error);
+		error = panic_error;
+		goto out;
+	}
+    
+	/* last packet */
+	if ((panic_error = kdp_send_crashdump_pkt (KDP_EOF, NULL, 0, ((void *) 0))) < 0)
+	{
+		printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error);
+		error = panic_error;
+		goto out;
+	}
+out:
+	return (error);
 }
diff --git a/osfmk/kdp/ml/ppc/kdp_machdep.c b/osfmk/kdp/ml/ppc/kdp_machdep.c
index 2c7e59f81..f8de33c2a 100644
--- a/osfmk/kdp/ml/ppc/kdp_machdep.c
+++ b/osfmk/kdp/ml/ppc/kdp_machdep.c
@@ -35,6 +35,12 @@
 #include <kern/debug.h>
 #include <IOKit/IOPlatformExpert.h>
 
+
+#include <kern/thread.h>
+#include <ppc/thread.h>
+#include <vm/vm_map.h>
+#include <ppc/pmap.h>
+
 #define KDP_TEST_HARNESS 0
 #if KDP_TEST_HARNESS
 #define dprintf(x) kprintf x
@@ -48,6 +54,21 @@ void kdp_trap( unsigned int, struct savearea	*saved_state);
 int kdp_getc(void);
 boolean_t kdp_call_kdb(void);
 
+extern pmap_t kdp_pmap;
+extern uint32_t kdp_src_high32;
+
+
+extern unsigned kdp_vm_read(caddr_t src, caddr_t dst, unsigned len);
+
+int
+machine_trace_thread(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p);
+
+int
+machine_trace_thread64(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p);
+
+unsigned
+machine_read64(addr64_t srcaddr, caddr_t dstaddr, uint32_t len);
+
 void
 kdp_exception(
     unsigned char	*pkt,
@@ -634,3 +655,151 @@ unsigned int kdp_ml_get_breakinsn(void)
 {
   return 0x7fe00008;
 }
+#define LR_OFFSET 8
+#define LR_OFFSET64 16
+
+int
+machine_trace_thread(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p)
+{
+	uint32_t *tracebuf = (uint32_t *)tracepos;
+	uint32_t fence = 0;
+	uint32_t stackptr = 0;
+	uint32_t stacklimit = 0xb0000000;
+	int framecount = 0;
+	uint32_t init_srr0 = 0;
+	uint32_t prevsp = 0;
+	uint32_t framesize = 2 * sizeof(vm_offset_t);
+	
+	if (user_p) {
+		/* Examine the user savearea */
+		init_srr0 = thread->machine.upcb->save_srr0;
+		stackptr = thread->machine.upcb->save_r1;
+		/* This bound isn't useful, but it doesn't hinder us */
+		stacklimit = 0xffffffff;
+		kdp_pmap = thread->task->map->pmap;
+	}
+	else {
+		stackptr = thread->machine.pcb->save_r1;
+		init_srr0 = thread->machine.pcb->save_srr0;
+	}
+	/* Fill in the "current" program counter */
+	*tracebuf++ = init_srr0;
+
+	for (framecount = 0; framecount < nframes; framecount++) {
+/* Bounds check */
+		if ((tracebound - ((uint32_t) tracebuf)) < (4 * framesize)) {
+			tracebuf--;
+			break;
+		}
+
+		*tracebuf++ = stackptr;
+/* Invalid frame, or hit fence */
+		if (!stackptr || (stackptr == fence)) {
+			break;
+		}
+/* Stack grows downward */		
+		if (stackptr < prevsp) {
+			break;
+		}
+/* Unaligned frame */		
+		if (stackptr & 0x000000F) {
+			break;
+		}
+		if (stackptr > stacklimit) {
+			break;
+		}
+/* Assume there's a saved link register, and read it */
+		if (kdp_vm_read((caddr_t) (stackptr + LR_OFFSET), (caddr_t) tracebuf, sizeof(caddr_t)) != sizeof(caddr_t)) {
+			break;
+		}
+
+		tracebuf++;
+		prevsp = stackptr;
+/* Next frame */
+		if (kdp_vm_read((caddr_t) stackptr, (caddr_t) &stackptr, sizeof(caddr_t)) != sizeof(caddr_t)) {
+			*tracebuf++ = 0;
+			break;
+		}
+	}
+/* Reset the target pmap */
+	kdp_pmap = 0;
+	return ((uint32_t) tracebuf - tracepos);
+}
+
+/* Routine to encapsulate the 64-bit address read hack*/
+unsigned
+machine_read64(addr64_t srcaddr, caddr_t dstaddr, uint32_t len)
+{
+	uint32_t kdp_vm_read_low32;
+	unsigned retval;
+	
+	kdp_src_high32 = srcaddr >> 32;
+	kdp_vm_read_low32 = srcaddr & 0x00000000FFFFFFFFUL;
+	retval = kdp_vm_read((caddr_t)kdp_vm_read_low32, dstaddr, len);
+	kdp_src_high32 = 0;
+	return retval;
+}
+
+int
+machine_trace_thread64(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p)
+{
+	uint64_t *tracebuf = (uint64_t *)tracepos;
+	uint32_t fence = 0;
+	addr64_t stackptr = 0;
+	uint64_t stacklimit = 0xb0000000;
+	int framecount = 0;
+	addr64_t init_srr0 = 0;
+	addr64_t prevsp = 0;
+	unsigned framesize = 2 * sizeof(addr64_t);
+	
+	if (user_p) {
+		init_srr0 = thread->machine.upcb->save_srr0;
+		stackptr = thread->machine.upcb->save_r1;
+		stacklimit = 0xffffffffffffffffULL;
+		kdp_pmap = thread->task->map->pmap;
+	}
+	else {
+		stackptr = thread->machine.pcb->save_r1;
+		init_srr0 = thread->machine.pcb->save_srr0;
+	}
+	
+	*tracebuf++ = init_srr0;
+
+	for (framecount = 0; framecount < nframes; framecount++) {
+
+		if ((tracebound - ((uint32_t) tracebuf)) < (4 * framesize)) {
+			tracebuf--;
+			break;
+		}
+
+		*tracebuf++ = stackptr;
+
+		if (!stackptr || (stackptr == fence)){
+			break;
+		}
+		if (stackptr < prevsp) {
+			break;
+		}
+		if (stackptr & 0x000000F) {
+			break;
+		}
+		if (stackptr > stacklimit) {
+			break;
+		}
+
+		if (machine_read64(stackptr+LR_OFFSET64, (caddr_t) tracebuf, sizeof(addr64_t)) != sizeof(addr64_t)) {
+			break;
+		}
+		tracebuf++;
+		
+		prevsp = stackptr;
+		if (machine_read64(stackptr, (caddr_t) &stackptr, sizeof(addr64_t)) != sizeof(addr64_t)) {
+			*tracebuf++ = 0;
+			break;
+		}
+	}
+
+	kdp_pmap = 0;
+
+	return ((uint32_t) tracebuf - tracepos);
+}
diff --git a/osfmk/kdp/ml/ppc/kdp_vm.c b/osfmk/kdp/ml/ppc/kdp_vm.c
index c907eeccc..bce212e4b 100644
--- a/osfmk/kdp/ml/ppc/kdp_vm.c
+++ b/osfmk/kdp/ml/ppc/kdp_vm.c
@@ -57,6 +57,7 @@
 pmap_t kdp_pmap=0;
 boolean_t kdp_trans_off=0;
 boolean_t kdp_read_io  =0;
+uint32_t kdp_src_high32 = 0;
 
 unsigned kdp_vm_read( caddr_t, caddr_t, unsigned);
 unsigned kdp_vm_write( caddr_t, caddr_t, unsigned);
@@ -66,42 +67,18 @@ extern int sectSizeTEXT, sectSizeDATA, sectSizeLINK, sectSizePRELINK;
 
 /* XXX prototypes which should be in a commmon header file */
 addr64_t	kdp_vtophys(pmap_t pmap, addr64_t va);
-int	kern_dump(void);
-int	kdp_dump_trap(int type, struct savearea *regs);
-/*
- * XXX the following prototype doesn't match the declaration because the
- * XXX actual declaration is wrong.
- */
-extern int	kdp_send_panic_packets(unsigned int request, char *corename,
-				unsigned int length, caddr_t txstart);
-
-
-
+int		kern_dump(void);
+int		kdp_dump_trap(int type, struct savearea *regs);
 
 typedef struct {
   int	flavor;			/* the number for this flavor */
   int	count;			/* count of ints in this flavor */
 } mythread_state_flavor_t;
 
-/* These will need to be uncommented and completed
- *if we support other architectures 
- */
-
-/*
-#if defined (__ppc__)
-*/
 static mythread_state_flavor_t thread_flavor_array[] = {
   {PPC_THREAD_STATE , PPC_THREAD_STATE_COUNT},
 };
-/*
-#elif defined (__i386__)
-mythread_state_flavor_t thread_flavor_array [] = { 
-  {i386_THREAD_STATE, i386_THREAD_STATE_COUNT},
-};
-#else
-#error architecture not supported
-#endif
-*/
+
 static int kdp_mynum_flavors = 1;
 static int MAX_TSTATE_FLAVORS = 1;
 
@@ -116,8 +93,6 @@ unsigned int not_in_kdp = 1; /* Cleared when we begin to access vm functions in
 
 char command_buffer[512];
 
-// XXX static struct vm_object test_object;
-
 /*
  *
  */
@@ -156,13 +131,11 @@ unsigned kdp_vm_read(
     kprintf("kdp_vm_read1: src %x dst %x len %x - %08X %08X\n", src, dst, len, ((unsigned long *)src)[0], ((unsigned long *)src)[1]);
 #endif
 
-	cur_virt_src = (addr64_t)((unsigned int)src);
+	cur_virt_src = (addr64_t)((unsigned int)src | (((uint64_t)kdp_src_high32) << 32));
 	cur_virt_dst = (addr64_t)((unsigned int)dst);
 	
 	if (kdp_trans_off) {
-		
-		
-		resid = len;								/* Get the length to copy */
+		resid = len;	/* Get the length to copy */
 
 		while (resid != 0) {
 
@@ -323,7 +296,7 @@ kdp_dump_trap(
 	      __unused struct savearea *regs)
 {
   printf ("An unexpected trap (type %d) occurred during the kernel dump, terminating.\n", type);
-  kdp_send_panic_pkt (KDP_EOF, NULL, 0, ((void *) 0));
+  kdp_send_crashdump_pkt (KDP_EOF, NULL, 0, ((void *) 0));
   abort_panic_transfer();
   kdp_flag &= ~KDP_PANIC_DUMP_ENABLED;
   kdp_flag &= ~PANIC_CORE_ON_NMI;
@@ -423,25 +396,25 @@ kern_dump(void)
    * to begin data transmission 
    */
 
-   if ((panic_error = kdp_send_panic_pkt (KDP_SEEK, NULL, sizeof(nfoffset) , &nfoffset)) < 0) { 
-     printf ("kdp_send_panic_pkt failed with error %d\n", panic_error); 
+   if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(nfoffset) , &nfoffset)) < 0) { 
+     printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error); 
      return -1; 
    } 
 
-   if ((panic_error = kdp_send_panic_packets (KDP_DATA, NULL, sizeof(struct mach_header), (caddr_t) mh) < 0)) {
-     printf ("kdp_send_panic_packets failed with error %d\n", panic_error);
+   if ((panic_error = kdp_send_crashdump_data (KDP_DATA, NULL, sizeof(struct mach_header), (caddr_t) mh) < 0)) {
+     printf ("kdp_send_crashdump_data failed with error %d\n", panic_error);
      return -1 ;
    }
 
-   if ((panic_error = kdp_send_panic_pkt (KDP_SEEK, NULL, sizeof(foffset) , &foffset) < 0)) {
-     printf ("kdp_send_panic_pkt failed with error %d\n", panic_error);
+   if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(foffset) , &foffset) < 0)) {
+     printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error);
      return (-1);
    }
   printf ("Transmitting kernel state, please wait: ");
 
   while ((segment_count > 0) || (kret == KERN_SUCCESS)){
     /* Check if we've transmitted all the kernel sections */
-    if (num_sects_txed == mach_section_count-1) {
+    if (num_sects_txed == mach_section_count) {
       
     while (1) {
 
@@ -525,13 +498,13 @@ kern_dump(void)
     sc->initprot = prot;
     sc->nsects = 0;
 
-    if ((panic_error = kdp_send_panic_pkt (KDP_SEEK, NULL, sizeof(hoffset) , &hoffset)) < 0) { 
-	printf ("kdp_send_panic_pkt failed with error %d\n", panic_error); 
+    if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(hoffset) , &hoffset)) < 0) { 
+	printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error); 
 	return -1; 
       } 
     
-    if ((panic_error = kdp_send_panic_packets (KDP_DATA, NULL, sizeof(struct segment_command) , (caddr_t) sc)) < 0) {
-	printf ("kdp_send_panic_packets failed with error %d\n", panic_error);
+    if ((panic_error = kdp_send_crashdump_data (KDP_DATA, NULL, sizeof(struct segment_command) , (caddr_t) sc)) < 0) {
+	printf ("kdp_send_crashdump_data failed with error %d\n", panic_error);
 	return -1 ;
       }
 
@@ -541,15 +514,15 @@ kern_dump(void)
 
     if ((vbr.user_tag != VM_MEMORY_IOKIT)) {
       
-      if ((panic_error = kdp_send_panic_pkt (KDP_SEEK, NULL, sizeof(foffset) , &foffset)) < 0) {
-	  printf ("kdp_send_panic_pkt failed with error %d\n", panic_error);
+      if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(foffset) , &foffset)) < 0) {
+	  printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error);
 	  return (-1);
 	}
 
       txstart = vmoffset;
 
-      if ((panic_error = kdp_send_panic_packets (KDP_DATA, NULL, size, (caddr_t) txstart)) < 0)	{
-	  printf ("kdp_send_panic_packets failed with error %d\n", panic_error);
+      if ((panic_error = kdp_send_crashdump_data (KDP_DATA, NULL, size, (caddr_t) txstart)) < 0)	{
+	  printf ("kdp_send_crashdump_data failed with error %d\n", panic_error);
 	  return -1 ;
 	}
     }
@@ -574,20 +547,20 @@ kern_dump(void)
    */
   kern_collectth_state (current_thread(), &tir1);
 
-  if ((panic_error = kdp_send_panic_pkt (KDP_SEEK, NULL, sizeof(hoffset) , &hoffset)) < 0) { 
-      printf ("kdp_send_panic_pkt failed with error %d\n", panic_error); 
+  if ((panic_error = kdp_send_crashdump_pkt(KDP_SEEK, NULL, sizeof(hoffset) , &hoffset)) < 0) { 
+      printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error); 
       return -1; 
     } 
   
-    if ((panic_error = kdp_send_panic_packets (KDP_DATA, NULL, tir1.hoffset , (caddr_t) header)) < 0) {
-	printf ("kdp_send_panic_packets failed with error %d\n", panic_error);
+    if ((panic_error = kdp_send_crashdump_data(KDP_DATA, NULL, tir1.hoffset , (caddr_t) header)) < 0) {
+	printf ("kdp_send_crashdump_data failed with error %d\n", panic_error);
 	return -1 ;
       }
     
     /* last packet */
-    if ((panic_error = kdp_send_panic_pkt (KDP_EOF, NULL, 0, ((void *) 0))) < 0)
+    if ((panic_error = kdp_send_crashdump_pkt(KDP_EOF, NULL, 0, ((void *) 0))) < 0)
       {
-	printf ("kdp_send_panic_pkt failed with error %d\n", panic_error);
+	printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error);
 	return (-1) ;
       }
     
diff --git a/osfmk/kern/Makefile b/osfmk/kern/Makefile
index ab810f2c2..383bac18c 100644
--- a/osfmk/kern/Makefile
+++ b/osfmk/kern/Makefile
@@ -15,6 +15,7 @@ EXPORT_ONLY_FILES = \
 	cpu_number.h \
 	cpu_data.h \
 	debug.h \
+	etimer.h \
 	ipc_mig.h \
 	kalloc.h \
 	kern_types.h \
@@ -23,6 +24,8 @@ EXPORT_ONLY_FILES = \
 	host.h \
 	mach_param.h \
 	macro_help.h \
+	pms.h \
+	page_decrypt.h \
 	processor.h \
 	queue.h \
 	sched_prim.h \
diff --git a/osfmk/kern/ast.c b/osfmk/kern/ast.c
index 614c77365..d8ac52a86 100644
--- a/osfmk/kern/ast.c
+++ b/osfmk/kern/ast.c
@@ -79,10 +79,7 @@
 #include <kern/spl.h>
 #include <kern/wait_queue.h>
 #include <mach/policy.h>
-
-#ifdef __ppc__
-#include <ppc/trap.h> // for CHUD AST hook
-#endif
+#include <machine/trap.h> // for CHUD AST hook
 
 void
 ast_init(void)
@@ -102,12 +99,11 @@ ast_taken(
 	ast_t			*myast = ast_pending();
 	thread_t		thread = current_thread();
 
-#ifdef __ppc__
 	/*
 	 * CHUD hook - all threads including idle processor threads
 	 */
 	if(perfASTHook) {
-		if(*myast & AST_PPC_CHUD_ALL) {
+		if(*myast & AST_CHUD_ALL) {
 			perfASTHook(0, NULL, 0, 0);
 			
 			if(*myast == AST_NONE) {
@@ -115,9 +111,8 @@ ast_taken(
 			}
 		}
 	} else {
-		*myast &= ~AST_PPC_CHUD_ALL;
+		*myast &= ~AST_CHUD_ALL;
 	}
-#endif
 
 	reasons &= *myast;
 	*myast &= ~reasons;
diff --git a/osfmk/kern/ast.h b/osfmk/kern/ast.h
index 1ceaf3f9c..6a689028c 100644
--- a/osfmk/kern/ast.h
+++ b/osfmk/kern/ast.h
@@ -113,6 +113,11 @@ typedef uint32_t		ast_t;
 
 #endif  /* MACHINE_AST */
 
+#define AST_CHUD_URGENT     0x800
+#define AST_CHUD            0x400
+
+#define AST_CHUD_ALL        (AST_CHUD_URGENT|AST_CHUD)
+
 /* Initialize module */
 extern void		ast_init(void);
 
diff --git a/osfmk/kern/bsd_kern.c b/osfmk/kern/bsd_kern.c
index 64f02fb12..1a7d3f2dd 100644
--- a/osfmk/kern/bsd_kern.c
+++ b/osfmk/kern/bsd_kern.c
@@ -37,6 +37,7 @@
 #include <ipc/ipc_port.h>
 #include <ipc/ipc_object.h>
 #include <vm/vm_map.h>
+#include <vm/vm_kern.h>
 #include <vm/pmap.h>
 #include <vm/vm_protos.h> /* last */
 
@@ -266,6 +267,9 @@ swap_task_map(task_t task,vm_map_t map)
 	old_map = task->map;
 	thread->map = task->map = map;
 	task_unlock(task);
+
+	inval_copy_windows(thread);
+
 	return old_map;
 }
 
@@ -523,3 +527,159 @@ astbsd_on(void)
 	ast_on_fast(AST_BSD);
 	(void)ml_set_interrupts_enabled(reenable);
 }
+
+
+#include <sys/bsdtask_info.h>
+
+void
+fill_taskprocinfo(task_t task, struct proc_taskinfo_internal * ptinfo)
+{
+	vm_map_t map;
+	task_absolutetime_info_data_t   tinfo;
+	thread_t thread;
+	int numrunning = 0;
+	
+	map = (task == kernel_task)? kernel_map: task->map;
+
+	ptinfo->pti_virtual_size  = map->size;
+	ptinfo->pti_resident_size  = (mach_vm_size_t)(pmap_resident_count(map->pmap)
+				   * PAGE_SIZE);
+
+	task_lock(task);
+
+	ptinfo->pti_policy = ((task != kernel_task)?
+                                          POLICY_TIMESHARE: POLICY_RR);
+
+	tinfo.threads_user = tinfo.threads_system = 0;
+	tinfo.total_user = task->total_user_time;
+	tinfo.total_system = task->total_system_time;
+
+	queue_iterate(&task->threads, thread, thread_t, task_threads) {
+		uint64_t    tval;
+
+		if ((thread->state & TH_RUN) == TH_RUN)
+			numrunning++;
+		tval = timer_grab(&thread->user_timer);
+		tinfo.threads_user += tval;
+		tinfo.total_user += tval;
+
+		tval = timer_grab(&thread->system_timer);
+		tinfo.threads_system += tval;
+		tinfo.total_system += tval;
+	}
+
+	ptinfo->pti_total_system = tinfo.total_system;
+	ptinfo->pti_total_user = tinfo.total_user;
+	ptinfo->pti_threads_system = tinfo.threads_system;
+	ptinfo->pti_threads_user = tinfo.threads_user;
+	
+	ptinfo->pti_faults = task->faults;
+	ptinfo->pti_pageins = task->pageins;
+	ptinfo->pti_cow_faults = task->cow_faults;
+	ptinfo->pti_messages_sent = task->messages_sent;
+	ptinfo->pti_messages_received = task->messages_received;
+	ptinfo->pti_syscalls_mach = task->syscalls_mach;
+	ptinfo->pti_syscalls_unix = task->syscalls_unix;
+	ptinfo->pti_csw = task->csw;
+	ptinfo->pti_threadnum = task->thread_count;
+	ptinfo->pti_numrunning = numrunning;
+	ptinfo->pti_priority = task->priority;
+
+	task_unlock(task);
+}
+
+int 
+fill_taskthreadinfo(task_t task, uint64_t thaddr, struct proc_threadinfo_internal * ptinfo)
+{
+	thread_t  thact;
+	int err=0, count;
+	thread_basic_info_data_t basic_info;
+	kern_return_t kret;
+
+	task_lock(task);
+
+	for (thact  = (thread_t)queue_first(&task->threads);
+			!queue_end(&task->threads, (queue_entry_t)thact); ) {
+#if defined(__ppc__)
+		if (thact->machine.cthread_self == thaddr)
+#elif defined (__i386__)
+		if (thact->machine.pcb->cthread_self == thaddr)
+#else
+#error architecture not supported
+#endif
+		{
+		
+			count = THREAD_BASIC_INFO_COUNT;
+			if ((kret = thread_info_internal(thact, THREAD_BASIC_INFO, &basic_info, &count)) != KERN_SUCCESS) {
+				err = 1;
+				goto out;	
+			}
+#if 0
+			ptinfo->pth_user_time = timer_grab(&basic_info.user_time);
+			ptinfo->pth_system_time = timer_grab(&basic_info.system_time);
+#else
+			ptinfo->pth_user_time = ((basic_info.user_time.seconds * NSEC_PER_SEC) + (basic_info.user_time.microseconds * NSEC_PER_USEC));
+			ptinfo->pth_system_time = ((basic_info.system_time.seconds * NSEC_PER_SEC) + (basic_info.system_time.microseconds * NSEC_PER_USEC));
+
+#endif
+			ptinfo->pth_cpu_usage = basic_info.cpu_usage;
+			ptinfo->pth_policy = basic_info.policy;
+			ptinfo->pth_run_state = basic_info.run_state;
+			ptinfo->pth_flags = basic_info.flags;
+			ptinfo->pth_sleep_time = basic_info.sleep_time;
+			ptinfo->pth_curpri = thact->sched_pri;
+			ptinfo->pth_priority = thact->priority;
+			ptinfo->pth_maxpriority = thact->max_priority;
+			
+			err = 0;
+			goto out; 
+		}
+		thact = (thread_t)queue_next(&thact->task_threads);
+	}
+	err = 1;
+
+out:
+	task_unlock(task);
+	return(err);
+}
+
+int
+fill_taskthreadlist(task_t task, void * buffer, int thcount)
+{
+	int numthr=0;
+	thread_t thact;
+	uint64_t * uptr;
+	uint64_t  thaddr;
+
+	uptr = (uint64_t *)buffer;
+
+	task_lock(task);
+
+	for (thact  = (thread_t)queue_first(&task->threads);
+			!queue_end(&task->threads, (queue_entry_t)thact); ) {
+#if defined(__ppc__)
+		thaddr = thact->machine.cthread_self;
+#elif defined (__i386__)
+		thaddr = thact->machine.pcb->cthread_self;
+#else
+#error architecture not supported
+#endif
+		*uptr++ = thaddr;
+		numthr++;
+		if (numthr >= thcount)
+			goto out;
+		thact = (thread_t)queue_next(&thact->task_threads);
+	}
+
+out:
+	task_unlock(task);
+	return(numthr * sizeof(uint64_t));
+	
+}
+
+int
+get_numthreads(task_t task)
+{
+	return(task->thread_count);
+}
+
diff --git a/osfmk/kern/clock.c b/osfmk/kern/clock.c
index a0b3f9077..b9d0a075b 100644
--- a/osfmk/kern/clock.c
+++ b/osfmk/kern/clock.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -29,109 +29,89 @@
  * @OSF_COPYRIGHT@
  */
 /*
- *	File:		kern/clock.c
- *	Purpose:	Routines for the creation and use of kernel
- *			alarm clock services. This file and the ipc
- *			routines in kern/ipc_clock.c constitute the
- *			machine-independent clock service layer.
  */
 
-#include <mach_host.h>
-
 #include <mach/mach_types.h>
-#include <mach/boolean.h>
-#include <mach/processor_info.h>
-#include <mach/vm_param.h>
 
-#include <kern/cpu_number.h>
-#include <kern/misc_protos.h>
 #include <kern/lock.h>
-#include <kern/host.h>
 #include <kern/spl.h>
 #include <kern/sched_prim.h>
 #include <kern/thread.h>
-#include <kern/ipc_host.h>
 #include <kern/clock.h>
-#include <kern/zalloc.h>
+#include <kern/host_notify.h>
+
+#include <IOKit/IOPlatformExpert.h>
 
-#include <ipc/ipc_types.h>
-#include <ipc/ipc_port.h>
+#include <machine/commpage.h>
 
 #include <mach/mach_traps.h>
-#include <mach/clock_reply.h>
 #include <mach/mach_time.h>
 
-#include <mach/clock_server.h>
-#include <mach/clock_priv_server.h>
-#include <mach/host_priv_server.h>
+decl_simple_lock_data(static,clock_lock)
 
 /*
- * Exported interface
+ *	Time of day (calendar) variables.
+ *
+ *	Algorithm:
+ *
+ *	TOD <- (seconds + epoch, fraction) <- CONV(current absolute time + offset)
+ *
+ *	where CONV converts absolute time units into seconds and a fraction.
  */
+static struct clock_calend {
+	uint64_t			epoch;
+	uint64_t			offset;
+}					clock_calend;
 
-#include <mach/clock_server.h>
-#include <mach/mach_host_server.h>
+/*
+ *	Calendar adjustment variables and values.
+ */
+#define calend_adjperiod	(NSEC_PER_SEC / 100)	/* adjustment period, ns */
+#define calend_adjskew		(40 * NSEC_PER_USEC)	/* "standard" skew, ns / period */
+#define	calend_adjbig		(NSEC_PER_SEC)			/* use 10x skew above adjbig ns */
+
+static uint64_t			calend_adjstart;		/* Absolute time value for start of this adjustment period */
+static uint32_t			calend_adjoffset;		/* Absolute time offset for this adjustment period as absolute value */
 
-/* local data declarations */
-decl_simple_lock_data(static,ClockLock)		/* clock system synchronization */
-static struct	zone		*alarm_zone;	/* zone for user alarms */
-static struct	alarm		*alrmfree;		/* alarm free list pointer */
-static struct	alarm		*alrmdone;		/* alarm done list pointer */
-static long					alrm_seqno;		/* uniquely identifies alarms */
-static thread_call_data_t	alarm_deliver;
+static int32_t			calend_adjdelta;		/* Nanosecond time delta for this adjustment period */
+static int64_t			calend_adjtotal;		/* Nanosecond remaining total adjustment */
 
-decl_simple_lock_data(static,calend_adjlock)
+static uint64_t			calend_adjdeadline;		/* Absolute time value for next adjustment period */
+static uint32_t			calend_adjinterval;		/* Absolute time interval of adjustment period */
 
 static timer_call_data_t	calend_adjcall;
-static uint64_t				calend_adjdeadline;
+static uint32_t				calend_adjactive;
+
+static uint32_t		calend_set_adjustment(
+						int32_t			*secs,
+						int32_t			*microsecs);
+
+static void			calend_adjust_call(void);
+static uint32_t		calend_adjust(void);
 
 static thread_call_data_t	calend_wakecall;
 
-/* external declarations */
-extern	struct clock	clock_list[];
-extern	int		clock_count;
-
-/* local clock subroutines */
-static
-void	flush_alarms(
-			clock_t			clock);
-
-static
-void	post_alarm(
-			clock_t			clock,
-			alarm_t			alarm);
-
-static
-int		check_time(
-			alarm_type_t	alarm_type,
-			mach_timespec_t	*alarm_time,
-			mach_timespec_t	*clock_time);
-
-static
-void	clock_alarm_deliver(
-			thread_call_param_t		p0,
-			thread_call_param_t		p1);
-
-static
-void	calend_adjust_call(
-			timer_call_param_t	p0,
-			timer_call_param_t	p1);
-
-static
-void	calend_dowakeup(
-			thread_call_param_t		p0,
-			thread_call_param_t		p1);
+extern	void	IOKitResetTime(void);
 
-/*
- *	Macros to lock/unlock clock system.
- */
-#define LOCK_CLOCK(s)			\
-	s = splclock();			\
-	simple_lock(&ClockLock);
+static uint64_t		clock_boottime;				/* Seconds boottime epoch */
 
-#define UNLOCK_CLOCK(s)			\
-	simple_unlock(&ClockLock);	\
-	splx(s);
+#define TIME_ADD(rsecs, secs, rfrac, frac, unit)	\
+MACRO_BEGIN											\
+	if (((rfrac) += (frac)) >= (unit)) {			\
+		(rfrac) -= (unit);							\
+		(rsecs) += 1;								\
+	}												\
+	(rsecs) += (secs);								\
+MACRO_END
+
+#define TIME_SUB(rsecs, secs, rfrac, frac, unit)	\
+MACRO_BEGIN											\
+	if ((int32_t)((rfrac) -= (frac)) < 0) {			\
+		(rfrac) += (unit);							\
+		(rsecs) -= 1;								\
+	}												\
+	(rsecs) -= (secs);								\
+MACRO_END
 
 /*
  *	clock_config:
@@ -141,37 +121,17 @@ void	calend_dowakeup(
 void
 clock_config(void)
 {
-	clock_t			clock;
-	register int 	i;
-
-	assert(cpu_number() == master_cpu);
-
-	simple_lock_init(&ClockLock, 0);
-	thread_call_setup(&alarm_deliver, clock_alarm_deliver, NULL);
+	simple_lock_init(&clock_lock, 0);
 
-	simple_lock_init(&calend_adjlock, 0);
-	timer_call_setup(&calend_adjcall, calend_adjust_call, NULL);
+	timer_call_setup(&calend_adjcall, (timer_call_func_t)calend_adjust_call, NULL);
+	thread_call_setup(&calend_wakecall, (thread_call_func_t)IOKitResetTime, NULL);
 
-	thread_call_setup(&calend_wakecall, calend_dowakeup, NULL);
-
-	/*
-	 * Configure clock devices.
-	 */
-	for (i = 0; i < clock_count; i++) {
-		clock = &clock_list[i];
-		if (clock->cl_ops) {
-			if ((*clock->cl_ops->c_config)() == 0)
-				clock->cl_ops = 0;
-		}
-	}
+	clock_oldconfig();
 
 	/*
 	 * Initialize the timer callouts.
 	 */
 	timer_call_initialize();
-
-	/* start alarm sequence numbers at 0 */
-	alrm_seqno = 0;
 }
 
 /*
@@ -182,696 +142,506 @@ clock_config(void)
 void
 clock_init(void)
 {
-	clock_t			clock;
-	register int	i;
-
-	/*
-	 * Initialize basic clock structures.
-	 */
-	for (i = 0; i < clock_count; i++) {
-		clock = &clock_list[i];
-		if (clock->cl_ops && clock->cl_ops->c_init)
-			(*clock->cl_ops->c_init)();
-	}
+	clock_oldinit();
 }
 
 /*
- * Called by machine dependent code
- * to initialize areas dependent on the
- * timebase value.  May be called multiple
- * times during start up.
+ *	clock_timebase_init:
+ *
+ *	Called by machine dependent code
+ *	to initialize areas dependent on the
+ *	timebase value.  May be called multiple
+ *	times during start up.
  */
 void
 clock_timebase_init(void)
 {
-	sched_timebase_init();
-}
-
-/*
- * Initialize the clock ipc service facility.
- */
-void
-clock_service_create(void)
-{
-	clock_t			clock;
-	register int	i;
+	uint64_t	abstime;
 
-	/*
-	 * Initialize ipc clock services.
-	 */
-	for (i = 0; i < clock_count; i++) {
-		clock = &clock_list[i];
-		if (clock->cl_ops) {
-			ipc_clock_init(clock);
-			ipc_clock_enable(clock);
-		}
-	}
+	nanoseconds_to_absolutetime(calend_adjperiod, &abstime);
+	calend_adjinterval = abstime;
 
-	/*
-	 * Perform miscellaneous late
-	 * initialization.
-	 */
-	i = sizeof(struct alarm);
-	alarm_zone = zinit(i, (4096/i)*i, 10*i, "alarms");
+	sched_timebase_init();
 }
 
 /*
- * Get the service port on a clock.
+ *	mach_timebase_info_trap:
+ *
+ *	User trap returns timebase constant.
  */
 kern_return_t
-host_get_clock_service(
-	host_t			host,
-	clock_id_t		clock_id,
-	clock_t			*clock)		/* OUT */
+mach_timebase_info_trap(
+	struct mach_timebase_info_trap_args *args)
 {
-	if (host == HOST_NULL || clock_id < 0 || clock_id >= clock_count) {
-		*clock = CLOCK_NULL;
-		return (KERN_INVALID_ARGUMENT);
-	}
+	mach_vm_address_t 			out_info_addr = args->info;
+	mach_timebase_info_data_t	info;
 
-	*clock = &clock_list[clock_id];
-	if ((*clock)->cl_ops == 0)
-		return (KERN_FAILURE);
-	return (KERN_SUCCESS);
-}
+	clock_timebase_info(&info);
 
-/*
- * Get the control port on a clock.
- */
-kern_return_t
-host_get_clock_control(
-	host_priv_t		host_priv,
-	clock_id_t		clock_id,
-	clock_t			*clock)		/* OUT */
-{
-	if (host_priv == HOST_PRIV_NULL || clock_id < 0 || clock_id >= clock_count) {
-		*clock = CLOCK_NULL;
-		return (KERN_INVALID_ARGUMENT);
-	}
+	copyout((void *)&info, out_info_addr, sizeof (info));
 
-	*clock = &clock_list[clock_id];
-	if ((*clock)->cl_ops == 0)
-		return (KERN_FAILURE);
 	return (KERN_SUCCESS);
 }
 
 /*
- * Get the current clock time.
+ *	Calendar routines.
  */
-kern_return_t
-clock_get_time(
-	clock_t			clock,
-	mach_timespec_t	*cur_time)	/* OUT */
-{
-	if (clock == CLOCK_NULL)
-		return (KERN_INVALID_ARGUMENT);
-	return ((*clock->cl_ops->c_gettime)(cur_time));
-}
 
 /*
- * Get clock attributes.
+ *	clock_get_calendar_microtime:
+ *
+ *	Returns the current calendar value,
+ *	microseconds as the fraction.
  */
-kern_return_t
-clock_get_attributes(
-	clock_t					clock,
-	clock_flavor_t			flavor,
-	clock_attr_t			attr,		/* OUT */
-	mach_msg_type_number_t	*count)		/* IN/OUT */
+void
+clock_get_calendar_microtime(
+	uint32_t			*secs,
+	uint32_t			*microsecs)
 {
-	if (clock == CLOCK_NULL)
-		return (KERN_INVALID_ARGUMENT);
-	if (clock->cl_ops->c_getattr)
-		return(clock->cl_ops->c_getattr(flavor, attr, count));
-	else
-		return (KERN_FAILURE);
-}
+	uint64_t		now;
+	spl_t			s;
 
-/*
- * Set the current clock time.
- */
-kern_return_t
-clock_set_time(
-	clock_t			clock,
-	mach_timespec_t	new_time)
-{
-	mach_timespec_t	*clock_time;
+	s = splclock();
+	simple_lock(&clock_lock);
 
-	if (clock == CLOCK_NULL)
-		return (KERN_INVALID_ARGUMENT);
-	if (clock->cl_ops->c_settime == NULL)
-		return (KERN_FAILURE);
-	clock_time = &new_time;
-	if (BAD_MACH_TIMESPEC(clock_time))
-		return (KERN_INVALID_VALUE);
+	now = mach_absolute_time();
 
-	/*
-	 * Flush all outstanding alarms.
-	 */
-	flush_alarms(clock);
+	if (calend_adjdelta < 0) {
+		uint32_t	t32;
 
-	/*
-	 * Set the new time.
-	 */
-	return (clock->cl_ops->c_settime(clock_time));
+		if (now > calend_adjstart) {
+			t32 = now - calend_adjstart;
+
+			if (t32 > calend_adjoffset)
+				now -= calend_adjoffset;
+			else
+				now = calend_adjstart;
+		}
+	}
+
+	now += clock_calend.offset;
+
+	absolutetime_to_microtime(now, secs, microsecs);
+
+	*secs += clock_calend.epoch;
+
+	simple_unlock(&clock_lock);
+	splx(s);
 }
 
 /*
- * Set the clock alarm resolution.
+ *	clock_get_calendar_nanotime:
+ *
+ *	Returns the current calendar value,
+ *	nanoseconds as the fraction.
+ *
+ *	Since we do not have an interface to
+ *	set the calendar with resolution greater
+ *	than a microsecond, we honor that here.
  */
-kern_return_t
-clock_set_attributes(
-	clock_t					clock,
-	clock_flavor_t			flavor,
-	clock_attr_t			attr,
-	mach_msg_type_number_t	count)
+void
+clock_get_calendar_nanotime(
+	uint32_t			*secs,
+	uint32_t			*nanosecs)
 {
-	if (clock == CLOCK_NULL)
-		return (KERN_INVALID_ARGUMENT);
-	if (clock->cl_ops->c_setattr)
-		return (clock->cl_ops->c_setattr(flavor, attr, count));
-	else
-		return (KERN_FAILURE);
+	uint64_t		now;
+	spl_t			s;
+
+	s = splclock();
+	simple_lock(&clock_lock);
+
+	now = mach_absolute_time();
+
+	if (calend_adjdelta < 0) {
+		uint32_t	t32;
+
+		if (now > calend_adjstart) {
+			t32 = now - calend_adjstart;
+
+			if (t32 > calend_adjoffset)
+				now -= calend_adjoffset;
+			else
+				now = calend_adjstart;
+		}
+	}
+
+	now += clock_calend.offset;
+
+	absolutetime_to_microtime(now, secs, nanosecs);
+	*nanosecs *= NSEC_PER_USEC;
+
+	*secs += clock_calend.epoch;
+
+	simple_unlock(&clock_lock);
+	splx(s);
 }
 
 /*
- * Setup a clock alarm.
+ *	clock_gettimeofday:
+ *
+ *	Kernel interface for commpage implementation of
+ *	gettimeofday() syscall.
+ *
+ *	Returns the current calendar value, and updates the
+ *	commpage info as appropriate.  Because most calls to
+ *	gettimeofday() are handled in user mode by the commpage,
+ *	this routine should be used infrequently.
  */
-kern_return_t
-clock_alarm(
-	clock_t					clock,
-	alarm_type_t			alarm_type,
-	mach_timespec_t			alarm_time,
-	ipc_port_t				alarm_port,
-	mach_msg_type_name_t	alarm_port_type)
+void
+clock_gettimeofday(
+	uint32_t			*secs,
+	uint32_t			*microsecs)
 {
-	alarm_t					alarm;
-	mach_timespec_t			clock_time;
-	int						chkstat;
-	kern_return_t			reply_code;
-	spl_t					s;
-
-	if (clock == CLOCK_NULL)
-		return (KERN_INVALID_ARGUMENT);
-	if (clock->cl_ops->c_setalrm == 0)
-		return (KERN_FAILURE);
-	if (IP_VALID(alarm_port) == 0)
-		return (KERN_INVALID_CAPABILITY);
+	uint64_t		now;
+	spl_t			s;
 
-	/*
-	 * Check alarm parameters. If parameters are invalid,
-	 * send alarm message immediately.
-	 */
-	(*clock->cl_ops->c_gettime)(&clock_time);
-	chkstat = check_time(alarm_type, &alarm_time, &clock_time);
-	if (chkstat <= 0) {
-		reply_code = (chkstat < 0 ? KERN_INVALID_VALUE : KERN_SUCCESS);
-		clock_alarm_reply(alarm_port, alarm_port_type,
-				  reply_code, alarm_type, clock_time);
-		return (KERN_SUCCESS);
+	s = splclock();
+	simple_lock(&clock_lock);
+
+	now = mach_absolute_time();
+
+	if (calend_adjdelta >= 0) {
+		clock_gettimeofday_set_commpage(now, clock_calend.epoch, clock_calend.offset, secs, microsecs);
 	}
+	else {
+		uint32_t	t32;
 
-	/*
-	 * Get alarm and add to clock alarm list.
-	 */
+		if (now > calend_adjstart) {
+			t32 = now - calend_adjstart;
+
+			if (t32 > calend_adjoffset)
+				now -= calend_adjoffset;
+			else
+				now = calend_adjstart;
+		}
+
+		now += clock_calend.offset;
 
-	LOCK_CLOCK(s);
-	if ((alarm = alrmfree) == 0) {
-		UNLOCK_CLOCK(s);
-		alarm = (alarm_t) zalloc(alarm_zone);
-		if (alarm == 0)
-			return (KERN_RESOURCE_SHORTAGE);
-		LOCK_CLOCK(s);
+		absolutetime_to_microtime(now, secs, microsecs);
+
+		*secs += clock_calend.epoch;
 	}
-	else
-		alrmfree = alarm->al_next;
-
-	alarm->al_status = ALARM_CLOCK;
-	alarm->al_time = alarm_time;
-	alarm->al_type = alarm_type;
-	alarm->al_port = alarm_port;
-	alarm->al_port_type = alarm_port_type;
-	alarm->al_clock = clock;
-	alarm->al_seqno = alrm_seqno++;
-	post_alarm(clock, alarm);
-	UNLOCK_CLOCK(s);
 
-	return (KERN_SUCCESS);
+	simple_unlock(&clock_lock);
+	splx(s);
 }
 
 /*
- * Sleep on a clock. System trap. User-level libmach clock_sleep
- * interface call takes a mach_timespec_t sleep_time argument which it
- * converts to sleep_sec and sleep_nsec arguments which are then
- * passed to clock_sleep_trap.
+ *	clock_set_calendar_microtime:
+ *
+ *	Sets the current calendar value by
+ *	recalculating the epoch and offset
+ *	from the system clock.
+ *
+ *	Also adjusts the boottime to keep the
+ *	value consistent, writes the new
+ *	calendar value to the platform clock,
+ *	and sends calendar change notifications.
  */
-kern_return_t
-clock_sleep_trap(
-	struct clock_sleep_trap_args *args)
+void
+clock_set_calendar_microtime(
+	uint32_t			secs,
+	uint32_t			microsecs)
 {
-	mach_port_name_t	clock_name = args->clock_name;
-	sleep_type_t		sleep_type = args->sleep_type;
-	int					sleep_sec = args->sleep_sec;
-	int					sleep_nsec = args->sleep_nsec;
-	mach_vm_address_t	wakeup_time_addr = args->wakeup_time;  
-	clock_t				clock;
-	mach_timespec_t		swtime;
-	kern_return_t		rvalue;
+	uint32_t		sys, microsys;
+	uint32_t		newsecs;
+	spl_t			s;
 
-	/*
-	 * Convert the trap parameters.
-	 */
-	if (clock_name != MACH_PORT_NULL)
-		clock = port_name_to_clock(clock_name);
-	else
-		clock = &clock_list[SYSTEM_CLOCK];
+	newsecs = (microsecs < 500*USEC_PER_SEC)?
+						secs: secs + 1;
+
+	s = splclock();
+	simple_lock(&clock_lock);
 
-	swtime.tv_sec  = sleep_sec;
-	swtime.tv_nsec = sleep_nsec;
+    commpage_set_timestamp(0,0,0);
 
 	/*
-	 * Call the actual clock_sleep routine.
+	 *	Calculate the new calendar epoch based on
+	 *	the new value and the system clock.
 	 */
-	rvalue = clock_sleep_internal(clock, sleep_type, &swtime);
+	clock_get_system_microtime(&sys, &microsys);
+	TIME_SUB(secs, sys, microsecs, microsys, USEC_PER_SEC);
 
 	/*
-	 * Return current time as wakeup time.
+	 *	Adjust the boottime based on the delta.
 	 */
-	if (rvalue != KERN_INVALID_ARGUMENT && rvalue != KERN_FAILURE) {
-		copyout((char *)&swtime, wakeup_time_addr, sizeof(mach_timespec_t));
-	}
-	return (rvalue);
-}	
-
-/*
- * Kernel internally callable clock sleep routine. The calling
- * thread is suspended until the requested sleep time is reached.
- */
-kern_return_t
-clock_sleep_internal(
-	clock_t				clock,
-	sleep_type_t		sleep_type,
-	mach_timespec_t		*sleep_time)
-{
-	alarm_t				alarm;
-	mach_timespec_t		clock_time;
-	kern_return_t		rvalue;
-	int					chkstat;
-	spl_t				s;
-
-	if (clock == CLOCK_NULL)
-		return (KERN_INVALID_ARGUMENT);
-	if (clock->cl_ops->c_setalrm == 0)
-		return (KERN_FAILURE);
+	clock_boottime += secs - clock_calend.epoch;
 
 	/*
-	 * Check sleep parameters. If parameters are invalid
-	 * return an error, otherwise post alarm request.
+	 *	Set the new calendar epoch.
 	 */
-	(*clock->cl_ops->c_gettime)(&clock_time);
+	clock_calend.epoch = secs;
+	nanoseconds_to_absolutetime((uint64_t)microsecs * NSEC_PER_USEC, &clock_calend.offset);
 
-	chkstat = check_time(sleep_type, sleep_time, &clock_time);
-	if (chkstat < 0)
-		return (KERN_INVALID_VALUE);
-	rvalue = KERN_SUCCESS;
-	if (chkstat > 0) {
-		wait_result_t wait_result;
+	/*
+	 *	Cancel any adjustment in progress.
+	 */
+	calend_adjdelta = calend_adjtotal = 0;
 
-		/*
-		 * Get alarm and add to clock alarm list.
-		 */
+	simple_unlock(&clock_lock);
 
-		LOCK_CLOCK(s);
-		if ((alarm = alrmfree) == 0) {
-			UNLOCK_CLOCK(s);
-			alarm = (alarm_t) zalloc(alarm_zone);
-			if (alarm == 0)
-				return (KERN_RESOURCE_SHORTAGE);
-			LOCK_CLOCK(s);
-		}
-		else
-			alrmfree = alarm->al_next;
+	/*
+	 *	Set the new value for the platform clock.
+	 */
+	PESetGMTTimeOfDay(newsecs);
 
-		/*
-		 * Wait for alarm to occur.
-		 */
-		wait_result = assert_wait((event_t)alarm, THREAD_ABORTSAFE);
-		if (wait_result == THREAD_WAITING) {
-			alarm->al_time = *sleep_time;
-			alarm->al_status = ALARM_SLEEP;
-			post_alarm(clock, alarm);
-			UNLOCK_CLOCK(s);
-
-			wait_result = thread_block(THREAD_CONTINUE_NULL);
-
-			/*
-			 * Note if alarm expired normally or whether it
-			 * was aborted. If aborted, delete alarm from
-			 * clock alarm list. Return alarm to free list.
-			 */
-			LOCK_CLOCK(s);
-			if (alarm->al_status != ALARM_DONE) {
-				assert(wait_result != THREAD_AWAKENED);
-				if (((alarm->al_prev)->al_next = alarm->al_next) != NULL)
-					(alarm->al_next)->al_prev = alarm->al_prev;
-				rvalue = KERN_ABORTED;
-			}
-			*sleep_time = alarm->al_time;
-			alarm->al_status = ALARM_FREE;
-		} else {
-			assert(wait_result == THREAD_INTERRUPTED);
-			assert(alarm->al_status == ALARM_FREE);
-			rvalue = KERN_ABORTED;
-		}
-		alarm->al_next = alrmfree;
-		alrmfree = alarm;
-		UNLOCK_CLOCK(s);
-	}
-	else
-		*sleep_time = clock_time;
+	splx(s);
 
-	return (rvalue);
+	/*
+	 *	Send host notifications.
+	 */
+	host_notify_calendar_change();
 }
 
 /*
- * CLOCK INTERRUPT SERVICE ROUTINES.
- */
-
-/*
- * Service clock alarm interrupts. Called from machine dependent
- * layer at splclock(). The clock_id argument specifies the clock,
- * and the clock_time argument gives that clock's current time.
+ *	clock_initialize_calendar:
+ *
+ *	Set the calendar and related clocks
+ *	from the platform clock at boot or
+ *	wake event.
+ *
+ *	Also sends host notifications.
  */
 void
-clock_alarm_intr(
-	clock_id_t			clock_id,
-	mach_timespec_t		*clock_time)
+clock_initialize_calendar(void)
 {
-	clock_t				clock;
-	register alarm_t	alrm1;
-	register alarm_t	alrm2;
-	mach_timespec_t		*alarm_time;
-	spl_t				s;
+	uint32_t		sys, microsys;
+	uint32_t		microsecs = 0, secs = PEGetGMTTimeOfDay();
+	spl_t			s;
 
-	clock = &clock_list[clock_id];
+	s = splclock();
+	simple_lock(&clock_lock);
 
-	/*
-	 * Update clock alarm list. All alarms that are due are moved
-	 * to the alarmdone list to be serviced by the alarm_thread.
-	 */
+    commpage_set_timestamp(0,0,0);
 
-	LOCK_CLOCK(s);
-	alrm1 = (alarm_t) &clock->cl_alarm;
-	while ((alrm2 = alrm1->al_next) != NULL) {
-		alarm_time = &alrm2->al_time;
-		if (CMP_MACH_TIMESPEC(alarm_time, clock_time) > 0)
-			break;
+	if ((int32_t)secs >= (int32_t)clock_boottime) {
+		/*
+		 *	Initialize the boot time based on the platform clock.
+		 */
+		if (clock_boottime == 0)
+			clock_boottime = secs;
 
 		/*
-		 * Alarm has expired, so remove it from the
-		 * clock alarm list.
-		 */  
-		if ((alrm1->al_next = alrm2->al_next) != NULL)
-			(alrm1->al_next)->al_prev = alrm1;
+		 *	Calculate the new calendar epoch based on
+		 *	the platform clock and the system clock.
+		 */
+		clock_get_system_microtime(&sys, &microsys);
+		TIME_SUB(secs, sys, microsecs, microsys, USEC_PER_SEC);
 
 		/*
-		 * If a clock_sleep() alarm, wakeup the thread
-		 * which issued the clock_sleep() call.
+		 *	Set the new calendar epoch.
 		 */
-		if (alrm2->al_status == ALARM_SLEEP) {
-			alrm2->al_next = 0;
-			alrm2->al_status = ALARM_DONE;
-			alrm2->al_time = *clock_time;
-			thread_wakeup((event_t)alrm2);
-		}
+		clock_calend.epoch = secs;
+		nanoseconds_to_absolutetime((uint64_t)microsecs * NSEC_PER_USEC, &clock_calend.offset);
 
- 		/*
-		 * If a clock_alarm() alarm, place the alarm on
-		 * the alarm done list and schedule the alarm
-		 * delivery mechanism.
+		/*
+		 *	 Cancel any adjustment in progress.
 		 */
-		else {
-			assert(alrm2->al_status == ALARM_CLOCK);
-			if ((alrm2->al_next = alrmdone) != NULL)
-				alrmdone->al_prev = alrm2;
-			else
-				thread_call_enter(&alarm_deliver);
-			alrm2->al_prev = (alarm_t) &alrmdone;
-			alrmdone = alrm2;
-			alrm2->al_status = ALARM_DONE;
-			alrm2->al_time = *clock_time;
-		}
+		calend_adjdelta = calend_adjtotal = 0;
 	}
 
+	simple_unlock(&clock_lock);
+	splx(s);
+
 	/*
-	 * Setup the clock dependent layer to deliver another
-	 * interrupt for the next pending alarm.
+	 *	Send host notifications.
 	 */
-	if (alrm2)
-		(*clock->cl_ops->c_setalrm)(alarm_time);
-	UNLOCK_CLOCK(s);
+	host_notify_calendar_change();
 }
 
 /*
- * ALARM DELIVERY ROUTINES.
+ *	clock_get_boottime_nanotime:
+ *
+ *	Return the boottime, used by sysctl.
  */
-
-static void
-clock_alarm_deliver(
-	__unused thread_call_param_t		p0,
-	__unused thread_call_param_t		p1)
+void
+clock_get_boottime_nanotime(
+	uint32_t			*secs,
+	uint32_t			*nanosecs)
 {
-	register alarm_t	alrm;
-	kern_return_t		code;
-	spl_t				s;
-
-	LOCK_CLOCK(s);
-	while ((alrm = alrmdone) != NULL) {
-		if ((alrmdone = alrm->al_next) != NULL)
-			alrmdone->al_prev = (alarm_t) &alrmdone;
-		UNLOCK_CLOCK(s);
-
-		code = (alrm->al_status == ALARM_DONE? KERN_SUCCESS: KERN_ABORTED);
-		if (alrm->al_port != IP_NULL) {
-			/* Deliver message to designated port */
-			if (IP_VALID(alrm->al_port)) {
-				clock_alarm_reply(alrm->al_port, alrm->al_port_type, code,
-								  				alrm->al_type, alrm->al_time);
-			}
-
-			LOCK_CLOCK(s);
-			alrm->al_status = ALARM_FREE;
-			alrm->al_next = alrmfree;
-			alrmfree = alrm;
-		}
-		else
-			panic("clock_alarm_deliver");
-	}
-
-	UNLOCK_CLOCK(s);
+	*secs = clock_boottime;
+	*nanosecs = 0;
 }
 
 /*
- * CLOCK PRIVATE SERVICING SUBROUTINES.
- */
-
-/*
- * Flush all pending alarms on a clock. All alarms
- * are activated and timestamped correctly, so any
- * programs waiting on alarms/threads will proceed
- * with accurate information.
+ *	clock_adjtime:
+ *
+ *	Interface to adjtime() syscall.
+ *
+ *	Calculates adjustment variables and
+ *	initiates adjustment.
  */
-static
 void
-flush_alarms(
-	clock_t				clock)
+clock_adjtime(
+	int32_t		*secs,
+	int32_t		*microsecs)
 {
-	register alarm_t	alrm1, alrm2;
-	spl_t				s;
+	uint32_t	interval;
+	spl_t		s;
 
-	/*
-	 * Flush all outstanding alarms.
-	 */
-	LOCK_CLOCK(s);
-	alrm1 = (alarm_t) &clock->cl_alarm;
-	while ((alrm2 = alrm1->al_next) != NULL) {
-		/*
-		 * Remove alarm from the clock alarm list.
-		 */  
-		if ((alrm1->al_next = alrm2->al_next) != NULL)
-			(alrm1->al_next)->al_prev = alrm1;
+	s = splclock();
+	simple_lock(&clock_lock);
 
-		/*
-		 * If a clock_sleep() alarm, wakeup the thread
-		 * which issued the clock_sleep() call.
-		 */
-		if (alrm2->al_status == ALARM_SLEEP) {
-			alrm2->al_next = 0;
-			thread_wakeup((event_t)alrm2);
-		}
-		else {
-			/*
-			 * If a clock_alarm() alarm, place the alarm on
-			 * the alarm done list and wakeup the dedicated
-			 * kernel alarm_thread to service the alarm.
-			 */
-			assert(alrm2->al_status == ALARM_CLOCK);
-			if ((alrm2->al_next = alrmdone) != NULL)
-				alrmdone->al_prev = alrm2;
-			else
-				thread_wakeup((event_t)&alrmdone);
-			alrm2->al_prev = (alarm_t) &alrmdone;
-			alrmdone = alrm2;
-		}
+	interval = calend_set_adjustment(secs, microsecs);
+	if (interval != 0) {
+		calend_adjdeadline = mach_absolute_time() + interval;
+		if (!timer_call_enter(&calend_adjcall, calend_adjdeadline))
+			calend_adjactive++;
 	}
-	UNLOCK_CLOCK(s);
+	else
+	if (timer_call_cancel(&calend_adjcall))
+		calend_adjactive--;
+
+	simple_unlock(&clock_lock);
+	splx(s);
 }
 
-/*
- * Post an alarm on a clock's active alarm list. The alarm is
- * inserted in time-order into the clock's active alarm list.
- * Always called from within a LOCK_CLOCK() code section.
- */
-static
-void
-post_alarm(
-	clock_t				clock,
-	alarm_t				alarm)
+static uint32_t
+calend_set_adjustment(
+	int32_t				*secs,
+	int32_t				*microsecs)
 {
-	register alarm_t	alrm1, alrm2;
-	mach_timespec_t		*alarm_time;
-	mach_timespec_t		*queue_time;
+	uint64_t		now, t64;
+	int64_t			total, ototal;
+	uint32_t		interval = 0;
 
-	/*
-	 * Traverse alarm list until queue time is greater
-	 * than alarm time, then insert alarm.
-	 */
-	alarm_time = &alarm->al_time;
-	alrm1 = (alarm_t) &clock->cl_alarm;
-	while ((alrm2 = alrm1->al_next) != NULL) {
-		queue_time = &alrm2->al_time;
-		if (CMP_MACH_TIMESPEC(queue_time, alarm_time) > 0)
-			break;
-		alrm1 = alrm2;
-	}
-	alrm1->al_next = alarm;
-	alarm->al_next = alrm2;
-	alarm->al_prev = alrm1;
-	if (alrm2)
-		alrm2->al_prev  = alarm;
+	total = (int64_t)*secs * NSEC_PER_SEC + *microsecs * NSEC_PER_USEC;
 
-	/*
-	 * If the inserted alarm is the 'earliest' alarm,
-	 * reset the device layer alarm time accordingly.
-	 */
-	if (clock->cl_alarm.al_next == alarm)
-		(*clock->cl_ops->c_setalrm)(alarm_time);
-}
+    commpage_set_timestamp(0,0,0);
 
-/*
- * Check the validity of 'alarm_time' and 'alarm_type'. If either
- * argument is invalid, return a negative value. If the 'alarm_time'
- * is now, return a 0 value. If the 'alarm_time' is in the future,
- * return a positive value.
- */
-static
-int
-check_time(
-	alarm_type_t		alarm_type,
-	mach_timespec_t		*alarm_time,
-	mach_timespec_t		*clock_time)
-{
-	int					result;
+	now = mach_absolute_time();
 
-	if (BAD_ALRMTYPE(alarm_type))
-		return (-1);
-	if (BAD_MACH_TIMESPEC(alarm_time))
-		return (-1);
-	if ((alarm_type & ALRMTYPE) == TIME_RELATIVE)
-		ADD_MACH_TIMESPEC(alarm_time, clock_time);
+	ototal = calend_adjtotal;
 
-	result = CMP_MACH_TIMESPEC(alarm_time, clock_time);
+	if (total != 0) {
+		int32_t		delta = calend_adjskew;
 
-	return ((result >= 0)? result: 0);
-}
+		if (total > 0) {
+			if (total > calend_adjbig)
+				delta *= 10;
+			if (delta > total)
+				delta = total;
 
-mach_timespec_t
-clock_get_system_value(void)
-{
-	clock_t				clock = &clock_list[SYSTEM_CLOCK];
-	mach_timespec_t		value;
+			nanoseconds_to_absolutetime((uint64_t)delta, &t64);
+			calend_adjoffset = t64;
+		}
+		else {
+			if (total < -calend_adjbig)
+				delta *= 10;
+			delta = -delta;
+			if (delta < total)
+				delta = total;
 
-	(void) (*clock->cl_ops->c_gettime)(&value);
+			calend_adjstart = now;
 
-	return value;
-}
+			nanoseconds_to_absolutetime((uint64_t)-delta, &t64);
+			calend_adjoffset = t64;
+		}
 
-mach_timespec_t
-clock_get_calendar_value(void)
-{
-	clock_t				clock = &clock_list[CALENDAR_CLOCK];
-	mach_timespec_t		value = MACH_TIMESPEC_ZERO;
+		calend_adjtotal = total;
+		calend_adjdelta = delta;
+
+		interval = calend_adjinterval;
+	}
+	else
+		calend_adjdelta = calend_adjtotal = 0;
 
-	(void) (*clock->cl_ops->c_gettime)(&value);
+	if (ototal != 0) {
+		*secs = ototal / NSEC_PER_SEC;
+		*microsecs = (ototal % NSEC_PER_SEC) / NSEC_PER_USEC;
+	}
+	else
+		*secs = *microsecs = 0;
 
-	return value;
+	return (interval);
 }
 
-void
-clock_deadline_for_periodic_event(
-	uint64_t			interval,
-	uint64_t			abstime,
-	uint64_t			*deadline)
+static void
+calend_adjust_call(void)
 {
-	assert(interval != 0);
+	uint32_t	interval;
+	spl_t		s;
 
-	*deadline += interval;
+	s = splclock();
+	simple_lock(&clock_lock);
 
-	if (*deadline <= abstime) {
-		*deadline = abstime + interval;
-		abstime = mach_absolute_time();
+	if (--calend_adjactive == 0) {
+		interval = calend_adjust();
+		if (interval != 0) {
+			clock_deadline_for_periodic_event(interval, mach_absolute_time(),
+																&calend_adjdeadline);
 
-		if (*deadline <= abstime)
-			*deadline = abstime + interval;
+			if (!timer_call_enter(&calend_adjcall, calend_adjdeadline))
+				calend_adjactive++;
+		}
 	}
+
+	simple_unlock(&clock_lock);
+	splx(s);
 }
 
-void
-mk_timebase_info_trap(
-	struct mk_timebase_info_trap_args *args)
+static uint32_t
+calend_adjust(void)
 {
-	uint32_t					*delta = args->delta;
-	uint32_t					*abs_to_ns_numer = args->abs_to_ns_numer;
-	uint32_t					*abs_to_ns_denom = args->abs_to_ns_denom;
-	uint32_t					*proc_to_abs_numer = args->proc_to_abs_numer;
-	uint32_t					*proc_to_abs_denom = args->proc_to_abs_denom;
-	mach_timebase_info_data_t	info;
-	uint32_t					one = 1;
+	uint64_t		now, t64;
+	int32_t			delta;
+	uint32_t		interval = 0;
 
-	clock_timebase_info(&info);
+    commpage_set_timestamp(0,0,0);
 
-	copyout((void *)&one, CAST_USER_ADDR_T(delta), sizeof (uint32_t));
+	now = mach_absolute_time();
 
-	copyout((void *)&info.numer, CAST_USER_ADDR_T(abs_to_ns_numer), sizeof (uint32_t));
-	copyout((void *)&info.denom, CAST_USER_ADDR_T(abs_to_ns_denom), sizeof (uint32_t));
+	delta = calend_adjdelta;
 
-	copyout((void *)&one, CAST_USER_ADDR_T(proc_to_abs_numer), sizeof (uint32_t));
-	copyout((void *)&one, CAST_USER_ADDR_T(proc_to_abs_denom), sizeof (uint32_t));
-}
+	if (delta > 0) {
+		clock_calend.offset += calend_adjoffset;
 
-kern_return_t
-mach_timebase_info_trap(
-	struct mach_timebase_info_trap_args *args)
-{
-	mach_vm_address_t 			out_info_addr = args->info;
-	mach_timebase_info_data_t	info;
+		calend_adjtotal -= delta;
+		if (delta > calend_adjtotal) {
+			calend_adjdelta = delta = calend_adjtotal;
 
-	clock_timebase_info(&info);
+			nanoseconds_to_absolutetime((uint64_t)delta, &t64);
+			calend_adjoffset = t64;
+		}
+	}
+	else
+	if (delta < 0) {
+		clock_calend.offset -= calend_adjoffset;
 
-	copyout((void *)&info, out_info_addr, sizeof (info));
+		calend_adjtotal -= delta;
+		if (delta < calend_adjtotal) {
+			calend_adjdelta = delta = calend_adjtotal;
 
-	return (KERN_SUCCESS);
+			nanoseconds_to_absolutetime((uint64_t)-delta, &t64);
+			calend_adjoffset = t64;
+		}
+
+		if (calend_adjdelta != 0)
+			calend_adjstart = now;
+	}
+
+	if (calend_adjdelta != 0)
+		interval = calend_adjinterval;
+
+	return (interval);
+}
+
+/*
+ *	clock_wakeup_calendar:
+ *
+ *	Interface to power management, used
+ *	to initiate the reset of the calendar
+ *	on wake from sleep event.
+ */
+void
+clock_wakeup_calendar(void)
+{
+	thread_call_enter(&calend_wakecall);
 }
 
+/*
+ *	Wait / delay routines.
+ */
 static void
 mach_wait_until_continue(
 	__unused void	*parameter,
@@ -895,9 +665,6 @@ mach_wait_until_trap(
 	return ((wresult == THREAD_INTERRUPTED)? KERN_ABORTED: KERN_SUCCESS);
 }
 
-/*
- * Delay primitives.
- */
 void
 clock_delay_until(
 	uint64_t		deadline)
@@ -937,69 +704,52 @@ delay(
 	delay_for_interval((usec < 0)? -usec: usec, NSEC_PER_USEC);
 }
 
+/*
+ *	Miscellaneous routines.
+ */
 void
-clock_adjtime(
-	int32_t		*secs,
-	int32_t		*microsecs)
+clock_interval_to_deadline(
+	uint32_t			interval,
+	uint32_t			scale_factor,
+	uint64_t			*result)
 {
-	uint32_t	interval;
-	spl_t		s;
-
-	s = splclock();
-	simple_lock(&calend_adjlock);
-
-	interval = clock_set_calendar_adjtime(secs, microsecs);
-	if (interval != 0) {
-		if (calend_adjdeadline >= interval)
-			calend_adjdeadline -= interval;
-		clock_deadline_for_periodic_event(interval, mach_absolute_time(),
-												&calend_adjdeadline);
+	uint64_t	abstime;
 
-		timer_call_enter(&calend_adjcall, calend_adjdeadline);
-	}
-	else
-		timer_call_cancel(&calend_adjcall);
+	clock_interval_to_absolutetime_interval(interval, scale_factor, &abstime);
 
-	simple_unlock(&calend_adjlock);
-	splx(s);
+	*result = mach_absolute_time() + abstime;
 }
 
-static void
-calend_adjust_call(
-	__unused timer_call_param_t		p0,
-	__unused timer_call_param_t		p1)
+void
+clock_absolutetime_interval_to_deadline(
+	uint64_t			abstime,
+	uint64_t			*result)
 {
-	uint32_t	interval;
-	spl_t		s;
-
-	s = splclock();
-	simple_lock(&calend_adjlock);
-
-	interval = clock_adjust_calendar();
-	if (interval != 0) {
-		clock_deadline_for_periodic_event(interval, mach_absolute_time(),
-								  				&calend_adjdeadline);
-
-		timer_call_enter(&calend_adjcall, calend_adjdeadline);
-	}
-
-	simple_unlock(&calend_adjlock);
-	splx(s);
+	*result = mach_absolute_time() + abstime;
 }
 
 void
-clock_wakeup_calendar(void)
+clock_get_uptime(
+	uint64_t	*result)
 {
-	thread_call_enter(&calend_wakecall);
+	*result = mach_absolute_time();
 }
 
-extern	void		IOKitResetTime(void); /* XXX */
-
-static void
-calend_dowakeup(
-	__unused thread_call_param_t		p0,
-	__unused thread_call_param_t		p1)
+void
+clock_deadline_for_periodic_event(
+	uint64_t			interval,
+	uint64_t			abstime,
+	uint64_t			*deadline)
 {
+	assert(interval != 0);
+
+	*deadline += interval;
+
+	if (*deadline <= abstime) {
+		*deadline = abstime + interval;
+		abstime = mach_absolute_time();
 
-	IOKitResetTime();
+		if (*deadline <= abstime)
+			*deadline = abstime + interval;
+	}
 }
diff --git a/osfmk/kern/clock.h b/osfmk/kern/clock.h
index cb605095f..676641763 100644
--- a/osfmk/kern/clock.h
+++ b/osfmk/kern/clock.h
@@ -29,10 +29,6 @@
  * @OSF_COPYRIGHT@
  */
 /*
- *	File:		kern/clock.h
- *	Purpose:	Data structures for the kernel alarm clock
- *			facility. This file is used only by kernel
- *			level clock facility routines.
  */
 
 #ifndef	_KERN_CLOCK_H_
@@ -50,44 +46,9 @@
 
 #ifdef	MACH_KERNEL_PRIVATE
 
-/*
- * Actual clock alarm structure. Used for user clock_sleep() and
- * clock_alarm() calls. Alarms are allocated from the alarm free
- * list and entered in time priority order into the active alarm
- * chain of the target clock.
- */
-struct	alarm {
-	struct	alarm	*al_next;		/* next alarm in chain */
-	struct	alarm	*al_prev;		/* previous alarm in chain */
-	int				al_status;		/* alarm status */
-	mach_timespec_t	al_time;		/* alarm time */
-	struct {				/* message alarm data */
-		int				type;		/* alarm type */
-		ipc_port_t		port;		/* alarm port */
-		mach_msg_type_name_t
-						port_type;	/* alarm port type */
-		struct	clock	*clock;		/* alarm clock */
-		void			*data;		/* alarm data */
-	} al_alrm;
-#define al_type		al_alrm.type
-#define al_port		al_alrm.port
-#define al_port_type	al_alrm.port_type
-#define al_clock	al_alrm.clock
-#define al_data		al_alrm.data
-	long			al_seqno;		/* alarm sequence number */
-};
-typedef struct alarm	alarm_data_t;
-
-/* alarm status */
-#define ALARM_FREE	0		/* alarm is on free list */
-#define	ALARM_SLEEP	1		/* active clock_sleep() */
-#define ALARM_CLOCK	2		/* active clock_alarm() */
-#define ALARM_DONE	4		/* alarm has expired */
-
 /*
  * Clock operations list structure. Contains vectors to machine
- * dependent clock routines. The routines c_config, c_init, and
- * c_gettime must be implemented for every clock device.
+ * dependent clock routines.
  */
 struct	clock_ops {
 	int		(*c_config)(void);		/* configuration */
@@ -97,37 +58,22 @@ struct	clock_ops {
 	kern_return_t	(*c_gettime)(	/* get time */
 				mach_timespec_t			*cur_time);
 
-	kern_return_t	(*c_settime)(	/* set time */
-				mach_timespec_t			*clock_time);
-
 	kern_return_t	(*c_getattr)(	/* get attributes */
 				clock_flavor_t			flavor,
 				clock_attr_t			attr,
 				mach_msg_type_number_t	*count);
-
-	kern_return_t	(*c_setattr)(	/* set attributes */
-				clock_flavor_t			flavor,
-				clock_attr_t			attr,
-				mach_msg_type_number_t	count);
-
-	void		(*c_setalrm)(		/* set next alarm */
-				mach_timespec_t			*alarm_time);
 };
 typedef struct clock_ops	*clock_ops_t;
 typedef struct clock_ops	clock_ops_data_t;
 
 /*
  * Actual clock object data structure. Contains the machine
- * dependent operations list, clock operations ports, and a
- * chain of pending alarms.
+ * dependent operations list and clock operation ports.
  */
 struct	clock {
 	clock_ops_t			cl_ops;			/* operations list */
 	struct ipc_port		*cl_service;	/* service port */
 	struct ipc_port		*cl_control;	/* control port */
-	struct	{							/* alarm chain head */
-		struct alarm 	*al_next;
-	} cl_alarm;
 };
 typedef struct clock		clock_data_t;
 
@@ -135,11 +81,13 @@ typedef struct clock		clock_data_t;
  * Configure the clock system.
  */
 extern void		clock_config(void);
+extern void		clock_oldconfig(void);
 
 /*
  * Initialize the clock system.
  */
 extern void		clock_init(void);
+extern void		clock_oldinit(void);
 
 extern void		clock_timebase_init(void);
 
@@ -148,20 +96,6 @@ extern void		clock_timebase_init(void);
  */
 extern void		clock_service_create(void);
 
-/*
- * Service clock alarm interrupts. Called from machine dependent
- * layer at splclock(). The clock_id argument specifies the clock,
- * and the clock_time argument gives that clock's current time.
- */
-extern void		clock_alarm_intr(
-					clock_id_t		clock_id,
-					mach_timespec_t	*clock_time);
-
-extern kern_return_t	clock_sleep_internal(
-							clock_t			clock,
-							sleep_type_t	sleep_type,
-							mach_timespec_t	*sleep_time);
-
 typedef void		(*clock_timer_func_t)(
 						uint64_t			timestamp);
 
@@ -171,11 +105,12 @@ extern void			clock_set_timer_func(
 extern void			clock_set_timer_deadline(
 						uint64_t			deadline);
 
-extern uint32_t		clock_set_calendar_adjtime(
-						int32_t				*secs,
-						int32_t				*microsecs);
-
-extern uint32_t		clock_adjust_calendar(void);
+extern void			clock_gettimeofday_set_commpage(
+						uint64_t				abstime,
+						uint64_t				epoch,
+						uint64_t				offset,
+						uint32_t				*secs,
+						uint32_t				*microsecs);
 
 extern void			machine_delay_until(
 						uint64_t		deadline);
@@ -194,6 +129,16 @@ extern void		absolutetime_to_microtime(
 					uint32_t		*secs,
 					uint32_t		*microsecs);
 
+extern void		absolutetime_to_nanotime(
+					uint64_t		abstime,
+					uint32_t		*secs,
+					uint32_t		*nanosecs);
+
+extern void		nanotime_to_absolutetime(
+				    uint32_t		secs,
+					uint32_t		nanosecs,
+					uint64_t		*result);
+
 #endif /* MACH_KERNEL_PRIVATE */
 
 __BEGIN_DECLS
diff --git a/osfmk/kern/clock_oldops.c b/osfmk/kern/clock_oldops.c
new file mode 100644
index 000000000..987036954
--- /dev/null
+++ b/osfmk/kern/clock_oldops.c
@@ -0,0 +1,808 @@
+/*
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+/*
+ *	DEPRECATED INTERFACES - Should be removed
+ *
+ *	Purpose:	Routines for the creation and use of kernel
+ *			alarm clock services. This file and the ipc
+ *			routines in kern/ipc_clock.c constitute the
+ *			machine-independent clock service layer.
+ */
+
+#include <mach/mach_types.h>
+
+#include <kern/lock.h>
+#include <kern/host.h>
+#include <kern/spl.h>
+#include <kern/sched_prim.h>
+#include <kern/thread.h>
+#include <kern/ipc_host.h>
+#include <kern/clock.h>
+#include <kern/zalloc.h>
+
+#include <ipc/ipc_types.h>
+#include <ipc/ipc_port.h>
+
+#include <mach/mach_traps.h>
+#include <mach/mach_time.h>
+
+#include <mach/clock_server.h>
+#include <mach/clock_reply.h>
+#include <mach/clock_priv_server.h>
+
+#include <mach/mach_host_server.h>
+#include <mach/host_priv_server.h>
+
+/*
+ * Actual clock alarm structure. Used for user clock_sleep() and
+ * clock_alarm() calls. Alarms are allocated from the alarm free
+ * list and entered in time priority order into the active alarm
+ * chain of the target clock.
+ */
+struct	alarm {
+	struct	alarm	*al_next;		/* next alarm in chain */
+	struct	alarm	*al_prev;		/* previous alarm in chain */
+	int				al_status;		/* alarm status */
+	mach_timespec_t	al_time;		/* alarm time */
+	struct {				/* message alarm data */
+		int				type;		/* alarm type */
+		ipc_port_t		port;		/* alarm port */
+		mach_msg_type_name_t
+						port_type;	/* alarm port type */
+		struct	clock	*clock;		/* alarm clock */
+		void			*data;		/* alarm data */
+	} al_alrm;
+#define al_type		al_alrm.type
+#define al_port		al_alrm.port
+#define al_port_type	al_alrm.port_type
+#define al_clock	al_alrm.clock
+#define al_data		al_alrm.data
+	long			al_seqno;		/* alarm sequence number */
+};
+typedef struct alarm	alarm_data_t;
+
+/* alarm status */
+#define ALARM_FREE	0		/* alarm is on free list */
+#define	ALARM_SLEEP	1		/* active clock_sleep() */
+#define ALARM_CLOCK	2		/* active clock_alarm() */
+#define ALARM_DONE	4		/* alarm has expired */
+
+/* local data declarations */
+decl_simple_lock_data(static,alarm_lock)	/* alarm synchronization */
+static struct	zone		*alarm_zone;	/* zone for user alarms */
+static struct	alarm		*alrmfree;		/* alarm free list pointer */
+static struct	alarm		*alrmdone;		/* alarm done list pointer */
+static struct	alarm		*alrmlist;
+static long					alrm_seqno;		/* uniquely identifies alarms */
+static thread_call_data_t	alarm_done_call;
+static timer_call_data_t	alarm_expire_timer;
+
+extern	struct clock	clock_list[];
+extern	int		clock_count;
+
+static void		post_alarm(
+					alarm_t			alarm);
+
+static void		set_alarm(
+					mach_timespec_t	*alarm_time);
+
+static int		check_time(
+					alarm_type_t	alarm_type,
+					mach_timespec_t	*alarm_time,
+					mach_timespec_t	*clock_time);
+
+static void		alarm_done(void);
+
+static void		alarm_expire(void);
+
+static kern_return_t	clock_sleep_internal(
+							clock_t				clock,
+							sleep_type_t		sleep_type,
+							mach_timespec_t		*sleep_time);
+
+int		rtclock_config(void);
+
+int		rtclock_init(void);
+
+kern_return_t	rtclock_gettime(
+	mach_timespec_t			*cur_time);
+
+kern_return_t	rtclock_getattr(
+	clock_flavor_t			flavor,
+	clock_attr_t			attr,
+	mach_msg_type_number_t	*count);
+
+struct clock_ops sysclk_ops = {
+	rtclock_config,			rtclock_init,
+	rtclock_gettime,
+	rtclock_getattr,
+};
+
+kern_return_t	calend_gettime(
+	mach_timespec_t			*cur_time);
+
+kern_return_t	calend_getattr(
+	clock_flavor_t			flavor,
+	clock_attr_t			attr,
+	mach_msg_type_number_t	*count);
+
+struct clock_ops calend_ops = {
+	0,					0,
+	calend_gettime,
+	calend_getattr,
+};
+
+/*
+ *	Macros to lock/unlock clock system.
+ */
+#define LOCK_ALARM(s)			\
+	s = splclock();			\
+	simple_lock(&alarm_lock);
+
+#define UNLOCK_ALARM(s)			\
+	simple_unlock(&alarm_lock);	\
+	splx(s);
+
+void
+clock_oldconfig(void)
+{
+	clock_t			clock;
+	register int 	i;
+
+	simple_lock_init(&alarm_lock, 0);
+	thread_call_setup(&alarm_done_call, (thread_call_func_t)alarm_done, NULL);
+	timer_call_setup(&alarm_expire_timer, (timer_call_func_t)alarm_expire, NULL);
+
+	/*
+	 * Configure clock devices.
+	 */
+	for (i = 0; i < clock_count; i++) {
+		clock = &clock_list[i];
+		if (clock->cl_ops && clock->cl_ops->c_config) {
+			if ((*clock->cl_ops->c_config)() == 0)
+				clock->cl_ops = 0;
+		}
+	}
+
+	/* start alarm sequence numbers at 0 */
+	alrm_seqno = 0;
+}
+
+void
+clock_oldinit(void)
+{
+	clock_t			clock;
+	register int	i;
+
+	/*
+	 * Initialize basic clock structures.
+	 */
+	for (i = 0; i < clock_count; i++) {
+		clock = &clock_list[i];
+		if (clock->cl_ops && clock->cl_ops->c_init)
+			(*clock->cl_ops->c_init)();
+	}
+}
+
+/*
+ * Initialize the clock ipc service facility.
+ */
+void
+clock_service_create(void)
+{
+	clock_t			clock;
+	register int	i;
+
+	/*
+	 * Initialize ipc clock services.
+	 */
+	for (i = 0; i < clock_count; i++) {
+		clock = &clock_list[i];
+		if (clock->cl_ops) {
+			ipc_clock_init(clock);
+			ipc_clock_enable(clock);
+		}
+	}
+
+	/*
+	 * Perform miscellaneous late
+	 * initialization.
+	 */
+	i = sizeof(struct alarm);
+	alarm_zone = zinit(i, (4096/i)*i, 10*i, "alarms");
+}
+
+/*
+ * Get the service port on a clock.
+ */
+kern_return_t
+host_get_clock_service(
+	host_t			host,
+	clock_id_t		clock_id,
+	clock_t			*clock)		/* OUT */
+{
+	if (host == HOST_NULL || clock_id < 0 || clock_id >= clock_count) {
+		*clock = CLOCK_NULL;
+		return (KERN_INVALID_ARGUMENT);
+	}
+
+	*clock = &clock_list[clock_id];
+	if ((*clock)->cl_ops == 0)
+		return (KERN_FAILURE);
+	return (KERN_SUCCESS);
+}
+
+/*
+ * Get the control port on a clock.
+ */
+kern_return_t
+host_get_clock_control(
+	host_priv_t		host_priv,
+	clock_id_t		clock_id,
+	clock_t			*clock)		/* OUT */
+{
+	if (host_priv == HOST_PRIV_NULL ||
+			clock_id < 0 || clock_id >= clock_count) {
+		*clock = CLOCK_NULL;
+		return (KERN_INVALID_ARGUMENT);
+	}
+
+	*clock = &clock_list[clock_id];
+	if ((*clock)->cl_ops == 0)
+		return (KERN_FAILURE);
+	return (KERN_SUCCESS);
+}
+
+/*
+ * Get the current clock time.
+ */
+kern_return_t
+clock_get_time(
+	clock_t			clock,
+	mach_timespec_t	*cur_time)	/* OUT */
+{
+	if (clock == CLOCK_NULL)
+		return (KERN_INVALID_ARGUMENT);
+	return ((*clock->cl_ops->c_gettime)(cur_time));
+}
+
+kern_return_t
+rtclock_gettime(
+	mach_timespec_t		*time)	/* OUT */
+{
+	clock_get_system_nanotime(&time->tv_sec, (uint32_t *)&time->tv_nsec);
+
+	return (KERN_SUCCESS);
+}
+
+kern_return_t
+calend_gettime(
+	mach_timespec_t		*time)	/* OUT */
+{
+	clock_get_calendar_nanotime(&time->tv_sec, (uint32_t *)&time->tv_nsec);
+
+	return (KERN_SUCCESS);
+}
+
+/*
+ * Get clock attributes.
+ */
+kern_return_t
+clock_get_attributes(
+	clock_t					clock,
+	clock_flavor_t			flavor,
+	clock_attr_t			attr,		/* OUT */
+	mach_msg_type_number_t	*count)		/* IN/OUT */
+{
+	if (clock == CLOCK_NULL)
+		return (KERN_INVALID_ARGUMENT);
+	if (clock->cl_ops->c_getattr)
+		return (clock->cl_ops->c_getattr(flavor, attr, count));
+	return (KERN_FAILURE);
+}
+
+kern_return_t
+rtclock_getattr(
+	clock_flavor_t			flavor,
+	clock_attr_t			attr,		/* OUT */
+	mach_msg_type_number_t	*count)		/* IN/OUT */
+{
+	if (*count != 1)
+		return (KERN_FAILURE);
+
+	switch (flavor) {
+
+	case CLOCK_GET_TIME_RES:	/* >0 res */
+	case CLOCK_ALARM_CURRES:	/* =0 no alarm */
+	case CLOCK_ALARM_MINRES:
+	case CLOCK_ALARM_MAXRES:
+		*(clock_res_t *) attr = NSEC_PER_SEC / 100;
+		break;
+
+	default:
+		return (KERN_INVALID_VALUE);
+	}
+
+	return (KERN_SUCCESS);
+}
+
+kern_return_t
+calend_getattr(
+	clock_flavor_t			flavor,
+	clock_attr_t			attr,		/* OUT */
+	mach_msg_type_number_t	*count)		/* IN/OUT */
+{
+	if (*count != 1)
+		return (KERN_FAILURE);
+
+	switch (flavor) {
+
+	case CLOCK_GET_TIME_RES:	/* >0 res */
+		*(clock_res_t *) attr = NSEC_PER_SEC / 100;
+		break;
+
+	case CLOCK_ALARM_CURRES:	/* =0 no alarm */
+	case CLOCK_ALARM_MINRES:
+	case CLOCK_ALARM_MAXRES:
+		*(clock_res_t *) attr = 0;
+		break;
+
+	default:
+		return (KERN_INVALID_VALUE);
+	}
+
+	return (KERN_SUCCESS);
+}
+
+/*
+ * Set the current clock time.
+ */
+kern_return_t
+clock_set_time(
+	clock_t					clock,
+__unused mach_timespec_t	new_time)
+{
+	if (clock == CLOCK_NULL)
+		return (KERN_INVALID_ARGUMENT);
+	return (KERN_FAILURE);
+}
+
+/*
+ * Set the clock alarm resolution.
+ */
+kern_return_t
+clock_set_attributes(
+	clock_t						clock,
+__unused clock_flavor_t			flavor,
+__unused clock_attr_t			attr,
+__unused mach_msg_type_number_t	count)
+{
+	if (clock == CLOCK_NULL)
+		return (KERN_INVALID_ARGUMENT);
+	return (KERN_FAILURE);
+}
+
+/*
+ * Setup a clock alarm.
+ */
+kern_return_t
+clock_alarm(
+	clock_t					clock,
+	alarm_type_t			alarm_type,
+	mach_timespec_t			alarm_time,
+	ipc_port_t				alarm_port,
+	mach_msg_type_name_t	alarm_port_type)
+{
+	alarm_t					alarm;
+	mach_timespec_t			clock_time;
+	int						chkstat;
+	kern_return_t			reply_code;
+	spl_t					s;
+
+	if (clock == CLOCK_NULL)
+		return (KERN_INVALID_ARGUMENT);
+	if (clock != &clock_list[SYSTEM_CLOCK])
+		return (KERN_FAILURE);
+	if (IP_VALID(alarm_port) == 0)
+		return (KERN_INVALID_CAPABILITY);
+
+	/*
+	 * Check alarm parameters. If parameters are invalid,
+	 * send alarm message immediately.
+	 */
+	(*clock->cl_ops->c_gettime)(&clock_time);
+	chkstat = check_time(alarm_type, &alarm_time, &clock_time);
+	if (chkstat <= 0) {
+		reply_code = (chkstat < 0 ? KERN_INVALID_VALUE : KERN_SUCCESS);
+		clock_alarm_reply(alarm_port, alarm_port_type,
+				  reply_code, alarm_type, clock_time);
+		return (KERN_SUCCESS);
+	}
+
+	/*
+	 * Get alarm and add to clock alarm list.
+	 */
+
+	LOCK_ALARM(s);
+	if ((alarm = alrmfree) == 0) {
+		UNLOCK_ALARM(s);
+		alarm = (alarm_t) zalloc(alarm_zone);
+		if (alarm == 0)
+			return (KERN_RESOURCE_SHORTAGE);
+		LOCK_ALARM(s);
+	}
+	else
+		alrmfree = alarm->al_next;
+
+	alarm->al_status = ALARM_CLOCK;
+	alarm->al_time = alarm_time;
+	alarm->al_type = alarm_type;
+	alarm->al_port = alarm_port;
+	alarm->al_port_type = alarm_port_type;
+	alarm->al_clock = clock;
+	alarm->al_seqno = alrm_seqno++;
+	post_alarm(alarm);
+	UNLOCK_ALARM(s);
+
+	return (KERN_SUCCESS);
+}
+
+/*
+ * Sleep on a clock. System trap. User-level libmach clock_sleep
+ * interface call takes a mach_timespec_t sleep_time argument which it
+ * converts to sleep_sec and sleep_nsec arguments which are then
+ * passed to clock_sleep_trap.
+ */
+kern_return_t
+clock_sleep_trap(
+	struct clock_sleep_trap_args *args)
+{
+	mach_port_name_t	clock_name = args->clock_name;
+	sleep_type_t		sleep_type = args->sleep_type;
+	int					sleep_sec = args->sleep_sec;
+	int					sleep_nsec = args->sleep_nsec;
+	mach_vm_address_t	wakeup_time_addr = args->wakeup_time;  
+	clock_t				clock;
+	mach_timespec_t		swtime;
+	kern_return_t		rvalue;
+
+	/*
+	 * Convert the trap parameters.
+	 */
+	if (clock_name == MACH_PORT_NULL)
+		clock = &clock_list[SYSTEM_CLOCK];
+	else
+		clock = port_name_to_clock(clock_name);
+
+	swtime.tv_sec  = sleep_sec;
+	swtime.tv_nsec = sleep_nsec;
+
+	/*
+	 * Call the actual clock_sleep routine.
+	 */
+	rvalue = clock_sleep_internal(clock, sleep_type, &swtime);
+
+	/*
+	 * Return current time as wakeup time.
+	 */
+	if (rvalue != KERN_INVALID_ARGUMENT && rvalue != KERN_FAILURE) {
+		copyout((char *)&swtime, wakeup_time_addr, sizeof(mach_timespec_t));
+	}
+	return (rvalue);
+}	
+
+static kern_return_t
+clock_sleep_internal(
+	clock_t				clock,
+	sleep_type_t		sleep_type,
+	mach_timespec_t		*sleep_time)
+{
+	alarm_t				alarm;
+	mach_timespec_t		clock_time;
+	kern_return_t		rvalue;
+	int					chkstat;
+	spl_t				s;
+
+	if (clock == CLOCK_NULL)
+		return (KERN_INVALID_ARGUMENT);
+
+	if (clock != &clock_list[SYSTEM_CLOCK])
+		return (KERN_FAILURE);
+
+	/*
+	 * Check sleep parameters. If parameters are invalid
+	 * return an error, otherwise post alarm request.
+	 */
+	(*clock->cl_ops->c_gettime)(&clock_time);
+
+	chkstat = check_time(sleep_type, sleep_time, &clock_time);
+	if (chkstat < 0)
+		return (KERN_INVALID_VALUE);
+	rvalue = KERN_SUCCESS;
+	if (chkstat > 0) {
+		wait_result_t wait_result;
+
+		/*
+		 * Get alarm and add to clock alarm list.
+		 */
+
+		LOCK_ALARM(s);
+		if ((alarm = alrmfree) == 0) {
+			UNLOCK_ALARM(s);
+			alarm = (alarm_t) zalloc(alarm_zone);
+			if (alarm == 0)
+				return (KERN_RESOURCE_SHORTAGE);
+			LOCK_ALARM(s);
+		}
+		else
+			alrmfree = alarm->al_next;
+
+		/*
+		 * Wait for alarm to occur.
+		 */
+		wait_result = assert_wait((event_t)alarm, THREAD_ABORTSAFE);
+		if (wait_result == THREAD_WAITING) {
+			alarm->al_time = *sleep_time;
+			alarm->al_status = ALARM_SLEEP;
+			post_alarm(alarm);
+			UNLOCK_ALARM(s);
+
+			wait_result = thread_block(THREAD_CONTINUE_NULL);
+
+			/*
+			 * Note if alarm expired normally or whether it
+			 * was aborted. If aborted, delete alarm from
+			 * clock alarm list. Return alarm to free list.
+			 */
+			LOCK_ALARM(s);
+			if (alarm->al_status != ALARM_DONE) {
+				assert(wait_result != THREAD_AWAKENED);
+				if (((alarm->al_prev)->al_next = alarm->al_next) != NULL)
+					(alarm->al_next)->al_prev = alarm->al_prev;
+				rvalue = KERN_ABORTED;
+			}
+			*sleep_time = alarm->al_time;
+			alarm->al_status = ALARM_FREE;
+		} else {
+			assert(wait_result == THREAD_INTERRUPTED);
+			assert(alarm->al_status == ALARM_FREE);
+			rvalue = KERN_ABORTED;
+		}
+		alarm->al_next = alrmfree;
+		alrmfree = alarm;
+		UNLOCK_ALARM(s);
+	}
+	else
+		*sleep_time = clock_time;
+
+	return (rvalue);
+}
+
+/*
+ * Service clock alarm expirations.
+ */
+static void
+alarm_expire(void)
+{
+	clock_t				clock;
+	register alarm_t	alrm1;
+	register alarm_t	alrm2;
+	mach_timespec_t		clock_time;
+	mach_timespec_t		*alarm_time;
+	spl_t				s;
+
+	clock = &clock_list[SYSTEM_CLOCK];
+	(*clock->cl_ops->c_gettime)(&clock_time);
+
+	/*
+	 * Update clock alarm list. Alarms that are due are moved
+	 * to the alarmdone list to be serviced by a thread callout.
+	 */
+	LOCK_ALARM(s);
+	alrm1 = (alarm_t)&alrmlist;
+	while ((alrm2 = alrm1->al_next) != NULL) {
+		alarm_time = &alrm2->al_time;
+		if (CMP_MACH_TIMESPEC(alarm_time, &clock_time) > 0)
+			break;
+
+		/*
+		 * Alarm has expired, so remove it from the
+		 * clock alarm list.
+		 */  
+		if ((alrm1->al_next = alrm2->al_next) != NULL)
+			(alrm1->al_next)->al_prev = alrm1;
+
+		/*
+		 * If a clock_sleep() alarm, wakeup the thread
+		 * which issued the clock_sleep() call.
+		 */
+		if (alrm2->al_status == ALARM_SLEEP) {
+			alrm2->al_next = 0;
+			alrm2->al_status = ALARM_DONE;
+			alrm2->al_time = clock_time;
+			thread_wakeup((event_t)alrm2);
+		}
+
+ 		/*
+		 * If a clock_alarm() alarm, place the alarm on
+		 * the alarm done list and schedule the alarm
+		 * delivery mechanism.
+		 */
+		else {
+			assert(alrm2->al_status == ALARM_CLOCK);
+			if ((alrm2->al_next = alrmdone) != NULL)
+				alrmdone->al_prev = alrm2;
+			else
+				thread_call_enter(&alarm_done_call);
+			alrm2->al_prev = (alarm_t)&alrmdone;
+			alrmdone = alrm2;
+			alrm2->al_status = ALARM_DONE;
+			alrm2->al_time = clock_time;
+		}
+	}
+
+	/*
+	 * Setup to expire for the next pending alarm.
+	 */
+	if (alrm2)
+		set_alarm(alarm_time);
+	UNLOCK_ALARM(s);
+}
+
+static void
+alarm_done(void)
+{
+	register alarm_t	alrm;
+	kern_return_t		code;
+	spl_t				s;
+
+	LOCK_ALARM(s);
+	while ((alrm = alrmdone) != NULL) {
+		if ((alrmdone = alrm->al_next) != NULL)
+			alrmdone->al_prev = (alarm_t)&alrmdone;
+		UNLOCK_ALARM(s);
+
+		code = (alrm->al_status == ALARM_DONE? KERN_SUCCESS: KERN_ABORTED);
+		if (alrm->al_port != IP_NULL) {
+			/* Deliver message to designated port */
+			if (IP_VALID(alrm->al_port)) {
+				clock_alarm_reply(alrm->al_port, alrm->al_port_type, code,
+								  				alrm->al_type, alrm->al_time);
+			}
+
+			LOCK_ALARM(s);
+			alrm->al_status = ALARM_FREE;
+			alrm->al_next = alrmfree;
+			alrmfree = alrm;
+		}
+		else
+			panic("clock_alarm_deliver");
+	}
+
+	UNLOCK_ALARM(s);
+}
+
+/*
+ * Post an alarm on the active alarm list.
+ *
+ * Always called from within a LOCK_ALARM() code section.
+ */
+static void
+post_alarm(
+	alarm_t				alarm)
+{
+	register alarm_t	alrm1, alrm2;
+	mach_timespec_t		*alarm_time;
+	mach_timespec_t		*queue_time;
+
+	/*
+	 * Traverse alarm list until queue time is greater
+	 * than alarm time, then insert alarm.
+	 */
+	alarm_time = &alarm->al_time;
+	alrm1 = (alarm_t)&alrmlist;
+	while ((alrm2 = alrm1->al_next) != NULL) {
+		queue_time = &alrm2->al_time;
+		if (CMP_MACH_TIMESPEC(queue_time, alarm_time) > 0)
+			break;
+		alrm1 = alrm2;
+	}
+	alrm1->al_next = alarm;
+	alarm->al_next = alrm2;
+	alarm->al_prev = alrm1;
+	if (alrm2)
+		alrm2->al_prev  = alarm;
+
+	/*
+	 * If the inserted alarm is the 'earliest' alarm,
+	 * reset the device layer alarm time accordingly.
+	 */
+	if (alrmlist == alarm)
+		set_alarm(alarm_time);
+}
+
+static void
+set_alarm(
+	mach_timespec_t		*alarm_time)
+{
+	uint64_t	abstime;
+
+	nanotime_to_absolutetime(alarm_time->tv_sec, alarm_time->tv_nsec, &abstime);
+	timer_call_enter(&alarm_expire_timer, abstime);
+}
+
+/*
+ * Check the validity of 'alarm_time' and 'alarm_type'. If either
+ * argument is invalid, return a negative value. If the 'alarm_time'
+ * is now, return a 0 value. If the 'alarm_time' is in the future,
+ * return a positive value.
+ */
+static int
+check_time(
+	alarm_type_t		alarm_type,
+	mach_timespec_t		*alarm_time,
+	mach_timespec_t		*clock_time)
+{
+	int					result;
+
+	if (BAD_ALRMTYPE(alarm_type))
+		return (-1);
+	if (BAD_MACH_TIMESPEC(alarm_time))
+		return (-1);
+	if ((alarm_type & ALRMTYPE) == TIME_RELATIVE)
+		ADD_MACH_TIMESPEC(alarm_time, clock_time);
+
+	result = CMP_MACH_TIMESPEC(alarm_time, clock_time);
+
+	return ((result >= 0)? result: 0);
+}
+
+mach_timespec_t
+clock_get_system_value(void)
+{
+	clock_t				clock = &clock_list[SYSTEM_CLOCK];
+	mach_timespec_t		value;
+
+	(void) (*clock->cl_ops->c_gettime)(&value);
+
+	return value;
+}
+
+mach_timespec_t
+clock_get_calendar_value(void)
+{
+	clock_t				clock = &clock_list[CALENDAR_CLOCK];
+	mach_timespec_t		value = MACH_TIMESPEC_ZERO;
+
+	(void) (*clock->cl_ops->c_gettime)(&value);
+
+	return value;
+}
diff --git a/osfmk/kern/debug.h b/osfmk/kern/debug.h
index 60504d606..ed3bb2aa9 100644
--- a/osfmk/kern/debug.h
+++ b/osfmk/kern/debug.h
@@ -100,7 +100,7 @@ extern void	panic_init(void);
 
 __BEGIN_DECLS
 
-extern void	panic(const char	*string, ...);
+extern void	panic(const char *string, ...) __dead2;
 
 __END_DECLS
 
diff --git a/osfmk/kern/etimer.c b/osfmk/kern/etimer.c
new file mode 100644
index 000000000..c4eb5f223
--- /dev/null
+++ b/osfmk/kern/etimer.c
@@ -0,0 +1,200 @@
+/*
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+/*
+ * @APPLE_FREE_COPYRIGHT@
+ */
+/*
+ *	File:		etimer.c
+ *	Purpose:	Routines for handling the machine independent
+ *				event timer.
+ */
+
+#include <mach/mach_types.h>
+
+#include <kern/clock.h>
+#include <kern/thread.h>
+#include <kern/processor.h>
+#include <kern/macro_help.h>
+#include <kern/spl.h>
+#include <kern/etimer.h>
+#include <kern/pms.h>
+
+#include <machine/commpage.h>
+#include <machine/machine_routines.h>
+
+#include <sys/kdebug.h>
+
+#ifdef __ppc__
+#include <ppc/exception.h>
+#else
+#include <i386/cpu_data.h>
+#endif
+
+#include <sys/kdebug.h>
+
+
+/* XXX from <arch>/rtclock.c */
+uint32_t		 	rtclock_tick_interval;
+clock_timer_func_t		rtclock_timer_expire;
+
+#ifdef __ppc__
+# define PER_PROC_INFO		struct per_proc_info
+# define GET_PER_PROC_INFO()	getPerProc()
+#else
+# define PER_PROC_INFO 		cpu_data_t
+# define GET_PER_PROC_INFO()	current_cpu_datap()
+#endif
+
+/*
+ * 	Event timer interrupt.
+ *
+ * XXX a drawback of this implementation is that events serviced earlier must not set deadlines
+ *     that occur before the entire chain completes.
+ *
+ * XXX a better implementation would use a set of generic callouts and iterate over them
+ */
+void etimer_intr(int inuser, uint64_t iaddr) {
+	
+	uint64_t		abstime;
+	rtclock_timer_t		*mytimer;
+	PER_PROC_INFO		*pp;
+
+	pp = GET_PER_PROC_INFO();
+
+	mytimer = &pp->rtclock_timer;				/* Point to the event timer */
+
+	abstime = mach_absolute_time();				/* Get the time now */
+
+	/* is it time for power management state change? */	
+	if (pp->pms.pmsPop <= abstime) {
+
+	        KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 3) | DBG_FUNC_START, 0, 0, 0, 0, 0);
+		pmsStep(1);					/* Yes, advance step */
+	        KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 3) | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+		abstime = mach_absolute_time();			/* Get the time again since we ran a bit */
+	}
+
+	/* have we passed the rtclock pop time? */
+	if (pp->rtclock_intr_deadline <= abstime) {
+
+	        KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 4) | DBG_FUNC_START, (int)rtclock_tick_interval, 0, 0, 0, 0);
+
+		clock_deadline_for_periodic_event(rtclock_tick_interval,
+		    abstime,
+		    &pp->rtclock_intr_deadline);
+
+	        KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 4) | DBG_FUNC_END, 0, 0, 0, 0, 0);
+#if STAT_TIME
+		hertz_tick(NSEC_PER_HZ, inuser, iaddr);		/* Accumulate hertz */
+#else
+		hertz_tick(inuser, iaddr);			/* Accumulate hertz */
+#endif
+
+		abstime = mach_absolute_time();			/* Refresh the current time since we went away */
+	}
+
+	/* has a pending clock timer expired? */
+	if (mytimer->deadline <= abstime) {			/* Have we expired the deadline? */
+		mytimer->has_expired = TRUE;			/* Remember that we popped */
+		mytimer->deadline = EndOfAllTime;		/* Set timer request to the end of all time in case we have no more events */
+		(*rtclock_timer_expire)(abstime);		/* Process pop */
+		mytimer->has_expired = FALSE;
+	}
+
+	/* schedule our next deadline */
+	pp->rtcPop = EndOfAllTime;				/* any real deadline will be earlier */
+	etimer_resync_deadlines();
+}
+
+/*
+ * Set the clock deadline; called by the thread scheduler.
+ */
+void etimer_set_deadline(uint64_t deadline)
+{
+	rtclock_timer_t		*mytimer;
+	spl_t			s;
+	PER_PROC_INFO		*pp;
+
+	s = splclock();					/* no interruptions */
+	pp = GET_PER_PROC_INFO();
+
+	mytimer = &pp->rtclock_timer;			/* Point to the timer itself */
+	mytimer->deadline = deadline;			/* Set the new expiration time */
+
+	etimer_resync_deadlines();
+
+	splx(s);
+}
+
+/*
+ * Re-evaluate the outstanding deadlines and select the most proximate.
+ *
+ * Should be called at splclock.
+ */
+void
+etimer_resync_deadlines(void)
+{
+	uint64_t		deadline;
+	rtclock_timer_t		*mytimer;
+	spl_t			s = splclock();		/* No interruptions please */
+	PER_PROC_INFO		*pp;
+
+	pp = GET_PER_PROC_INFO();
+
+	deadline = 0;
+	
+	/* next rtclock interrupt? */
+	if (pp->rtclock_intr_deadline > 0)
+		deadline = pp->rtclock_intr_deadline;
+
+	/* if we have a clock timer set sooner, pop on that */
+	mytimer = &pp->rtclock_timer;			/* Point to the timer itself */
+	if ((!mytimer->has_expired) && (mytimer->deadline > 0) && (mytimer->deadline < deadline))
+		deadline = mytimer->deadline;
+
+	/* if we have a power management event coming up, how about that? */
+	if ((pp->pms.pmsPop > 0) && (pp->pms.pmsPop < deadline))
+		deadline = pp->pms.pmsPop;
+	
+#ifdef __ppc__
+#endif
+
+        if ((deadline > 0) && (deadline < pp->rtcPop)) {
+		int     decr;
+
+		pp->rtcPop = deadline;
+		decr = setPop(deadline);
+
+		KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 1) | DBG_FUNC_NONE, decr, 2, 0, 0, 0);
+	}
+	splx(s);
+}
diff --git a/osfmk/kern/etimer.h b/osfmk/kern/etimer.h
new file mode 100644
index 000000000..2f9268fd4
--- /dev/null
+++ b/osfmk/kern/etimer.h
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+/*
+ * @APPLE_FREE_COPYRIGHT@
+ */
+/*
+ *	File:		etimer.h
+ *	Purpose:	Routines for handling the machine independent
+ *				real-time clock.
+ */
+
+#ifndef _ETIMER_H_
+#define _ETIMER_H_
+
+#define EndOfAllTime	0xFFFFFFFFFFFFFFFFULL
+
+/* extern void rtclock_intr(int inuser, uint64_t iaddr);  - this is currently MD */
+typedef	void (*etimer_intr_t)(int, uint64_t);
+
+extern int setTimerReq(void);
+extern void etimer_intr(int inuser, uint64_t iaddr);
+
+extern void etimer_set_deadline(uint64_t deadline);
+extern int setPop(uint64_t time);
+
+extern void etimer_resync_deadlines(void);
+
+extern uint32_t rtclock_tick_interval;
+
+extern uint64_t tsc_to_nanoseconds(uint64_t abstime);
+
+#if 0 /* this is currently still MD */
+#pragma pack(push,4)
+struct rtclock_timer_t  {
+	uint64_t		deadline;
+	uint32_t
+	/*boolean_t*/	is_set:1,
+					has_expired:1,
+					:0;
+};
+#pragma pack(pop)
+typedef struct rtclock_timer_t rtclock_timer_t;
+#endif /* MD */
+
+
+#endif /* _ETIMER_H_ */
diff --git a/osfmk/kern/hibernate.c b/osfmk/kern/hibernate.c
index 4d9b6b130..96c0275e1 100644
--- a/osfmk/kern/hibernate.c
+++ b/osfmk/kern/hibernate.c
@@ -57,17 +57,13 @@ hibernate_page_list_zero(hibernate_page_list_t *list)
     bitmap = &list->bank_bitmap[0];
     for (bank = 0; bank < list->bank_count; bank++)
     {
-        uint32_t bit, last_bit;
-        uint32_t *bitmap_word;
+        uint32_t last_bit;
 
 	bzero((void *) &bitmap->bitmap[0], bitmap->bitmapwords << 2); 
-
-        // Set out-of-bound bits at end of bitmap.
-        bitmap_word = &bitmap->bitmap[bitmap->bitmapwords - 1];
-        last_bit = ((bitmap->last_page - bitmap->first_page) & 31);
-        for (bit = 31; bit > last_bit; bit--) {
-            *bitmap_word |= (0x80000000 >> bit);
-        }
+        // set out-of-bound bits at end of bitmap.
+        last_bit = ((bitmap->last_page - bitmap->first_page + 1) & 31);
+	if (last_bit)
+	    bitmap->bitmap[bitmap->bitmapwords - 1] = (0xFFFFFFFF >> last_bit);
 
 	bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
     }
@@ -171,6 +167,7 @@ discard_page(vm_page_t m)
  pages known to VM to not need saving are subtracted.
  Wired pages to be saved are present in page_list_wired, pageable in page_list.
 */
+extern vm_page_t vm_lopage_queue_free;
 
 void
 hibernate_page_list_setall(hibernate_page_list_t * page_list,
@@ -212,6 +209,16 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
 	m = (vm_page_t) m->pageq.next;
     }
 
+    m = (vm_page_t) vm_lopage_queue_free;
+    while(m)
+    {
+	pages--;
+	count_wire--;
+	hibernate_page_bitset(page_list,       TRUE, m->phys_page);
+	hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
+	m = (vm_page_t) m->pageq.next;
+    }
+
     queue_iterate( &vm_page_queue_zf,
                     m,
                     vm_page_t,
diff --git a/osfmk/kern/ipc_kobject.h b/osfmk/kern/ipc_kobject.h
index 05b5891a5..60cb14640 100644
--- a/osfmk/kern/ipc_kobject.h
+++ b/osfmk/kern/ipc_kobject.h
@@ -101,7 +101,7 @@ typedef natural_t	ipc_kobject_type_t;
 #define IKOT_HOST_SECURITY		17
 #define	IKOT_LEDGER				18
 #define IKOT_MASTER_DEVICE		19
-/* (unused)						20	*/
+#define IKOT_TASK_NAME			20
 #define IKOT_SUBSYSTEM			21
 #define IKOT_IO_DONE_QUEUE		22
 #define IKOT_SEMAPHORE			23
@@ -113,11 +113,14 @@ typedef natural_t	ipc_kobject_type_t;
 #define IKOT_IOKIT_CONNECT		29
 #define IKOT_IOKIT_OBJECT		30
 #define IKOT_UPL				31
-								/* << new entries here	*/
-#define	IKOT_UNKNOWN			32	/* magic catchall	*/
-#define	IKOT_MAX_TYPE			33	/* # of IKOT_ types	*/
+#define IKOT_MEM_OBJ_CONTROL		32
+/*
+ * Add new entries here and adjust IKOT_UNKNOWN.
+ * Please keep ipc/ipc_object.c:ikot_print_array up to date.
+ */
+#define	IKOT_UNKNOWN			33	/* magic catchall	*/
+#define	IKOT_MAX_TYPE	(IKOT_UNKNOWN+1)	/* # of IKOT_ types	*/
 
-/* Please keep ipc/ipc_object.c:ikot_print_array up to date	*/
 
 #define is_ipc_kobject(ikot)	((ikot) != IKOT_NONE)
 
diff --git a/osfmk/kern/ipc_tt.c b/osfmk/kern/ipc_tt.c
index 2f6a9521f..30a0e7b13 100644
--- a/osfmk/kern/ipc_tt.c
+++ b/osfmk/kern/ipc_tt.c
@@ -113,6 +113,7 @@ ipc_task_init(
 {
 	ipc_space_t space;
 	ipc_port_t kport;
+	ipc_port_t nport;
 	kern_return_t kr;
 	int i;
 
@@ -126,8 +127,13 @@ ipc_task_init(
 	if (kport == IP_NULL)
 		panic("ipc_task_init");
 
+	nport = ipc_port_alloc_kernel();
+	if (nport == IP_NULL)
+		panic("ipc_task_init");
+
 	itk_lock_init(task);
 	task->itk_self = kport;
+	task->itk_nself = nport;
 	task->itk_sself = ipc_port_make_send(kport);
 	task->itk_space = space;
 	space->is_fast = FALSE;
@@ -192,11 +198,15 @@ ipc_task_enable(
 	task_t		task)
 {
 	ipc_port_t kport;
+	ipc_port_t nport;
 
 	itk_lock(task);
 	kport = task->itk_self;
 	if (kport != IP_NULL)
 		ipc_kobject_set(kport, (ipc_kobject_t) task, IKOT_TASK);
+	nport = task->itk_nself;
+	if (nport != IP_NULL)
+		ipc_kobject_set(nport, (ipc_kobject_t) task, IKOT_TASK_NAME);
 	itk_unlock(task);
 }
 
@@ -213,11 +223,15 @@ ipc_task_disable(
 	task_t		task)
 {
 	ipc_port_t kport;
+	ipc_port_t nport;
 
 	itk_lock(task);
 	kport = task->itk_self;
 	if (kport != IP_NULL)
 		ipc_kobject_set(kport, IKO_NULL, IKOT_NONE);
+	nport = task->itk_nself;
+	if (nport != IP_NULL)
+		ipc_kobject_set(nport, IKO_NULL, IKOT_NONE);
 	itk_unlock(task);
 }
 
@@ -235,6 +249,7 @@ ipc_task_terminate(
 	task_t		task)
 {
 	ipc_port_t kport;
+	ipc_port_t nport;
 	int i;
 
 	itk_lock(task);
@@ -245,8 +260,12 @@ ipc_task_terminate(
 		itk_unlock(task);
 		return;
 	}
-
 	task->itk_self = IP_NULL;
+
+	nport = task->itk_nself;
+	assert(nport != IP_NULL);
+	task->itk_nself = IP_NULL;
+
 	itk_unlock(task);
 
 	/* release the naked send rights */
@@ -273,15 +292,18 @@ ipc_task_terminate(
 	ipc_port_release_send(task->wired_ledger_port);
 	ipc_port_release_send(task->paged_ledger_port);
 
-	/* destroy the kernel port */
+	/* destroy the kernel ports */
 	ipc_port_dealloc_kernel(kport);
+	ipc_port_dealloc_kernel(nport);
 }
 
 /*
  *	Routine:	ipc_task_reset
  *	Purpose:
  *		Reset a task's IPC state to protect it when
- *		it enters an elevated security context.
+ *		it enters an elevated security context. The
+ *		task name port can remain the same - since
+ *		it represents no specific privilege.
  *	Conditions:
  *		Nothing locked.  The task must be suspended.
  *		(Or the current thread must be in the task.)
@@ -701,44 +723,45 @@ task_get_special_port(
 	int		which,
 	ipc_port_t	*portp)
 {
-	ipc_port_t *whichp;
 	ipc_port_t port;
 
 	if (task == TASK_NULL)
 		return KERN_INVALID_ARGUMENT;
 
+	itk_lock(task);
+	if (task->itk_self == IP_NULL) {
+		itk_unlock(task);
+		return KERN_FAILURE;
+	}
+
 	switch (which) {
 	    case TASK_KERNEL_PORT:
-		whichp = &task->itk_sself;
+		port = ipc_port_copy_send(task->itk_sself);
+		break;
+
+	    case TASK_NAME_PORT:
+		port = ipc_port_make_send(task->itk_nself);
 		break;
 
 	    case TASK_HOST_PORT:
-		whichp = &task->itk_host;
+		port = ipc_port_copy_send(task->itk_host);
 		break;
 
 	    case TASK_BOOTSTRAP_PORT:
-		whichp = &task->itk_bootstrap;
+		port = ipc_port_copy_send(task->itk_bootstrap);
 		break;
 
             case TASK_WIRED_LEDGER_PORT:
-                whichp = &task->wired_ledger_port;
+		port = ipc_port_copy_send(task->wired_ledger_port);
                 break;
 
             case TASK_PAGED_LEDGER_PORT:
-                whichp = &task->paged_ledger_port;
+		port = ipc_port_copy_send(task->paged_ledger_port);
                 break;
                     
 	    default:
 		return KERN_INVALID_ARGUMENT;
 	}
-
-	itk_lock(task);
-	if (task->itk_self == IP_NULL) {
-		itk_unlock(task);
-		return KERN_FAILURE;
-	}
-
-	port = ipc_port_copy_send(*whichp);
 	itk_unlock(task);
 
 	*portp = port;
@@ -1020,6 +1043,39 @@ convert_port_to_task(
 	return (task);
 }
 
+/*
+ *	Routine:	convert_port_to_task_name
+ *	Purpose:
+ *		Convert from a port to a task name.
+ *		Doesn't consume the port ref; produces a task name ref,
+ *		which may be null.
+ *	Conditions:
+ *		Nothing locked.
+ */
+task_name_t
+convert_port_to_task_name(
+	ipc_port_t		port)
+{
+	task_name_t		task = TASK_NULL;
+
+	if (IP_VALID(port)) {
+		ip_lock(port);
+
+		if (	ip_active(port)					&&
+				(ip_kotype(port) == IKOT_TASK	||
+				 ip_kotype(port) == IKOT_TASK_NAME)) {
+			task = (task_name_t)port->ip_kobject;
+			assert(task != TASK_NAME_NULL);
+
+			task_reference_internal(task);
+		}
+
+		ip_unlock(port);
+	}
+
+	return (task);
+}
+
 /*
  *	Routine:	convert_port_to_space
  *	Purpose:
@@ -1199,6 +1255,33 @@ convert_task_to_port(
 	return port;
 }
 
+/*
+ *	Routine:	convert_task_name_to_port
+ *	Purpose:
+ *		Convert from a task name ref to a port.
+ *		Consumes a task name ref; produces a naked send right
+ *		which may be invalid.  
+ *	Conditions:
+ *		Nothing locked.
+ */
+
+ipc_port_t
+convert_task_name_to_port(
+	task_name_t		task_name)
+{
+	ipc_port_t port;
+
+	itk_lock(task_name);
+	if (task_name->itk_nself != IP_NULL)
+		port = ipc_port_make_send(task_name->itk_nself);
+	else
+		port = IP_NULL;
+	itk_unlock(task_name);
+
+	task_name_deallocate(task_name);
+	return port;
+}
+
 /*
  *	Routine:	convert_thread_to_port
  *	Purpose:
diff --git a/osfmk/kern/ipc_tt.h b/osfmk/kern/ipc_tt.h
index 76f46d5e8..d5cef3e06 100644
--- a/osfmk/kern/ipc_tt.h
+++ b/osfmk/kern/ipc_tt.h
@@ -113,6 +113,10 @@ extern ipc_port_t retrieve_task_self_fast(
 extern ipc_port_t retrieve_thread_self_fast(
 	thread_t	thread);
 
+/* Convert from a port to a task name */
+extern task_name_t convert_port_to_task_name(
+	ipc_port_t	port);
+
 /* Convert from a port to a task */
 extern task_t convert_port_to_task(
 	ipc_port_t	port);
@@ -145,6 +149,10 @@ extern thread_t	port_name_to_thread(
 extern ipc_port_t convert_task_to_port(
 	task_t			task);
 
+/* Convert from a task name to a port */
+extern ipc_port_t convert_task_name_to_port(
+	task_name_t		task_name);
+
 /* Convert from a thread to a port */
 extern ipc_port_t	convert_thread_to_port(
 	thread_t		thread);
diff --git a/osfmk/kern/kalloc.c b/osfmk/kern/kalloc.c
index 3be39868f..79138f76b 100644
--- a/osfmk/kern/kalloc.c
+++ b/osfmk/kern/kalloc.c
@@ -86,10 +86,12 @@ vm_map_t kalloc_map;
 vm_size_t kalloc_map_size = 16 * 1024 * 1024;
 vm_size_t kalloc_max;
 vm_size_t kalloc_max_prerounded;
+vm_size_t kalloc_kernmap_size;	/* size of kallocs that can come from kernel map */
 
 unsigned int kalloc_large_inuse;
 vm_size_t    kalloc_large_total;
 vm_size_t    kalloc_large_max;
+vm_size_t    kalloc_largest_allocated = 0;
 
 /*
  *	All allocations of size less than kalloc_max are rounded to the
@@ -104,6 +106,8 @@ vm_size_t    kalloc_large_max;
  *	It represents the first power of two for which no zone exists.
  *	kalloc_max_prerounded is the smallest allocation size, before
  *	rounding, for which no zone exists.
+ *  Also if the allocation size is more than kalloc_kernmap_size 
+ *  then allocate from kernel map rather than kalloc_map.
  */
 
 int first_k_zone = -1;
@@ -195,6 +199,8 @@ kalloc_init(
 	else
 		kalloc_max = PAGE_SIZE;
 	kalloc_max_prerounded = kalloc_max / 2 + 1;
+	/* size it to be more than 16 times kalloc_max (256k) for allocations from kernel map */
+	kalloc_kernmap_size = (kalloc_max * 16) + 1;
 
 	/*
 	 *	Allocate a zone for each size we are going to handle.
@@ -221,6 +227,7 @@ kalloc_canblock(
 {
 	register int zindex;
 	register vm_size_t allocsize;
+	vm_map_t alloc_map = VM_MAP_NULL;
 
 	/*
 	 * If size is too large for a zone, then use kmem_alloc.
@@ -235,7 +242,16 @@ kalloc_canblock(
 		if (!canblock) {
 		  return(0);
 		}
-		if (kmem_alloc(kalloc_map, (vm_offset_t *)&addr, size) != KERN_SUCCESS)
+
+		if (size >=  kalloc_kernmap_size) {
+			alloc_map = kernel_map;
+
+			if (size > kalloc_largest_allocated)
+			        kalloc_largest_allocated = size;
+		} else
+			alloc_map = kalloc_map;
+
+		if (kmem_alloc(alloc_map, (vm_offset_t *)&addr, size) != KERN_SUCCESS) 
 			addr = 0;
 
 		if (addr) {
@@ -287,6 +303,7 @@ krealloc(
 	register int zindex;
 	register vm_size_t allocsize;
 	void *naddr;
+	vm_map_t alloc_map = VM_MAP_NULL;
 
 	/* can only be used for increasing allocation size */
 
@@ -305,11 +322,16 @@ krealloc(
 	/* if old block was kmem_alloc'd, then use kmem_realloc if necessary */
 
 	if (old_size >= kalloc_max_prerounded) {
+		if (old_size >=  kalloc_kernmap_size) 
+			alloc_map = kernel_map;
+		else
+			alloc_map = kalloc_map;
+
 		old_size = round_page(old_size);
 		new_size = round_page(new_size);
 		if (new_size > old_size) {
 
-			if (KERN_SUCCESS != kmem_realloc(kalloc_map, 
+			if (KERN_SUCCESS != kmem_realloc(alloc_map, 
 			    (vm_offset_t)*addrp, old_size,
 			    (vm_offset_t *)&naddr, new_size)) {
 				panic("krealloc: kmem_realloc");
@@ -320,7 +342,7 @@ krealloc(
 			*addrp = (void *) naddr;
 
 			/* kmem_realloc() doesn't free old page range. */
-			kmem_free(kalloc_map, (vm_offset_t)*addrp, old_size);
+			kmem_free(alloc_map, (vm_offset_t)*addrp, old_size);
 
 			kalloc_large_total += (new_size - old_size);
 
@@ -350,7 +372,11 @@ krealloc(
 
 	simple_unlock(lock);
 	if (new_size >= kalloc_max_prerounded) {
-		if (KERN_SUCCESS != kmem_alloc(kalloc_map, 
+		if (new_size >=  kalloc_kernmap_size) 
+			alloc_map = kernel_map;
+		else
+			alloc_map = kalloc_map;
+		if (KERN_SUCCESS != kmem_alloc(alloc_map, 
 		    (vm_offset_t *)&naddr, new_size)) {
 			panic("krealloc: kmem_alloc");
 			simple_lock(lock);
@@ -425,11 +451,38 @@ kfree(
 {
 	register int zindex;
 	register vm_size_t freesize;
+	vm_map_t alloc_map = VM_MAP_NULL;
 
 	/* if size was too large for a zone, then use kmem_free */
 
 	if (size >= kalloc_max_prerounded) {
-		kmem_free(kalloc_map, (vm_offset_t)data, size);
+	        if (size >=  kalloc_kernmap_size) {
+			alloc_map = kernel_map;
+
+			if (size > kalloc_largest_allocated)
+			        /*
+				 * work around double FREEs of small MALLOCs
+				 * this used to end up being a nop
+				 * since the pointer being freed from an
+				 * alloc backed by the zalloc world could
+				 * never show up in the kalloc_map... however,
+				 * the kernel_map is a different issue... since it
+				 * was released back into the zalloc pool, a pointer
+				 * would have gotten written over the 'size' that 
+				 * the MALLOC was retaining in the first 4 bytes of
+				 * the underlying allocation... that pointer ends up 
+				 * looking like a really big size on the 2nd FREE and
+				 * pushes the kfree into the kernel_map...  we
+				 * end up removing a ton of virutal space before we panic
+				 * this check causes us to ignore the kfree for a size
+				 * that must be 'bogus'... note that it might not be due
+				 * to the above scenario, but it would still be wrong and
+				 * cause serious damage.
+				 */
+			        return;
+		} else
+			alloc_map = kalloc_map;
+		kmem_free(alloc_map, (vm_offset_t)data, size);
 
 		kalloc_large_total -= size;
 		kalloc_large_inuse--;
diff --git a/osfmk/kern/locks.c b/osfmk/kern/locks.c
index 7acaf47b6..47f39aafb 100644
--- a/osfmk/kern/locks.c
+++ b/osfmk/kern/locks.c
@@ -353,6 +353,17 @@ lck_attr_setdebug(
 }
 
 
+/*
+ * Routine:	lck_attr_rw_shared_priority
+ */
+void
+lck_attr_rw_shared_priority(
+	lck_attr_t	*attr)
+{
+	(void)hw_atomic_or((uint32_t *)&attr->lck_attr_val, LCK_ATTR_RW_SHARED_PRIORITY);
+}
+
+
 /*
  * Routine:	lck_attr_free
  */
@@ -1041,7 +1052,7 @@ unsigned int
 usimple_lock_try_EXT(
 	lck_spin_t		*lock)
 {
-	lck_spin_try_lock(lock);
+	return(lck_spin_try_lock(lock));
 }
 
 void
diff --git a/osfmk/kern/locks.h b/osfmk/kern/locks.h
index e5e11691b..f2c9646f5 100644
--- a/osfmk/kern/locks.h
+++ b/osfmk/kern/locks.h
@@ -187,8 +187,10 @@ typedef	struct _lck_attr_ {
 
 extern lck_attr_t      LockDefaultLckAttr;
 
-#define 	LCK_ATTR_NONE		0
-#define	LCK_ATTR_DEBUG	0x1
+#define LCK_ATTR_NONE		0
+
+#define	LCK_ATTR_DEBUG				0x00000001
+#define	LCK_ATTR_RW_SHARED_PRIORITY	0x00010000
 
 #else
 typedef struct __lck_attr__ lck_attr_t;
@@ -207,6 +209,11 @@ extern	void			lck_attr_setdefault(
 extern	void			lck_attr_setdebug(
 									lck_attr_t		*attr);
 
+#ifdef	XNU_KERNEL_PRIVATE
+extern	void			lck_attr_rw_shared_priority(
+									lck_attr_t		*attr);
+#endif
+
 extern	void			lck_attr_free(
 									lck_attr_t		*attr);
 
diff --git a/osfmk/kern/machine.c b/osfmk/kern/machine.c
index 043a29508..40aa5c0b2 100644
--- a/osfmk/kern/machine.c
+++ b/osfmk/kern/machine.c
@@ -89,6 +89,7 @@
 #include <kern/thread.h>
 
 #include <IOKit/IOHibernatePrivate.h>
+#include <IOKit/IOPlatformExpert.h>
 
 /*
  *	Exported variables:
@@ -142,7 +143,12 @@ host_reboot(
 		return (KERN_SUCCESS);
 	}
 
-	halt_all_cpus(!(options & HOST_REBOOT_HALT));
+    if (options & HOST_REBOOT_UPSDELAY) {
+        // UPS power cutoff path
+        PEHaltRestart( kPEUPSDelayHaltCPU );
+    } else {
+       halt_all_cpus(!(options & HOST_REBOOT_HALT));
+    }
 
 	return (KERN_SUCCESS);
 }
diff --git a/osfmk/kern/misc_protos.h b/osfmk/kern/misc_protos.h
index d0779bcb8..b3663ec64 100644
--- a/osfmk/kern/misc_protos.h
+++ b/osfmk/kern/misc_protos.h
@@ -95,6 +95,10 @@ extern int copyoutmsg(
 	user_addr_t     user_addr,
 	mach_msg_size_t nbytes);
 
+/* Invalidate copy window(s) cache */
+extern void	inval_copy_windows(thread_t);
+
+
 extern int sscanf(const char *input, const char *fmt, ...);
 
 extern integer_t sprintf(char *buf, const char *fmt, ...);
@@ -103,7 +107,7 @@ extern void printf(const char *format, ...);
 
 extern void dbugprintf(const char *format, ...);
 
-extern void kdp_printf(const char *format, ...);
+extern void kdb_printf(const char *format, ...);
 
 extern void printf_init(void);
 
diff --git a/osfmk/kern/page_decrypt.c b/osfmk/kern/page_decrypt.c
new file mode 100644
index 000000000..43529e8ec
--- /dev/null
+++ b/osfmk/kern/page_decrypt.c
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+
+#include <kern/page_decrypt.h>
+#include <kern/task.h>
+#include <machine/commpage.h>
+
+/*#include <sys/kernel.h> */
+extern int hz;			/* system clock's frequency */
+extern void*   dsmos_blobs[];
+extern int     dsmos_blob_count;
+
+/* #include <sys/proc.h> */
+extern int	tsleep(void *chan, int pri, const char *wmesg, int timo);
+
+/* #include <sys/param.h> */
+#define	PZERO	22		/* No longer magic, shouldn't be here.  XXX */
+
+static int _dsmos_wait_for_callback(const void*,void*);
+
+static dsmos_page_transform_hook_t dsmos_hook = _dsmos_wait_for_callback;
+
+int
+_dsmos_wait_for_callback(const void* from, void *to)
+{
+/*	printf("%s\n", __FUNCTION__); */
+	while ( (dsmos_hook == NULL) || (dsmos_hook == _dsmos_wait_for_callback) )
+		tsleep(&dsmos_hook, PZERO, "dsmos", hz / 10);
+
+	return (*dsmos_hook) (from, to);
+}
+
+void
+dsmos_page_transform_hook(dsmos_page_transform_hook_t hook,
+			  void (*commpage_setup_dsmos_blob)(void**, int))
+{
+#ifdef i386
+	/* finish initializing the commpage here */
+	(*commpage_setup_dsmos_blob)(dsmos_blobs, dsmos_blob_count);
+#endif
+
+	/* set the hook now - new callers will run with it */
+	dsmos_hook = hook;
+}
+
+int
+dsmos_page_transform(const void* from, void* to)
+{
+/*	printf("%s\n", __FUNCTION__); */
+	if (dsmos_hook == NULL)
+		return KERN_FAILURE;
+	return (*dsmos_hook) (from, to);
+}
+
diff --git a/osfmk/kern/page_decrypt.h b/osfmk/kern/page_decrypt.h
new file mode 100644
index 000000000..2f896f313
--- /dev/null
+++ b/osfmk/kern/page_decrypt.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _KERN_PAGE_DECRYPT_H
+#define _KERN_PAGE_DECRYPT_H
+
+typedef	int	  (*dsmos_page_transform_hook_t) (const void*,void*);
+extern	void	dsmos_page_transform_hook(dsmos_page_transform_hook_t hook,
+					  void (*commpage_setup_dsmos_blob)(void**, int));	/* exported */
+
+extern	int		dsmos_page_transform(const void*,void*);
+
+#endif	/* _KERN_PAGE_DECRYPT_H */
diff --git a/osfmk/ppc/pms.c b/osfmk/kern/pms.c
similarity index 82%
rename from osfmk/ppc/pms.c
rename to osfmk/kern/pms.c
index 70f3dc516..4515de1c1 100644
--- a/osfmk/ppc/pms.c
+++ b/osfmk/kern/pms.c
@@ -25,19 +25,20 @@
  * 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-#include <ppc/machine_routines.h>
-#include <ppc/machine_cpu.h>
-#include <ppc/exception.h>
-#include <ppc/misc_protos.h>
-#include <ppc/Firmware.h>
-#include <ppc/pmap.h>
-#include <ppc/proc_reg.h>
-#include <ppc/pms.h>
-#include <ppc/savearea.h>
-#include <ppc/exception.h>
+#include <machine/machine_routines.h>
+#include <machine/machine_cpu.h>
+#ifdef __ppc__
+# include <ppc/exception.h>
+# include <ppc/misc_protos.h>
+#else
+# include <i386/cpu_data.h>
+# include <i386/misc_protos.h>
+#endif
+#include <machine/pmap.h>
+#include <kern/pms.h>
 #include <kern/processor.h>
-
-extern int real_ncpus;
+#include <kern/kalloc.h>
+#include <vm/vm_protos.h>
 
 static uint32_t pmsSyncrolator = 0;					/* Only one control operation at a time please */
 uint32_t pmsBroadcastWait = 0;						/* Number of outstanding broadcasts */
@@ -68,6 +69,15 @@ pmsSetFunc_t pmsFuncTab[pmsSetFuncMax] = {0};		/* This is the function index tab
 pmsQueryFunc_t pmsQueryFunc = 0;					/* Pointer to pmsQuery function */
 uint32_t pmsPlatformData = 0;						/* Data provided by and passed to platform functions */
 
+#ifdef __ppc__
+# define PER_PROC_INFO		struct per_proc_info
+# define GET_PER_PROC_INFO()	getPerProc()
+#else
+# define PER_PROC_INFO 		cpu_data_t
+# define GET_PER_PROC_INFO()	current_cpu_datap()
+#endif
+
+
 
 /*
  *	Do any initialization needed
@@ -80,6 +90,8 @@ void pmsInit(void) {
 	simple_lock_init(&pmsBuildLock, 0);				/* Initialize the build lock */
 	for(i = 0; i < pmsMaxStates; i++) pmsCtls.pmsDefs[i] = &pmsDummy;	/* Initialize the table to dummy steps */
 
+	pmsCPUMachineInit();
+
 	return;
 }
 
@@ -139,10 +151,10 @@ void pmsPark(void) {
 
 void pmsDown(void) {
 
-	struct per_proc_info *pp;
+	PER_PROC_INFO *pp;
 	uint32_t nstate;
 	
-	pp = getPerProc();								/* Get our per_proc */
+	pp = GET_PER_PROC_INFO();								/* Get our per_proc */
 	
 	if(!pmsInstalled || pp->pms.pmsState == pmsParked) return;		/* No stepping if parked or not installed */
 	
@@ -158,26 +170,83 @@ void pmsDown(void) {
  *
  *	Interrupts must be off...
  */
+
+int pmsStepIdleSneaks;
+int pmsStepIdleTries;
  
 void pmsStep(int timer) {
 
-	struct per_proc_info *pp;
-	uint32_t nstate;
-	int dir;
-	
-	pp = getPerProc();								/* Get our per_proc */
-
-	if(!pmsInstalled || pp->pms.pmsState == pmsParked) return;	/* No stepping if parked or not installed */
-	
-	nstate = pmsCtls.pmsDefs[pp->pms.pmsState]->pmsNext;	/* Assume a normal step */
-	dir = 1;										/* A normal step is a step up */
-	
-	if(timer && (pmsCtls.pmsDefs[pp->pms.pmsState]->pmsSetCmd == pmsDelay)) {	/* If the timer expired and we are in a delay step, use the delay branch */
-		nstate = pmsCtls.pmsDefs[pp->pms.pmsState]->pmsTDelay;	/* Get the delayed step */
-		dir = 0;									/* Delayed steps are a step down for accounting purposes. */
+	PER_PROC_INFO	*pp;
+	uint32_t	nstate;
+	uint32_t	tstate;
+	uint32_t	pkgstate;
+	int		dir;
+	int		i;
+	
+	pp = GET_PER_PROC_INFO();								/* Get our per_proc */
+
+	if(!pmsInstalled || pp->pms.pmsState == pmsParked)
+		return;	/* No stepping if parked or not installed */
+	
+	/*
+	 * Assume a normal step.
+	 */
+	nstate = pmsCtls.pmsDefs[pp->pms.pmsState]->pmsNext;
+
+	/*
+	 * If we are idling and being asked to step up, check to see whether
+	 * the package we're in is already at a non-idle power state.  If so,
+	 * attempt to work out what state that is, and go there directly to
+	 * avoid wasting time ramping up.
+	 */
+	if ((pp->pms.pmsState == pmsIdle)
+	    && ((pkgstate = pmsCPUPackageQuery()) != ~(uint32_t)0)) {
+		/*
+		 * Search forward through the stepper program,
+		 * avoid looping for too long.
+		 */
+		tstate = nstate;
+		pmsStepIdleTries++;
+		for (i = 0; i < 32; i++) {
+		    /*
+		     * Compare command with current package state
+		     */
+		    if ((pmsCtls.pmsDefs[tstate]->pmsSetCmd & pmsCPU) == pkgstate) {
+			nstate = tstate;
+			pmsStepIdleSneaks++;
+			break;
+		    }
+
+		    /*
+		     * Advance to the next step in the program.
+		     */
+		    if (pmsCtls.pmsDefs[tstate]->pmsNext == tstate)
+			break;	/* infinite loop */
+		    tstate = pmsCtls.pmsDefs[tstate]->pmsNext;
+		}
+        }
+
+	/*
+	 * Default to a step up.
+	 */
+	dir = 1;
+
+	/*
+	 * If we are stepping as a consequence of timer expiry, select the
+	 * alternate exit path and note this as downward step for accounting
+	 * purposes.
+	 */
+	if (timer
+	    && (pmsCtls.pmsDefs[pp->pms.pmsState]->pmsSetCmd == pmsDelay)) {
+	    nstate = pmsCtls.pmsDefs[pp->pms.pmsState]->pmsTDelay;
+
+	    /*
+	     * Delayed steps are a step down for accounting purposes.
+	     */
+	    dir = 0;
 	}
 
-	pmsSetStep(nstate, dir);						/* Step to it  */
+	pmsSetStep(nstate, dir);
 	return;
 }
 
@@ -192,13 +261,13 @@ void pmsStep(int timer) {
 
 void pmsSetStep(uint32_t nstep, int dir) {
 
-	struct per_proc_info *pp;
-	uint32_t pstate, ret, nCSetCmd, mCSetCmd;
+	PER_PROC_INFO *pp;
+	uint32_t pstate, nCSetCmd, mCSetCmd;
 	pmsDef *pnstate, *pcstate;
-	uint64_t tb, nt, dur;
-	int cpu, frompark;
+	uint64_t tb, dur;
+	int cpu;
 
-	pp = getPerProc();								/* Get our per_proc */
+	pp = GET_PER_PROC_INFO();								/* Get our per_proc */
 	cpu = cpu_number();								/* Get our processor */
 	
 	while(1) {										/* Keep stepping until we get a delay */
@@ -213,7 +282,7 @@ void pmsSetStep(uint32_t nstep, int dir) {
 			pp->pms.pmsStamp = tb;					/* Show transition now */
 			pp->pms.pmsPop = HalfwayToForever;		/* Set the pop way into the future */
 			pp->pms.pmsState = pmsParked;			/* Make sure we are parked */
-			setTimerReq();							/* Cancel our timer if going */
+			etimer_resync_deadlines();							/* Cancel our timer if going */
 			return;
 		}
 
@@ -258,7 +327,7 @@ void pmsSetStep(uint32_t nstep, int dir) {
  
 		if((pnstate->pmsSetCmd == pmsDelay) 
 			|| (!(pp->pms.pmsCSetCmd & pmsSync) && (pnstate->pmsLimit != 0))) {	/* Is this not syncronous and a non-zero delay or a delayed step? */
-			setTimerReq();							/* Start the timers ticking */
+			etimer_resync_deadlines();							/* Start the timers ticking */
 			break;									/* We've stepped as far as we're going to... */
 		}
 		
@@ -276,18 +345,16 @@ void pmsSetStep(uint32_t nstep, int dir) {
  
 void pmsRunLocal(uint32_t nstep) {
 
-	struct per_proc_info *pp;
-	uint32_t cstate, ret, lastState;
-	pmsDef *pnstate, *pcstate;
-	uint64_t tb, nt, dur;
-	int cpu, i, j;
+	PER_PROC_INFO *pp;
+	uint32_t lastState;
+	int cpu, i;
 	boolean_t	intr;
 
 	if(!pmsInstalled) return;						/* Ignore this if no step programs installed... */
 
 	intr = ml_set_interrupts_enabled(FALSE);		/* No interruptions in here */
 
-	pp = getPerProc();								/* Get our per_proc */
+	pp = GET_PER_PROC_INFO();								/* Get our per_proc */
 
 	if(nstep == pmsStartUp) {						/* Should we start up? */
 		pmsCPUInit();								/* Get us up to full with high voltage and park */
@@ -316,67 +383,70 @@ void pmsRunLocal(uint32_t nstep) {
 
 /*
  *	Control the Power Management Stepper.
- *	Called from user state by the superuser via a ppc system call.
+ *	Called from user state by the superuser.
  *	Interruptions disabled.
  *
  */
+kern_return_t pmsControl(uint32_t request, user_addr_t reqaddr, uint32_t reqsize) {
 
-int pmsCntrl(struct savearea *save) {
-
-	uint32_t request, nstep, reqsize, result, presult;
+	uint32_t nstep, result, presult;
 	int ret, cpu;
 	kern_return_t kret;
 	pmsDef *ndefs;
-	struct per_proc_info *pp;
+	PER_PROC_INFO *pp;
 
-	pp = getPerProc();								/* Get our per_proc */
+	pp = GET_PER_PROC_INFO();								/* Get our per_proc */
 	cpu = cpu_number();								/* Get our processor */
 	
 	if(!is_suser()) {								/* We are better than most, */
-		save->save_r3 = KERN_FAILURE;				/* so we will only talk to the superuser. */
-		return 1;									/* Turn up our noses, say "harrumph," and walk away... */
+		return KERN_FAILURE;				/* so we will only talk to the superuser. */
 	}
-	
-	if(save->save_r3 >= pmsCFree) {					/* Can we understand the request? */
-		save->save_r3 = KERN_INVALID_ARGUMENT;		/* What language are these guys talking in, anyway? */
-		return 1;									/* Cock head like a confused puppy and run away... */
+
+	if(request >= pmsCFree) {					/* Can we understand the request? */
+		return KERN_INVALID_ARGUMENT;		/* What language are these guys talking in, anyway? */
 	}
 	
-	request = (int)save->save_r3;					/* Remember the request */
-	reqsize = (uint32_t)save->save_r5;				/* Get the size of the config table */
-
 	if(request == pmsCQuery) {						/* Are we just checking? */
-		result = pmsCPUquery() & pmsCPU;			/* Get the processor data and make sure there is no slop */
+		result = pmsCPUQuery() & pmsCPU;			/* Get the processor data and make sure there is no slop */
 		presult = 0;								/* Assume nothing */
 		if((uint32_t)pmsQueryFunc) presult = pmsQueryFunc(cpu, pmsPlatformData);	/* Go get the platform state */
 		result = result | (presult & (pmsXClk | pmsVoltage | pmsPowerID));	/* Merge the platform state with no slop */
-		save->save_r3 = result;						/* Tell 'em... */
-		return 1;
+		return result;						/* Tell 'em... */
 	}
 	
 	if(request == pmsCExperimental) {				/* Enter experimental mode? */
 	
 		if(pmsInstalled || (pmsExperimental & 1)) {	/* Are we already running or in experimental? */
-			save->save_r3 = KERN_FAILURE;			/* Fail, since we are already running */
-			return 1;
+			return KERN_FAILURE;			/* Fail, since we are already running */
 		}
 		
 		pmsExperimental |= 1;						/* Flip us into experimental but don't change other flags */
 		
 		pmsCPUConf();								/* Configure for this machine */
 		pmsStart();									/* Start stepping */
-		save->save_r3 = KERN_SUCCESS;				/* We are victorious... */
-		return 1;
+		return KERN_SUCCESS;				/* We are victorious... */
 	
 	}
 
 	if(request == pmsCCnfg) {						/* Do some up-front checking before we commit to doing this */
 		if((reqsize > (pmsMaxStates * sizeof(pmsDef))) || (reqsize < (pmsFree * sizeof(pmsDef)))) {	/* Check that the size is reasonable */
-			save->save_r3 = KERN_NO_SPACE;			/* Tell them that they messed up */
-			return 1;								/* l8r... */
+			return KERN_NO_SPACE;			/* Tell them that they messed up */
 		}
 	}
 
+	if (request == pmsGCtls) {
+		if (reqsize != sizeof(pmsCtls))
+			return(KERN_FAILURE);
+		ret = copyout(&pmsCtls, reqaddr, reqsize);
+		return kret;
+	}
+			
+	if (request == pmsGStats) {
+		if (reqsize != sizeof(pmsStatsd))		/* request size is fixed */
+			return KERN_FAILURE;
+		ret = copyout(&pmsStatsd, reqaddr, reqsize);
+		return kret;							/* All done... */
+	}
 
 /*
  *	We are committed after here.  If there are any errors detected, we shouldn't die, but we
@@ -387,10 +457,8 @@ int pmsCntrl(struct savearea *save) {
  */
  		
 	if(!hw_compare_and_store(0, 1, &pmsSyncrolator)) {	/* Are we already doing this? */
-		save->save_r3 = KERN_RESOURCE_SHORTAGE;		/* Tell them that we are already busy and to try again */
-		return 1;									/* G'wan away and don't bother me... */
+		return KERN_RESOURCE_SHORTAGE;		/* Tell them that we are already busy and to try again */
 	}
-	save->save_r3 = KERN_SUCCESS;					/* Assume success */
 
 //	NOTE:  We will block in the following code until everyone has finished the prepare
 
@@ -398,7 +466,7 @@ int pmsCntrl(struct savearea *save) {
 	
 	if(request == pmsCPark) {						/* Is all we're supposed to do park? */
 		pmsSyncrolator = 0;							/* Free us up */
-		return 1;									/* Well, then we're done... */
+		return KERN_SUCCESS;									/* Well, then we're done... */
 	}
 	
 	switch(request) {								/* Select the routine */
@@ -418,25 +486,22 @@ int pmsCntrl(struct savearea *save) {
 		case pmsCCnfg:								/* Loads new stepper program */
 			
 			if(!(ndefs = (pmsDef *)kalloc(reqsize))) {	/* Get memory for the whole thing */
-				save->save_r3 = KERN_INVALID_ADDRESS;	/* Return invalid address */
 				pmsSyncrolator = 0;					/* Free us up */
-				return 1;							/* All done... */
+				return KERN_INVALID_ADDRESS;			/* All done... */
 			}
 			
-			ret = copyin((user_addr_t)((unsigned int)(save->save_r4)), (void *)ndefs, reqsize);	/* Get the new config table */
+			ret = copyin(reqaddr, (void *)ndefs, reqsize);	/* Get the new config table */
 			if(ret) {								/* Hmmm, something went wrong with the copyin */
-				save->save_r3 = KERN_INVALID_ADDRESS;	/* Return invalid address */
 				kfree((vm_offset_t)ndefs, reqsize);	/* Free up the copied in data */
 				pmsSyncrolator = 0;					/* Free us up */
-				return 1;							/* All done... */
+				return KERN_INVALID_ADDRESS;				/* All done... */
 			}
 
 			kret = pmsBuild(ndefs, reqsize, 0, 0, 0);	/* Go build and replace the tables.  Make sure we keep the old platform stuff */
 			if(kret) {								/* Hmmm, something went wrong with the compilation */
-				save->save_r3 = kret;				/* Pass back the passed back return code */
 				kfree((vm_offset_t)ndefs, reqsize);	/* Free up the copied in data */
 				pmsSyncrolator = 0;					/* Free us up */
-				return 1;							/* All done... */
+				return kret;							/* All done... */
 			}
 
 			nstep = pmsNormHigh;					/* Set the request */
@@ -449,46 +514,21 @@ int pmsCntrl(struct savearea *save) {
 
 	pmsRun(nstep);									/* Get everyone into step */
 	pmsSyncrolator = 0;								/* Free us up */
-	return 1;										/* All done... */
+	return KERN_SUCCESS;										/* All done... */
 
 }
 
 /*
  *	Broadcast a change to all processors including ourselves.
- *	This must transition before broadcasting because we may block and end up on a different processor.
- *
- *	This will block until all processors have transitioned, so
- *	obviously, this can block.
- *
- *	Called with interruptions disabled.
  *
+ *	Interruptions are disabled.
  */
  
 void pmsRun(uint32_t nstep) {
 
-	pmsRunLocal(nstep);								/* If we aren't parking (we are already parked), transition ourselves */
-	(void)cpu_broadcast(&pmsBroadcastWait, pmsRemote, nstep);	/* Tell everyone else to do it too */
-
-	return;
-	
+	pmsCPURun(nstep);
 }
 
-/*
- *	Receive a broadcast and react.
- *	This is called from the interprocessor signal handler.
- *	We wake up the initiator after we are finished.
- *
- */
-	
-void pmsRemote(uint32_t nstep) {
-
-	pmsRunLocal(nstep);								/* Go set the step */
-	if(!hw_atomic_sub(&pmsBroadcastWait, 1)) {		/* Drop the wait count */
-		thread_wakeup((event_t)&pmsBroadcastWait);	/* If we were the last, wake up the signaller */
-	}
-	return;
-}	
-
 
 /*
  *	Build the tables needed for the stepper.  This includes both the step definitions and the step control table.
@@ -531,7 +571,7 @@ kern_return_t pmsBuild(pmsDef *pd, uint32_t pdsize, pmsSetFunc_t *functab, uint3
 	steps = pdsize / sizeof(pmsDef);				/* Get the number of steps supplied */
 
 	if((steps >= pmsMaxStates) || (steps < pmsFree))	/* Complain if too big or too small */
-			return KERN_INVALID_ARGUMENT;			/* Squeak loudly!!! */
+		return KERN_INVALID_ARGUMENT;			/* Squeak loudly!!! */
 			
 	if((uint32_t)functab && (uint32_t)functab[0])	/* Verify that if they supplied a new function table, entry 0 is 0 */
 		return KERN_INVALID_ARGUMENT;				/* Fail because they didn't reserve entry 0 */
@@ -672,15 +712,15 @@ kern_return_t pmsBuild(pmsDef *pd, uint32_t pdsize, pmsSetFunc_t *functab, uint3
 		newpd[i].pmsTDelay  = pd[i].pmsTDelay;		/* Set the delayed setp */
 		pmsCtls.pmsDefs[i]  = &newpd[i];			/* Copy it in */
 	}
-	
+#ifdef __ppc__	
 	pmsCtlp = (uint32_t)&pmsCtls;					/* Point to the new pms table */
-	
+#endif
  	pmsInstalled = 1;								/* The stepper has been born or born again... */
 
 	simple_unlock(&pmsBuildLock);					/* Free play! */
 	(void)ml_set_interrupts_enabled(intr);			/* Interrupts back the way there were */
 
-	if((uint32_t)oldAlt) kfree((vm_offset_t)oldAlt, oldAltSize);	/* If we already had an alternate, free it */
+	if((uint32_t)oldAlt) kfree(oldAlt, oldAltSize);	/* If we already had an alternate, free it */
 
 	if(xdsply) kprintf("Stepper table installed\n");
 	
diff --git a/osfmk/kern/pms.h b/osfmk/kern/pms.h
new file mode 100644
index 000000000..1a4947e94
--- /dev/null
+++ b/osfmk/kern/pms.h
@@ -0,0 +1,197 @@
+/*
+ * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifdef KERNEL_PRIVATE
+
+#ifndef _KERN_PMS_H_
+#define _KERN_PMS_H_
+
+#define pmsMaxStates 64
+#define HalfwayToForever 0x7FFFFFFFFFFFFFFFULL
+#define century 790560000000000ULL
+
+typedef void (*pmsSetFunc_t)(uint32_t, uint32_t, uint32_t);	/* Function used to set hardware power state */
+typedef uint32_t (*pmsQueryFunc_t)(uint32_t, uint32_t);	/* Function used to query hardware power state */
+
+typedef struct pmsStat {
+	uint64_t	stTime[2];			/* Total time until switch to next step */
+	uint32_t	stCnt[2];			/* Number of times switched to next step */
+} pmsStat;
+
+typedef struct pmsDef {
+	uint64_t	pmsLimit;			/* Max time in this state in microseconds */
+	uint32_t	pmsStepID;			/* Unique ID for this step */
+	uint32_t	pmsSetCmd;			/* Command to select power state */
+#define pmsCngXClk  0x80000000		/* Change external clock */
+#define pmsXUnk   	0x7F			/* External clock unknown  */
+#define pmsXClk     0x7F000000		/* External clock frequency */
+#define pmsCngCPU   0x00800000		/* Change CPU parameters */
+#define pmsSync     0x00400000		/* Make changes synchronously, i.e., spin until delay finished */
+#define pmsMustCmp  0x00200000		/* Delay must complete before next change */
+#define pmsCPU      0x001F0000		/* CPU frequency */
+#define pmsCPUUnk	0x1F			/* CPU frequency unknown */
+#define pmsCngVolt  0x00008000		/* Change voltage */
+#define pmsVoltage  0x00007F00		/* Voltage */
+#define pmsVoltUnk	0x7F			/* Voltage unknown */
+#define pmsPowerID  0x000000FF		/* Identify power state to HW */
+
+/*	Special commands - various things */
+#define pmsDelay    0xFFFFFFFD		/* Delayed step, no processor or platform changes.  Timer expiration causes transition to pmsTDelay */
+#define pmsParkIt	0xFFFFFFFF		/* Enters the parked state.  No processor or platform changes.  Timers cancelled */
+#define pmsCInit	((pmsXUnk << 24) | (pmsCPUUnk << 16) | (pmsVoltUnk << 8))	/* Initial current set command value */
+/*	Note:  pmsSetFuncInd is an index into a table of function pointers and pmsSetFunc is the address
+ *	of a function.  Initially, when you create a step table, this field is set as an index into
+ *	a table of function addresses that gets passed as a parameter to pmsBuild.  When pmsBuild
+ *	internalizes the step and function tables, it converts the index to the function address.
+ */
+	union sf {
+		pmsSetFunc_t	pmsSetFunc;	/* Function used to set platform power state */
+		uint32_t	pmsSetFuncInd;	/* Index to function in function table */
+	} sf;
+
+	uint32_t	pmsDown;			/* Next state if going lower */
+	uint32_t	pmsNext;			/* Normal next state */
+	uint32_t	pmsTDelay;			/* State if command was pmsDelay and timer expired */
+} pmsDef;
+
+typedef struct pmsCtl {
+	pmsStat		(*pmsStats)[pmsMaxStates];	/* Pointer to statistics information, 0 if not enabled */
+	pmsDef		*pmsDefs[pmsMaxStates];	/* Indexed pointers to steps */
+} pmsCtl;
+
+/*
+ *	Note that this block is in the middle of the per_proc and the size (32 bytes)
+ *	can't be changed without moving it.
+ */
+
+typedef struct pmsd {
+	uint32_t	pmsState;			/* Current power management state */
+	uint32_t	pmsCSetCmd;			/* Current select command */
+	uint64_t	pmsPop;				/* Time of next step */
+	uint64_t	pmsStamp;			/* Time of transition to current state */
+	uint64_t	pmsTime;			/* Total time in this state */
+} pmsd;
+
+/*
+ *	Required power management step programs
+ */
+ 
+enum {
+	pmsIdle      = 0,				/* Power state in idle loop */
+	pmsNorm      = 1,				/* Normal step - usually low power */
+	pmsNormHigh  = 2,				/* Highest power in normal step */
+	pmsBoost     = 3,				/* Boost/overdrive step */
+	pmsLow       = 4,				/* Lowest non-idle power state, no transitions */
+	pmsHigh      = 5,				/* Power step for full on, no transitions */
+	pmsPrepCng   = 6,				/* Prepare for step table change */
+	pmsPrepSleep = 7,				/* Prepare for sleep */
+	pmsOverTemp  = 8,				/* Machine is too hot */
+	pmsEnterNorm = 9,				/* Enter into the normal step program */
+	pmsFree      = 10,				/* First available empty step */
+	pmsStartUp   = 0xFFFFFFFE,		/* Start stepping */
+	pmsParked    = 0xFFFFFFFF		/* Power parked - used when changing stepping table */
+};
+
+/*
+ *	Power Management Stepper Control requests
+ */
+ 
+enum {
+	pmsCPark = 0,					/* Parks the stepper */
+	pmsCStart = 1,					/* Starts normal steppping */
+	pmsCFLow = 2,					/* Forces low power */
+	pmsCFHigh = 3,					/* Forces high power */
+	pmsCCnfg = 4,					/* Loads new stepper program */
+	pmsCQuery = 5,					/* Query current step and state */
+	pmsCExperimental = 6,			/* Enter experimental mode */
+	pmsGCtls = 7,
+	pmsGStats = 8,
+	pmsCVID = 9,
+	pmsCFree = 10					/* Next control command to be assigned */
+};
+
+/*
+ *	User request control structure passed to sysctl
+ */
+typedef struct {
+	uint32_t	request;			/* stepper control request */
+	uint32_t	reqsize;			/* size of data */
+	void		*reqaddr;			/* read/write data buffer */
+} pmsctl_t;
+
+extern pmsCtl pmsCtls;				/* Power Management Stepper control */
+#ifdef __ppc__
+extern uint32_t pmsCtlp;
+#endif
+extern uint32_t pmsBroadcastWait;	/* Number of outstanding broadcasts */
+extern int pmsInstalled;
+extern int pmsExperimental;
+
+#define pmsSetFuncMax 32
+extern pmsSetFunc_t pmsFuncTab[pmsSetFuncMax];
+extern pmsQueryFunc_t pmsQueryFunc;
+extern uint32_t pmsPlatformData;
+
+#ifdef __ppc__
+extern int pmsCntrl(struct savearea *save);
+#endif
+extern kern_return_t pmsControl(uint32_t request, user_addr_t reqaddr, uint32_t reqsize);
+extern void pmsInit(void);
+extern void pmsStep(int timer);
+extern void pmsDown(void);
+extern void pmsSetStep(uint32_t nstep, int dir);
+extern void pmsRunLocal(uint32_t nstep);
+extern void pmsCPUSet(uint32_t sel);
+extern uint32_t pmsCPUQuery(void);
+extern uint32_t pmsCPUPackageQuery(void);
+extern void pmsCPUConf(void);
+extern void pmsCPUMachineInit(void);
+extern void pmsCPUInit(void);
+extern void pmsCPURun(uint32_t nstep);
+
+extern void pmsCPUYellowFlag(void);
+extern void pmsCPUGreenFlag(void);
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern kern_return_t pmsBuild(pmsDef *pd, uint32_t pdsize, pmsSetFunc_t *functab, uint32_t platformData, pmsQueryFunc_t queryFunc);
+extern void pmsRun(uint32_t nstep);
+extern void pmsPark(void);
+extern void pmsStart(void);
+# ifndef __ppc__
+extern kern_return_t pmsCPULoadVIDTable(uint16_t *tablep, int nstates);	/* i386 only */
+extern kern_return_t pmsCPUSetPStateLimit(uint32_t limit);
+# endif
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _KERN_PMS_H_ */
+#endif /* KERNEL_PRIVATE */
diff --git a/osfmk/kern/printf.c b/osfmk/kern/printf.c
index 9045fbcde..a90469d95 100644
--- a/osfmk/kern/printf.c
+++ b/osfmk/kern/printf.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -772,6 +772,8 @@ printf(const char *fmt, ...)
 	enable_preemption();
 }
 
+extern unsigned int disableSerialOuput;
+
 void
 consdebug_putc(
 	char c)
@@ -783,10 +785,9 @@ consdebug_putc(
 
 	debug_putc(c);
 
-#ifdef __ppc__
 	if (!console_is_serial())
-                PE_kputc(c);
-#endif
+		if (!disableSerialOuput)
+			PE_kputc(c);
 }
 
 void
diff --git a/osfmk/kern/sched_prim.c b/osfmk/kern/sched_prim.c
index f85cbcb65..acc37468c 100644
--- a/osfmk/kern/sched_prim.c
+++ b/osfmk/kern/sched_prim.c
@@ -76,6 +76,7 @@
 
 #include <machine/machine_routines.h>
 #include <machine/sched_param.h>
+#include <machine/machine_cpu.h>
 
 #include <kern/kern_types.h>
 #include <kern/clock.h>
@@ -102,9 +103,7 @@
 
 #include <sys/kdebug.h>
 
-#ifdef __ppc__
-#include <ppc/pms.h>
-#endif
+#include <kern/pms.h>
 
 #define		DEFAULT_PREEMPTION_RATE		100		/* (1/s) */
 int			default_preemption_rate = DEFAULT_PREEMPTION_RATE;
@@ -1575,10 +1574,8 @@ thread_dispatch(
 	 *	If blocked at a continuation, discard
 	 *	the stack.
 	 */
-#ifndef i386
     if (thread->continuation != NULL && thread->kernel_stack)
 		stack_free(thread);
-#endif
 
 	if (!(thread->state & TH_IDLE)) {
 		wake_lock(thread);
@@ -1659,6 +1656,16 @@ thread_block_reason(
 
 	s = splsched();
 
+#if 0
+#if	MACH_KDB
+	{
+		extern void db_chkpmgr(void);
+		db_chkpmgr();						/* (BRINGUP) See if pm config changed */
+
+	}
+#endif
+#endif
+
 	if (!(reason & AST_PREEMPT))
 		funnel_release_check(self, 2);
 
@@ -2494,8 +2501,10 @@ delay_idle(
 
 			timer_event((uint32_t)abstime, &processor->idle_thread->system_timer);
 		}
-		else
+		else {
+			cpu_pause();
 			abstime = mach_absolute_time();
+		}
 	}
 
 	timer_event((uint32_t)abstime, &self->system_timer);
@@ -2536,9 +2545,7 @@ idle_thread(void)
 
 	(void)splsched();			/* Turn interruptions off */
 
-#ifdef __ppc__
 	pmsDown();					/* Step power down.  Note: interruptions must be disabled for this call */
-#endif
 
 	while (	(*threadp == THREAD_NULL)				&&
 				(*gcount == 0) && (*lcount == 0)	) {
@@ -2562,9 +2569,7 @@ idle_thread(void)
 	pset = processor->processor_set;
 	simple_lock(&pset->sched_lock);
 
-#ifdef __ppc__
 	pmsStep(0);					/* Step up out of idle power, may start timer for next step */
-#endif
 
 	state = processor->state;
 	if (state == PROCESSOR_DISPATCHING) {
diff --git a/osfmk/kern/stack.c b/osfmk/kern/stack.c
index 840ff6743..f9fb6682c 100644
--- a/osfmk/kern/stack.c
+++ b/osfmk/kern/stack.c
@@ -99,7 +99,7 @@ stack_init(void)
 		panic("stack_init: kmem_suballoc");
 
 	map_addr = vm_map_min(stack_map);
-	if (vm_map_enter(stack_map, &map_addr, vm_map_round_page(PAGE_SIZE), 0, VM_FLAGS_FIXED,
+	if (vm_map_enter(stack_map, &map_addr, vm_map_round_page(PAGE_SIZE), 0, (VM_MAKE_TAG(VM_MEMORY_STACK) | VM_FLAGS_FIXED),
 						VM_OBJECT_NULL, 0, FALSE, VM_PROT_NONE, VM_PROT_NONE, VM_INHERIT_DEFAULT) != KERN_SUCCESS)
 		panic("stack_init: vm_map_enter");
 }
diff --git a/osfmk/kern/startup.c b/osfmk/kern/startup.c
index 9f51c90e3..7f5f71bb9 100644
--- a/osfmk/kern/startup.c
+++ b/osfmk/kern/startup.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -87,6 +87,7 @@
 #include <kern/xpr.h>
 #include <kern/zalloc.h>
 #include <kern/locks.h>
+#include <console/serial_protos.h>
 #include <vm/vm_shared_memory_server.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_init.h>
@@ -101,7 +102,6 @@
 #ifdef __ppc__
 #include <ppc/Firmware.h>
 #include <ppc/mappings.h>
-#include <ppc/serial_io.h>
 #endif
 
 static void		kernel_bootstrap_thread(void);
@@ -109,9 +109,15 @@ static void		kernel_bootstrap_thread(void);
 static void		load_context(
 					thread_t	thread);
 
+#ifdef i386
+extern void	cpu_window_init(int);
+#endif
+
+
 /*
  *	Running in virtual memory, on the interrupt stack.
  */
+
 void
 kernel_bootstrap(void)
 {
@@ -153,8 +159,8 @@ kernel_bootstrap(void)
 	 *	Create a kernel thread to execute the kernel bootstrap.
 	 */
 	result = kernel_thread_create((thread_continue_t)kernel_bootstrap_thread, NULL, MAXPRI_KERNEL, &thread);
-	if (result != KERN_SUCCESS)
-		panic("kernel_bootstrap");
+
+	if (result != KERN_SUCCESS) panic("kernel_bootstrap: result = %08X\n", result);
 
 	thread->state = TH_RUN;
 	thread_deallocate(thread);
@@ -217,10 +223,18 @@ kernel_bootstrap_thread(void)
 	
 	(void) spllo();		/* Allow interruptions */
 
-    /*
-     *	Fill in the comm area (mapped into every task address space.)
-     */
-    commpage_populate();
+	/*
+	 *	Fill in the comm area (mapped into every task address space.)
+	 */
+	commpage_populate();
+
+#ifdef i386
+	/*
+	 *	create and initialize a copy window
+	 * 	for processor 0
+	 */
+	cpu_window_init(0);
+#endif
 
 	/*
 	 *	Start the user bootstrap.
@@ -231,9 +245,7 @@ kernel_bootstrap_thread(void)
 	}
 #endif
 
-#if __ppc__
 	serial_keyboard_init();		/* Start serial keyboard if wanted */
-#endif
 
 	thread_bind(self, PROCESSOR_NULL);
 
diff --git a/osfmk/kern/syscall_sw.c b/osfmk/kern/syscall_sw.c
index ab2795321..4a8dd5e30 100644
--- a/osfmk/kern/syscall_sw.c
+++ b/osfmk/kern/syscall_sw.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -136,7 +136,7 @@ mach_trap_t	mach_trap_table[MACH_TRAP_TABLE_COUNT] = {
 /* 41 */	MACH_TRAP(init_process, 0, NULL, NULL),
 /* 42 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
 /* 43 */	MACH_TRAP(map_fd, 5, munge_wwwww, munge_ddddd),
-/* 44 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 44 */	MACH_TRAP(task_name_for_pid, 3, munge_www, munge_ddd),
 /* 45 */ 	MACH_TRAP(task_for_pid, 3, munge_www, munge_ddd),
 /* 46 */	MACH_TRAP(pid_for_task, 2, munge_ww,munge_dd),
 /* 47 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
@@ -188,7 +188,7 @@ mach_trap_t	mach_trap_table[MACH_TRAP_TABLE_COUNT] = {
 /* 92 */	MACH_TRAP(mk_timer_destroy_trap, 1, munge_w, munge_d),
 /* 93 */	MACH_TRAP(mk_timer_arm_trap, 3, munge_wl, munge_dd),
 /* 94 */	MACH_TRAP(mk_timer_cancel_trap, 2, munge_ww, munge_dd),		
-/* 95 */	MACH_TRAP(mk_timebase_info_trap, 5, munge_wwwww, munge_ddddd),		
+/* 95 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),		
 /* traps 64 - 95 reserved (debo) */
 /* 96 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
 /* 97 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
diff --git a/osfmk/kern/task.c b/osfmk/kern/task.c
index 62176da70..b800c7421 100644
--- a/osfmk/kern/task.c
+++ b/osfmk/kern/task.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -120,7 +120,7 @@
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>		/* for kernel_map, ipc_kernel_map */
 #include <vm/vm_pageout.h>
-#include <vm/vm_protos.h>	/* for vm_map_remove_commpage64 */
+#include <vm/vm_protos.h>	/* for vm_map_remove_commpage */
 
 #if	MACH_KDB
 #include <ddb/db_sym.h>
@@ -194,29 +194,44 @@ task_set_64bit(
 		task_t task,
 		boolean_t is64bit)
 {
-	if(is64bit) {
+        thread_t thread;
+
+	if (is64bit) {
+	        if (task_has_64BitAddr(task))
+		        return;
+
 		/* LP64todo - no task working set for 64-bit */
 		task_set_64BitAddr(task);
 		task_working_set_disable(task);
-		task->map->max_offset = MACH_VM_MAX_ADDRESS;
 	} else {
+	        if ( !task_has_64BitAddr(task))
+		        return;
+
 		/*
 		 * Deallocate all memory previously allocated
 		 * above the 32-bit address space, since it won't
 		 * be accessible anymore.
 		 */
 		/* LP64todo - make this clean */
-#ifdef __ppc__
-		vm_map_remove_commpage64(task->map);
+		vm_map_remove_commpage(task->map);
 		pmap_unmap_sharedpage(task->map->pmap);	/* Unmap commpage */
-#endif
 		(void) vm_map_remove(task->map,
 				     (vm_map_offset_t) VM_MAX_ADDRESS,
 				     MACH_VM_MAX_ADDRESS,
 				     VM_MAP_NO_FLAGS);
 		task_clear_64BitAddr(task);
-		task->map->max_offset = (vm_map_offset_t)VM_MAX_ADDRESS;
 	}
+	/* FIXME: On x86, the thread save state flavor can diverge from the
+	 * task's 64-bit feature flag due to the 32-bit/64-bit register save
+	 * state dichotomy. Since we can be pre-empted in this interval,
+	 * certain routines may observe the thread as being in an inconsistent
+	 * state with respect to its task's 64-bitness.
+	 */
+#ifdef __i386__
+	queue_iterate(&task->threads, thread, thread_t, task_threads) {
+	        machine_thread_switch_addrmode(thread, !is64bit);
+	}
+#endif
 }
 
 void
@@ -231,7 +246,7 @@ task_init(void)
 	/*
 	 * Create the kernel task as the first task.
 	 */
-	if (task_create_internal(TASK_NULL, FALSE, &kernel_task) != KERN_SUCCESS)
+	if (task_create_internal(TASK_NULL, FALSE, FALSE, &kernel_task) != KERN_SUCCESS)
 		panic("task_init\n");
 
 	vm_map_deallocate(kernel_task->map);
@@ -313,7 +328,7 @@ task_create(
 		return(KERN_INVALID_ARGUMENT);
 
 	return task_create_internal(
-	    		parent_task, inherit_memory, child_task);
+	    		parent_task, inherit_memory, task_has_64BitAddr(parent_task), child_task);
 }
 
 kern_return_t
@@ -337,7 +352,7 @@ host_security_create_task_token(
 		return(KERN_INVALID_SECURITY);
 
 	result = task_create_internal(
-			parent_task, inherit_memory, child_task);
+			parent_task, inherit_memory, task_has_64BitAddr(parent_task), child_task);
 
         if (result != KERN_SUCCESS)
                 return(result);
@@ -358,6 +373,7 @@ kern_return_t
 task_create_internal(
 	task_t		parent_task,
 	boolean_t	inherit_memory,
+	boolean_t	is_64bit,
 	task_t		*child_task)		/* OUT */
 {
 	task_t		new_task;
@@ -374,7 +390,7 @@ task_create_internal(
 	if (inherit_memory)
 		new_task->map = vm_map_fork(parent_task->map);
 	else
-		new_task->map = vm_map_create(pmap_create(0),
+		new_task->map = vm_map_create(pmap_create(0, is_64bit),
 					(vm_map_offset_t)(VM_MIN_ADDRESS),
 					(vm_map_offset_t)(VM_MAX_ADDRESS), TRUE);
 
@@ -407,6 +423,10 @@ task_create_internal(
 	new_task->bsd_info = 0;
 #endif /* MACH_BSD */
 
+#ifdef __i386__
+	new_task->i386_ldt = 0;
+#endif
+
 #ifdef __ppc__
 	if(BootProcInfo.pf.Available & pf64Bit) new_task->taskFeatures[0] |= tf64BitData;	/* If 64-bit machine, show we have 64-bit registers at least */
 #endif
@@ -452,6 +472,11 @@ task_create_internal(
 			convert_port_to_ledger(parent_task->paged_ledger_port));
 		if(task_has_64BitAddr(parent_task))
 			task_set_64BitAddr(new_task);
+
+#ifdef __i386__
+		if (inherit_memory && parent_task->i386_ldt)
+			new_task->i386_ldt = user_ldt_copy(parent_task->i386_ldt);
+#endif
 	}
 	else {
 		pset = &default_pset;
@@ -520,6 +545,19 @@ task_deallocate(
 	zfree(task_zone, task);
 }
 
+/*
+ *	task_name_deallocate:
+ *
+ *	Drop a reference on a task name.
+ */
+void
+task_name_deallocate(
+	task_name_t		task_name)
+{
+	return(task_deallocate((task_t)task_name));
+}
+
+
 /*
  *	task_terminate:
  *
@@ -633,10 +671,11 @@ task_terminate_internal(
 	ipc_space_destroy(task->itk_space);
 
 /* LP64todo - make this clean */
-#ifdef __ppc__
-	vm_map_remove_commpage64(task->map);
+	vm_map_remove_commpage(task->map);
 	pmap_unmap_sharedpage(task->map->pmap);		/* Unmap commpage */
-#endif
+
+	if (vm_map_has_4GB_pagezero(task->map))
+		vm_map_clear_4GB_pagezero(task->map);
 
 	/*
 	 * If the current thread is a member of the task
diff --git a/osfmk/kern/task.h b/osfmk/kern/task.h
index 15ee4ff39..5b986d897 100644
--- a/osfmk/kern/task.h
+++ b/osfmk/kern/task.h
@@ -97,6 +97,7 @@
 #include <mach/mach_param.h>
 #include <mach/task_info.h>
 #include <mach/exception_types.h>
+#include <machine/task.h>
 
 #include <kern/cpu_data.h>
 #include <kern/queue.h>
@@ -154,6 +155,7 @@ struct task {
 	/* IPC structures */
 	decl_mutex_data(,itk_lock_data)
 	struct ipc_port *itk_self;	/* not a right, doesn't hold ref */
+	struct ipc_port *itk_nself;	/* not a right, doesn't hold ref */
 	struct ipc_port *itk_sself;	/* a send right */
 	struct exception_action exc_actions[EXC_TYPES_COUNT];
 		 			/* a send right each valid element  */
@@ -175,6 +177,8 @@ struct task {
 	struct ipc_port *paged_ledger_port;
 	unsigned int	priv_flags;			/* privilege resource flags */
 #define VM_BACKING_STORE_PRIV	0x1
+
+	MACHINE_TASK
         
 	integer_t faults;              /* faults counter */
         integer_t pageins;             /* pageins counter */
@@ -267,6 +271,7 @@ extern kern_return_t	task_terminate_internal(
 extern kern_return_t	task_create_internal(
 							task_t		parent_task,
 							boolean_t	inherit_memory,
+							boolean_t	is_64bit,
 							task_t		*child_task);	/* OUT */
 
 extern kern_return_t	task_importance(
@@ -283,6 +288,7 @@ extern void		task_backing_store_privileged(
 extern void		task_working_set_disable(
 					task_t		task);
 
+
 /* Get number of activations in a task */
 extern int		get_task_numacts(
 					task_t		task);
@@ -310,6 +316,8 @@ extern task_t	kernel_task;
 extern void		task_deallocate(
 					task_t		task);
 
+extern void		task_name_deallocate(
+					task_name_t		task_name);
 __END_DECLS
 
 #endif	/* _KERN_TASK_H_ */
diff --git a/osfmk/kern/thread.c b/osfmk/kern/thread.c
index 1f1c4e49a..9d38b578f 100644
--- a/osfmk/kern/thread.c
+++ b/osfmk/kern/thread.c
@@ -780,11 +780,9 @@ kernel_thread_create(
 	pset_unlock(task->processor_set);
 	task_unlock(task);
 
-#if !defined(i386)
 	stack_alloc(thread);
 	assert(thread->kernel_stack != 0);
 	thread->reserved_stack = thread->kernel_stack;
-#endif /* !defined(i386) */
 
 	thread->parameter = parameter;
 
@@ -1140,12 +1138,10 @@ funnel_alloc(
 
 	if (funnel_lck_grp == LCK_GRP_NULL) {
 		funnel_lck_grp_attr = lck_grp_attr_alloc_init();
-		//lck_grp_attr_setstat(funnel_lck_grp_attr);
 
 		funnel_lck_grp = lck_grp_alloc_init("Funnel",  funnel_lck_grp_attr);
 
 		funnel_lck_attr = lck_attr_alloc_init();
-		//lck_attr_setdebug(funnel_lck_attr);
 	}
 	if ((fnl = (funnel_t *)kalloc(sizeof(funnel_t))) != 0){
 		bzero((void *)fnl, sizeof(funnel_t));
diff --git a/osfmk/kern/thread.h b/osfmk/kern/thread.h
index 990f094de..a8625c461 100644
--- a/osfmk/kern/thread.h
+++ b/osfmk/kern/thread.h
@@ -521,6 +521,9 @@ extern void				machine_thread_init(void);
 extern kern_return_t	machine_thread_create(
 							thread_t		thread,
 							task_t			task);
+extern void		machine_thread_switch_addrmode(
+						       thread_t			thread,
+						       int			oldmode_is64bit);
 
 extern void 		    machine_thread_destroy(
 							thread_t		thread);
@@ -646,6 +649,10 @@ extern void			thread_setentrypoint(
 						thread_t		thread,
 						mach_vm_offset_t	entry);
 
+extern void		thread_setsinglestep(
+						thread_t		thread,
+						int			on);
+
 extern kern_return_t	thread_wire_internal(
 							host_priv_t		host_priv,
 							thread_t		thread,
@@ -659,6 +666,9 @@ extern boolean_t	is_thread_idle(thread_t); /* True is TH_IDLE */
 extern kern_return_t	thread_dup(thread_t);
 
 extern task_t	get_threadtask(thread_t);
+#define thread_is_64bit(thd)	\
+	task_has_64BitAddr(get_threadtask(thd))
+
 
 extern void		*get_bsdthread_info(thread_t);
 extern void		set_bsdthread_info(thread_t, void *);
diff --git a/osfmk/kern/timer_call.c b/osfmk/kern/timer_call.c
index c1ffdabcf..60ac2fc7a 100644
--- a/osfmk/kern/timer_call.c
+++ b/osfmk/kern/timer_call.c
@@ -43,6 +43,8 @@
 #include <kern/timer_call.h>
 #include <kern/call_entry.h>
 
+#include <sys/kdebug.h>
+
 decl_simple_lock_data(static,timer_call_lock)
 
 static struct {
@@ -125,7 +127,7 @@ void
 _set_delayed_call_timer(
 	timer_call_t			call)
 {
-	clock_set_timer_deadline(call->deadline);
+	etimer_set_deadline(call->deadline);
 }
 
 boolean_t
@@ -290,6 +292,7 @@ timer_call_interrupt(
 	call = TC(queue_first(queue));
 
 	while (!queue_end(queue, qe(call))) {
+
 		if (call->deadline <= timestamp) {
 			timer_call_func_t		func;
 			timer_call_param_t		param0, param1;
@@ -302,8 +305,12 @@ timer_call_interrupt(
 
 			simple_unlock(&timer_call_lock);
 
+			KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 2) | DBG_FUNC_START, (int)func, param0, param1, 0, 0);
+
 			(*func)(param0, param1);
 
+			KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 2) | DBG_FUNC_END, (int)func, param0, param1, 0, 0);
+
 			simple_lock(&timer_call_lock);
 		}
 		else
@@ -316,4 +323,5 @@ timer_call_interrupt(
 		_set_delayed_call_timer(call);
 
 	simple_unlock(&timer_call_lock);
+
 }
diff --git a/osfmk/kern/wait_queue.c b/osfmk/kern/wait_queue.c
index 1c36d3036..b5e9c2d1f 100644
--- a/osfmk/kern/wait_queue.c
+++ b/osfmk/kern/wait_queue.c
@@ -1065,7 +1065,11 @@ wait_queue_wakeup64_all_locked(
 	queue_t q = &wake_queue_head;
 	kern_return_t res;
 
-	assert(wait_queue_held(wq));
+//	assert(wait_queue_held(wq));
+	if(!wq->wq_interlock.lock_data) {		/* (BRINGUP */
+		panic("wait_queue_wakeup64_all_locked: lock not held on %08X\n", wq);	/* (BRINGUP) */
+	}
+
 	queue_init(q);
 
 	/*
@@ -1117,6 +1121,9 @@ wait_queue_wakeup_all(
 
 	s = splsched();
 	wait_queue_lock(wq);
+	if(!wq->wq_interlock.lock_data) {		/* (BRINGUP */
+		panic("wait_queue_wakeup_all: we did not get the lock on %08X\n", wq);	/* (BRINGUP) */
+	}
 	ret = wait_queue_wakeup64_all_locked(
 				wq, (event64_t)((uint32_t)event),
 				result, TRUE);
diff --git a/osfmk/kern/wait_queue.h b/osfmk/kern/wait_queue.h
index 560b25325..74c8fa6d7 100644
--- a/osfmk/kern/wait_queue.h
+++ b/osfmk/kern/wait_queue.h
@@ -155,13 +155,24 @@ typedef struct wait_queue_link {
  * a problem with a thread lock, it normally times out at the wait
  * queue level first, hiding the real problem.
  */
-#define wait_queue_lock(wq)	\
-	((void) (!hw_lock_to(&(wq)->wq_interlock, LockTimeOut * 2) ? \
-		 panic("wait queue deadlock - wq=0x%x, cpu=%d\n", \
-		       wq, cpu_number()) : 0))
 
-#define wait_queue_unlock(wq) \
-	(assert(wait_queue_held(wq)), hw_lock_unlock(&(wq)->wq_interlock))
+static inline void wait_queue_lock(wait_queue_t wq) {
+	if (!hw_lock_to(&(wq)->wq_interlock, LockTimeOut * 2))
+		panic("wait queue deadlock - wq=0x%x, cpu=%d\n", wq, cpu_number());
+}
+
+static inline void wait_queue_unlock(wait_queue_t wq) {
+	assert(wait_queue_held(wq));
+#if defined(__i386__)
+	/* On certain x86 systems, this spinlock is susceptible to
+	 * lock starvation. Hence use an unlock variant which performs
+	 * a cacheline flush to minimize cache affinity on acquisition.
+	 */
+	i386_lock_unlock_with_flush(&(wq)->wq_interlock);
+#else
+        hw_lock_unlock(&(wq)->wq_interlock);
+#endif
+}
 
 #define wqs_lock(wqs)		wait_queue_lock(&(wqs)->wqs_wait_queue)
 #define wqs_unlock(wqs)		wait_queue_unlock(&(wqs)->wqs_wait_queue)
diff --git a/osfmk/kern/zalloc.c b/osfmk/kern/zalloc.c
index a802fa03e..45de75913 100644
--- a/osfmk/kern/zalloc.c
+++ b/osfmk/kern/zalloc.c
@@ -113,7 +113,7 @@
 #else /* !defined(__alpha) */
 
 #define is_kernel_data_addr(a)						\
-  (!(a) || ((a) >= VM_MIN_KERNEL_ADDRESS && !((a) & 0x3)))
+  (!(a) || ((a) >= vm_min_kernel_address && !((a) & 0x3)))
 
 #endif /* defined(__alpha) */
 
@@ -1233,7 +1233,7 @@ zone_gc(void)
 		 */
 
 		scan = (void *)z->free_elements;
-		(void *)z->free_elements = NULL;
+		z->free_elements = 0;
 
 		unlock_zone(z);
 
@@ -1277,7 +1277,7 @@ zone_gc(void)
 
 					if (keep != NULL) {
 						tail->next = (void *)z->free_elements;
-						(void *)z->free_elements = keep;
+						z->free_elements = (vm_offset_t) keep;
 						tail = keep = NULL;
 					} else {
 						m =0;
@@ -1289,7 +1289,7 @@ zone_gc(void)
 						}
 						if (m !=0 ) {
 							prev->next = (void *)z->free_elements;
-							(void *)z->free_elements = (void *)base_elt;
+							z->free_elements = (vm_offset_t) base_elt;
 							base_prev->next = elt;
 							prev = base_prev;
 						}
@@ -1314,7 +1314,7 @@ zone_gc(void)
 			lock_zone(z);
 
 			tail->next = (void *)z->free_elements;
-			(void *)z->free_elements = keep;
+			z->free_elements = (vm_offset_t) keep;
 
 			unlock_zone(z);
 		}
@@ -1367,7 +1367,7 @@ zone_gc(void)
 
 				if (keep != NULL) {
 					tail->next = (void *)z->free_elements;
-					(void *)z->free_elements = keep;
+					z->free_elements = (vm_offset_t) keep;
 				}
 
 				if (z->waiting) {
@@ -1394,7 +1394,7 @@ zone_gc(void)
 
 			if (keep != NULL) {
 				tail->next = (void *)z->free_elements;
-				(void *)z->free_elements = keep;
+				z->free_elements = (vm_offset_t) keep;
 			}
 
 		}
@@ -1480,7 +1480,7 @@ host_zone_info(
 #ifdef ppc
 	max_zones = num_zones + 4;
 #else
-	max_zones = num_zones + 2;
+	max_zones = num_zones + 3; /* ATN: count the number below!! */
 #endif
 	z = first_zone;
 	simple_unlock(&all_zones_lock);
@@ -1564,6 +1564,15 @@ host_zone_info(
 	zn++;
 	zi++;
 #endif
+
+#ifdef i386
+	strcpy(zn->zn_name, "page_tables");
+	pt_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
+			  &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
+	zn++;
+	zi++;
+#endif
+
 	strcpy(zn->zn_name, "kalloc.large");
 	kalloc_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
 			       &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
diff --git a/osfmk/mach-o/loader.h b/osfmk/mach-o/loader.h
index a962d389e..eb724a3cf 100644
--- a/osfmk/mach-o/loader.h
+++ b/osfmk/mach-o/loader.h
@@ -69,7 +69,7 @@ struct mach_header {
 
 /* Constant for the magic field of the mach_header */
 #define	MH_MAGIC	0xfeedface	/* the mach magic number */
-#define MH_CIGAM	NXSwapInt(MH_MAGIC)
+#define MH_CIGAM	0xcefaedfe
 
 /*
  * The layout of the file depends on the filetype.  For all but the MH_OBJECT
diff --git a/osfmk/mach/host_info.h b/osfmk/mach/host_info.h
index 9cd243a21..ce5d9b64d 100644
--- a/osfmk/mach/host_info.h
+++ b/osfmk/mach/host_info.h
@@ -114,9 +114,7 @@ typedef struct host_basic_info_old	*host_basic_info_old_t;
 		(sizeof(host_basic_info_data_old_t)/sizeof(integer_t)))
 #endif
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 struct host_basic_info {
 	integer_t		max_cpus;		/* max number of CPUs possible */
@@ -132,9 +130,7 @@ struct host_basic_info {
 	uint64_t		max_mem;		/* actual size of physical memory */
 };
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 typedef	struct host_basic_info	host_basic_info_data_t;
 typedef struct host_basic_info	*host_basic_info_t;
diff --git a/osfmk/mach/host_reboot.h b/osfmk/mach/host_reboot.h
index b92e87c18..65609dd7f 100644
--- a/osfmk/mach/host_reboot.h
+++ b/osfmk/mach/host_reboot.h
@@ -32,7 +32,8 @@
 #ifndef _MACH_HOST_REBOOT_
 #define _MACH_HOST_REBOOT_
 
-#define HOST_REBOOT_HALT	0x8
+#define HOST_REBOOT_HALT        0x0008
+#define HOST_REBOOT_UPSDELAY    0x0100
 #define	HOST_REBOOT_DEBUGGER	0x1000
 
 #endif	/* _MACH_HOST_REBOOT_ */
diff --git a/osfmk/mach/i386/boolean.h b/osfmk/mach/i386/boolean.h
index 03a94ba6a..7fe45eca8 100644
--- a/osfmk/mach/i386/boolean.h
+++ b/osfmk/mach/i386/boolean.h
@@ -65,6 +65,10 @@
 #ifndef	_MACH_I386_BOOLEAN_H_
 #define _MACH_I386_BOOLEAN_H_
 
+#if defined(__x86_64__)
+typedef unsigned int	boolean_t;
+#else
 typedef int		boolean_t;
+#endif
 
 #endif	/* _MACH_I386_BOOLEAN_H_ */
diff --git a/osfmk/mach/i386/exception.h b/osfmk/mach/i386/exception.h
index 5aa293e52..3de13b611 100644
--- a/osfmk/mach/i386/exception.h
+++ b/osfmk/mach/i386/exception.h
@@ -88,6 +88,7 @@
 #define EXC_I386_EXTERR			5
 #define EXC_I386_EMERR			6
 #define EXC_I386_BOUND			7
+#define EXC_I386_SSEEXTERR              8
 
 /*
  *	EXC_SOFTWARE
diff --git a/osfmk/mach/i386/fp_reg.h b/osfmk/mach/i386/fp_reg.h
index 87632e944..6ac9a34e1 100644
--- a/osfmk/mach/i386/fp_reg.h
+++ b/osfmk/mach/i386/fp_reg.h
@@ -59,32 +59,9 @@
 #ifndef	_I386_FP_SAVE_H_
 #define	_I386_FP_SAVE_H_
 
-/*
- *	Floating point registers and status, as saved
- *	and restored by FP save/restore instructions.
- */
-struct i386_fp_save	{
-	unsigned short	fp_control;	/* control */
-	unsigned short	fp_unused_1;
-	unsigned short	fp_status;	/* status */
-	unsigned short	fp_unused_2;
-	unsigned short	fp_tag;		/* register tags */
-	unsigned short	fp_unused_3;
-	unsigned int	fp_eip;		/* eip at failed instruction */
-	unsigned short	fp_cs;		/* cs at failed instruction */
-	unsigned short	fp_opcode;	/* opcode of failed instruction */
-	unsigned int	fp_dp;		/* data address */
-	unsigned short	fp_ds;		/* data segment */
-	unsigned short	fp_unused_4;
-};
-
-struct i386_fp_regs {
-	unsigned short	fp_reg_word[5][8];
-					/* space for 8 80-bit FP registers */
-};
 
 /* note when allocating this data structure, it must be 16 byte aligned. */
-struct i386_fx_save {
+struct x86_fx_save {
         unsigned short  fx_control;     /* control */
         unsigned short  fx_status;      /* status */
         unsigned char  	fx_tag;         /* register tags */
@@ -98,9 +75,10 @@ struct i386_fx_save {
         unsigned short  fx_bbz3;	/* better be zero when calling fxrtstor */
         unsigned int  	fx_MXCSR;
         unsigned int  	fx_MXCSR_MASK;
-        unsigned short  fx_reg_word[8][8];      /* STx/MMx registers */
-        unsigned short  fx_XMM_reg[8][8];       /* XMM0-XMM7 */
-        unsigned char 	fx_reserved[16*14];     /* reserved by intel for future expansion */
+        unsigned short  fx_reg_word[8][8];	/* STx/MMx registers */
+        unsigned short  fx_XMM_reg[8][16];	/* XMM0-XMM15 on 64 bit processors */
+                                                /* XMM0-XMM7  on 32 bit processors... unused storage reserved */
+        unsigned char 	fx_reserved[16*6];	/* reserved by intel for future expansion */
 };
 
 
diff --git a/osfmk/mach/i386/machine_types.defs b/osfmk/mach/i386/machine_types.defs
index 1fec79f15..52e660ed7 100644
--- a/osfmk/mach/i386/machine_types.defs
+++ b/osfmk/mach/i386/machine_types.defs
@@ -58,8 +58,13 @@ type double = MACH_MSG_TYPE_REAL_64;
  * a port in user space as an integer and
  * in kernel space as a pointer.
  */
+#if defined(__LP64__)
+type uintptr_t = uint64_t;
+type intptr_t = int64_t;
+#else
 type uintptr_t = uint32_t;
 type intptr_t = int32_t;
+#endif
 
 /*
  * These are the legacy Mach types that are
@@ -67,7 +72,11 @@ type intptr_t = int32_t;
  * They were defined in terms of int, not
  * long int, so they remain separate.
  */
+#if defined(__LP64__)
+type register_t = int64_t;
+#else
 type register_t = int32_t;
+#endif
 type integer_t = int32_t;
 type natural_t = uint32_t;
 
@@ -75,18 +84,25 @@ type natural_t = uint32_t;
  * These are the VM types that scale with the address
  * space size of a given process.
  */
+
+#if defined(__LP64__)
+type vm_address_t = uint64_t;
+type vm_offset_t = uint64_t;
+type vm_size_t = uint64_t;
+#else
 type vm_address_t = natural_t;
 type vm_offset_t = natural_t;
 type vm_size_t = natural_t;
+#endif
 
 /*
  * The mach_vm_xxx_t types are sized to hold the
  * maximum pointer, offset, etc... supported on the
  * platform.
  */
-type mach_vm_address_t = uint32_t;
-type mach_vm_offset_t = uint32_t;
-type mach_vm_size_t = uint32_t;
+type mach_vm_address_t = uint64_t;
+type mach_vm_offset_t = uint64_t;
+type mach_vm_size_t = uint64_t;
 
 #if	MACH_IPC_COMPAT
 /*
diff --git a/osfmk/mach/i386/rpc.h b/osfmk/mach/i386/rpc.h
index dcfe3e07e..55561f9eb 100644
--- a/osfmk/mach/i386/rpc.h
+++ b/osfmk/mach/i386/rpc.h
@@ -32,5 +32,5 @@
 #ifndef	_MACH_I386_RPC_H_
 #define	_MACH_I386_RPC_H_
 
-#endif	_MACH_I386_RPC_H_
+#endif	/* _MACH_I386_RPC_H_ */
 
diff --git a/osfmk/mach/i386/syscall_sw.h b/osfmk/mach/i386/syscall_sw.h
index 94615db0f..ac8999730 100644
--- a/osfmk/mach/i386/syscall_sw.h
+++ b/osfmk/mach/i386/syscall_sw.h
@@ -63,14 +63,103 @@
 
 #include <architecture/i386/asm_help.h>
 
-#define MACHCALLSEL     $0x07
+#if defined(__i386__)
+/*
+ * Software interrupt codes for 32-bit system call entry:
+ */
+#define UNIX_INT     0x80
+#define MACH_INT     0x81
+#define MACHDEP_INT  0x82
+#define DIAG_INT     0x83
+
+#ifndef KERNEL
+/*
+ * Syscall entry macros for use in libc:
+ * [Note that the nop padding is temporary during 4/4 transition.]
+ */
+#define SYSENTER_PAD	nop;nop;
+#define SYSCALL_PAD	nop;nop;nop;nop;nop;
+#define UNIX_SYSCALL_TRAP	\
+	SYSCALL_PAD		\
+	int $(UNIX_INT)
+#define MACHDEP_SYSCALL_TRAP	\
+	SYSCALL_PAD		\
+	int $(MACHDEP_INT)
+
+/*
+ * Macro to generate Mach call stubs in libc:
+ */
+#define kernel_trap(trap_name,trap_number,number_args) \
+LEAF(_##trap_name,0) ;\
+	movl	$##trap_number,%eax	;\
+	int		$(MACH_INT)		;\
+END(_##trap_name)
+
+#endif
+#endif /* defined(__i386__) */
 
+#if defined(__x86_64__)
+
+#ifndef KERNEL
+
+#define UNIX_SYSCALL_TRAP	\
+	syscall
+#define MACHDEP_SYSCALL_TRAP	\
+	syscall
+
+/*
+ * Macro to generate Mach call stubs in Libc.
+ * Existing calls use negative numbers for Mach traps, so
+ * until we change those and change the 32-bit kernel_trap
+ * macro above, we negate those numbers here for the 64-bit
+ * code path.
+ */
 #define kernel_trap(trap_name,trap_number,number_args) \
 LEAF(_##trap_name,0) ;\
-	movl	$##trap_number,%eax   ;\
-        lcall   MACHCALLSEL, $0 ;\
+	movq	%rcx, %r10	;\
+	movl	$ SYSCALL_CONSTRUCT_MACH(-##trap_number), %eax	;\
+	syscall		;\
 END(_##trap_name)
 
+#endif
+#endif /* defined(__x86_64__) */
+
+/*
+ * Syscall classes for 64-bit system call entry.
+ * For 64-bit users, the 32-bit syscall number is partitioned
+ * with the high-order bits representing the class and low-order
+ * bits being the syscall number within that class.
+ * The high-order 32-bits of the 64-bit syscall number are unused.
+ * All system classes enter the kernel via the syscall instruction.
+ *
+ * These are not #ifdef'd for x86-64 because they might be used for
+ * 32-bit someday and so the 64-bit comm page in a 32-bit kernel
+ * can use them.
+ */
+#define SYSCALL_CLASS_SHIFT	24
+#define SYSCALL_CLASS_MASK	(0xFF << SYSCALL_CLASS_SHIFT)
+#define SYSCALL_NUMBER_MASK	(~SYSCALL_CLASS_MASK)
+
+#define SYSCALL_CLASS_NONE	0	/* Invalid */
+#define SYSCALL_CLASS_MACH	1	/* Mach */	
+#define SYSCALL_CLASS_UNIX	2	/* Unix/BSD */
+#define SYSCALL_CLASS_MDEP	3	/* Machine-dependent */
+#define SYSCALL_CLASS_DIAG	4	/* Diagnostics */
+
+/* Macros to simpllfy constructing syscall numbers. */
+#define SYSCALL_CONSTRUCT_MACH(syscall_number) \
+			((SYSCALL_CLASS_MACH << SYSCALL_CLASS_SHIFT) | \
+			 (SYSCALL_NUMBER_MASK & (syscall_number)))
+#define SYSCALL_CONSTRUCT_UNIX(syscall_number) \
+			((SYSCALL_CLASS_UNIX << SYSCALL_CLASS_SHIFT) | \
+			 (SYSCALL_NUMBER_MASK & (syscall_number)))
+#define SYSCALL_CONSTRUCT_MDEP(syscall_number) \
+			((SYSCALL_CLASS_MDEP << SYSCALL_CLASS_SHIFT) | \
+			 (SYSCALL_NUMBER_MASK & (syscall_number)))
+#define SYSCALL_CONSTRUCT_DIAG(syscall_number) \
+			((SYSCALL_CLASS_DIAG << SYSCALL_CLASS_SHIFT) | \
+			 (SYSCALL_NUMBER_MASK & (syscall_number)))
+
 #endif	/* _MACH_I386_SYSCALL_SW_H_ */
 
 #endif	/* PRIVATE */
diff --git a/osfmk/mach/i386/thread_state.h b/osfmk/mach/i386/thread_state.h
index a455d784a..c7791001e 100644
--- a/osfmk/mach/i386/thread_state.h
+++ b/osfmk/mach/i386/thread_state.h
@@ -32,9 +32,10 @@
 #ifndef _MACH_I386_THREAD_STATE_H_
 #define _MACH_I386_THREAD_STATE_H_
 
-#define I386_THREAD_STATE_MAX	144
+/* Size of maximum exported thread state in words */
+#define I386_THREAD_STATE_MAX	(144)    /* Size of biggest state possible */
 
-#if defined (__i386__)
+#if defined (__i386__) || defined(__x86_64__)
 #define THREAD_STATE_MAX	I386_THREAD_STATE_MAX
 #endif
 
diff --git a/osfmk/mach/i386/thread_status.h b/osfmk/mach/i386/thread_status.h
index 7ac40ba6d..be97409c2 100644
--- a/osfmk/mach/i386/thread_status.h
+++ b/osfmk/mach/i386/thread_status.h
@@ -70,39 +70,34 @@
 #include <mach/message.h>
 #include <mach/i386/fp_reg.h>
 #include <mach/i386/thread_state.h>
-#include <architecture/i386/frame.h>	/* FIXME */
-#include <architecture/i386/fpu.h>	/* FIXME */
+#include <i386/eflags.h>
+
+
+
 /*
- *	i386_thread_state	this is the structure that is exported
- *				to user threads for use in status/mutate
- *				calls.  This structure should never
- *				change.
- *
- *	i386_float_state	exported to use threads for access to 
- *				floating point registers. Try not to 
- *				change this one, either.
- *
- *	i386_isa_port_map_state	exported to user threads to allow
- *				selective in/out operations
- *
- * 	i386_v86_assist_state 
+ * the i386_xxxx form is kept for legacy purposes since these types
+ * are externally known... eventually they should be deprecated.
+ * our internal implementation has moved to the following naming convention
  *
- *	thread_syscall_state 
+ *   x86_xxxx32 names are used to deal with 32 bit states
+ *   x86_xxxx64 names are used to deal with 64 bit states
+ *   x86_xxxx   names are used to deal with either 32 or 64 bit states
+ *	via a self-describing mechanism
  */
 
-/*     THREAD_STATE_FLAVOR_LIST 0 */
-#define i386_NEW_THREAD_STATE	1	/* used to be i386_THREAD_STATE */
-#define i386_FLOAT_STATE	2
-#define i386_ISA_PORT_MAP_STATE	3
-#define i386_V86_ASSIST_STATE	4
-#define i386_REGS_SEGS_STATE	5
-#define THREAD_SYSCALL_STATE	6
-#define THREAD_STATE_NONE	7
-#define i386_SAVED_STATE	8
 
 
 /*
- * x86-64 compatibility
+ * these are the legacy names which should be deprecated in the future
+ * they are externally known which is the only reason we don't just get
+ * rid of them
+ */
+#define i386_THREAD_STATE		1
+#define i386_FLOAT_STATE		2
+#define i386_EXCEPTION_STATE		3
+
+
+/*
  * THREAD_STATE_FLAVOR_LIST 0
  * 	these are the supported flavors
  */
@@ -118,284 +113,46 @@
 #define x86_DEBUG_STATE32		10
 #define x86_DEBUG_STATE64		11
 #define x86_DEBUG_STATE			12
+#define THREAD_STATE_NONE		13
 
 
 
-/*
- * VALID_THREAD_STATE_FLAVOR is a platform specific macro that when passed
- * an exception flavor will return if that is a defined flavor for that
- * platform. The macro must be manually updated to include all of the valid
- * exception flavors as defined above.
- */
-#define VALID_THREAD_STATE_FLAVOR(x)            \
-        ((x == i386_NEW_THREAD_STATE)        || \
-	 (x == i386_FLOAT_STATE)             || \
-	 (x == i386_ISA_PORT_MAP_STATE)      || \
-	 (x == i386_V86_ASSIST_STATE)        || \
-	 (x == i386_REGS_SEGS_STATE)         || \
-	 (x == THREAD_SYSCALL_STATE)         || \
-	 (x == THREAD_STATE_NONE)            || \
-	 (x == i386_SAVED_STATE))
-
-
-/*
- * x86-64 compatibility
- */
-struct x86_state_hdr {
-    int 	flavor;
-    int		count;
-};
-typedef struct x86_state_hdr x86_state_hdr_t;
-
-
-/*
- * This structure is used for both
- * i386_THREAD_STATE and i386_REGS_SEGS_STATE.
- */
-struct i386_new_thread_state {
-	unsigned int	gs;
-	unsigned int	fs;
-	unsigned int	es;
-	unsigned int	ds;
-	unsigned int	edi;
-	unsigned int	esi;
-	unsigned int	ebp;
-	unsigned int	esp;
-	unsigned int	ebx;
-	unsigned int	edx;
-	unsigned int	ecx;
-	unsigned int	eax;
-	unsigned int	eip;
-	unsigned int	cs;
-	unsigned int	efl;
-	unsigned int	uesp;
-	unsigned int	ss;
-};
-#define i386_NEW_THREAD_STATE_COUNT	((mach_msg_type_number_t) \
-		(sizeof (struct i386_new_thread_state)/sizeof(unsigned int)))
-
-/*
- * Subset of saved state stored by processor on kernel-to-kernel
- * trap.  (Used by ddb to examine state guaranteed to be present
- * on all traps into debugger.)
- */
-struct i386_saved_state_from_kernel {
-	unsigned int	gs;
-	unsigned int	fs;
-	unsigned int	es;
-	unsigned int	ds;
-	unsigned int	edi;
-	unsigned int	esi;
-	unsigned int	ebp;
-	unsigned int	esp;		/* kernel esp stored by pusha -
-					   we save cr2 here later */
-	unsigned int	ebx;
-	unsigned int	edx;
-	unsigned int	ecx;
-	unsigned int	eax;
-	unsigned int	trapno;
-	unsigned int	err;
-	unsigned int	eip;
-	unsigned int	cs;
-	unsigned int	efl;
-};
-
-/*
- * The format in which thread state is saved by Mach on this machine.  This
- * state flavor is most efficient for exception RPC's to kernel-loaded
- * servers, because copying can be avoided:
- */
-struct i386_saved_state {
-	unsigned int	gs;
-	unsigned int	fs;
-	unsigned int	es;
-	unsigned int	ds;
-	unsigned int	edi;
-	unsigned int	esi;
-	unsigned int	ebp;
-	unsigned int	esp;		/* kernel esp stored by pusha -
-					   we save cr2 here later */
-	unsigned int	ebx;
-	unsigned int	edx;
-	unsigned int	ecx;
-	unsigned int	eax;
-	unsigned int	trapno;
-	unsigned int	err;
-	unsigned int	eip;
-	unsigned int	cs;
-	unsigned int	efl;
-	unsigned int	uesp;
-	unsigned int	ss;
-	struct v86_segs {
-	    unsigned int v86_es;	/* virtual 8086 segment registers */
-	    unsigned int v86_ds;
-	    unsigned int v86_fs;
-	    unsigned int v86_gs;
-	} v86_segs;
-#define i386_SAVED_ARGV_COUNT	7
-	unsigned int	argv_status;	/* Boolean flag indicating whether or
-					 * not Mach copied in the args */
-	unsigned int	argv[i386_SAVED_ARGV_COUNT];
-					/* The return address, and the first several
-					 * function call args from the stack, for
-					 * efficient syscall exceptions */
-};
-#define i386_SAVED_STATE_COUNT	((mach_msg_type_number_t) \
-	(sizeof (struct i386_saved_state)/sizeof(unsigned int)))
-#define i386_REGS_SEGS_STATE_COUNT	i386_SAVED_STATE_COUNT
-
-/*
- * Machine-independent way for servers and Mach's exception mechanism to
- * choose the most efficient state flavor for exception RPC's:
- */
-#define MACHINE_THREAD_STATE		i386_SAVED_STATE
-#define MACHINE_THREAD_STATE_COUNT	144
-
 /*
  * Largest state on this machine:
  * (be sure mach/machine/thread_state.h matches!)
  */
 #define THREAD_MACHINE_STATE_MAX	THREAD_STATE_MAX
 
-/* 
- * Floating point state.
- *
- * fpkind tells in what way floating point operations are supported.  
- * See the values for fp_kind in <mach/i386/fp_reg.h>.
- * 
- * If the kind is FP_NO, then calls to set the state will fail, and 
- * thread_getstatus will return garbage for the rest of the state.
- * If "initialized" is false, then the rest of the state is garbage.  
- * Clients can set "initialized" to false to force the coprocessor to 
- * be reset.
- * "exc_status" is non-zero if the thread has noticed (but not 
- * proceeded from) a coprocessor exception.  It contains the status 
- * word with the exception bits set.  The status word in "fp_status" 
- * will have the exception bits turned off.  If an exception bit in 
- * "fp_status" is turned on, then "exc_status" should be zero.  This 
- * happens when the coprocessor exception is noticed after the system 
- * has context switched to some other thread.
- * 
- * If kind is FP_387, then "state" is a i387_state.  Other kinds might
- * also use i387_state, but somebody will have to verify it (XXX).
- * Note that the registers are ordered from top-of-stack down, not
- * according to physical register number.
- */
-
-/*
- * x86-64 compatibility
- */
-/* defn of 80bit x87 FPU or MMX register  */
-struct mmst_reg {
-	char	mmst_reg[10];
-	char	mmst_rsrv[6];
-};
-
-
-/* defn of 128 bit XMM regs */
-struct xmm_reg {
-	char		xmm_reg[16];
-};
-
-#define FP_STATE_BYTES 512
-
-struct i386_float_state {
-	int		fpkind;			/* FP_NO..FP_387 (readonly) */
-	int		initialized;
-	unsigned char	hw_state[FP_STATE_BYTES]; /* actual "hardware" state */
-	int		exc_status;		/* exception status (readonly) */
-};
-#define i386_FLOAT_STATE_COUNT ((mach_msg_type_number_t) \
-		(sizeof(struct i386_float_state)/sizeof(unsigned int)))
-
-/*
- * x86-64 compatibility
- */
-typedef struct {
-	int 			fpu_reserved[2];
-	fp_control_t		fpu_fcw;			/* x87 FPU control word */
-	fp_status_t		fpu_fsw;			/* x87 FPU status word */
-	uint8_t			fpu_ftw;			/* x87 FPU tag word */
-	uint8_t			fpu_rsrv1;			/* reserved */ 
-	uint16_t		fpu_fop;			/* x87 FPU Opcode */
-	uint32_t		fpu_ip;				/* x87 FPU Instruction Pointer offset */
-	uint16_t		fpu_cs;				/* x87 FPU Instruction Pointer Selector */
-	uint16_t		fpu_rsrv2;			/* reserved */
-	uint32_t		fpu_dp;				/* x87 FPU Instruction Operand(Data) Pointer offset */
-	uint16_t		fpu_ds;				/* x87 FPU Instruction Operand(Data) Pointer Selector */
-	uint16_t		fpu_rsrv3;			/* reserved */
-	uint32_t		fpu_mxcsr;			/* MXCSR Register state */
-	uint32_t		fpu_mxcsrmask;		/* MXCSR mask */
-	struct mmst_reg	fpu_stmm0;		/* ST0/MM0   */
-	struct mmst_reg	fpu_stmm1;		/* ST1/MM1  */
-	struct mmst_reg	fpu_stmm2;		/* ST2/MM2  */
-	struct mmst_reg	fpu_stmm3;		/* ST3/MM3  */
-	struct mmst_reg	fpu_stmm4;		/* ST4/MM4  */
-	struct mmst_reg	fpu_stmm5;		/* ST5/MM5  */
-	struct mmst_reg	fpu_stmm6;		/* ST6/MM6  */
-	struct mmst_reg	fpu_stmm7;		/* ST7/MM7  */
-	struct xmm_reg	fpu_xmm0;		/* XMM 0  */
-	struct xmm_reg	fpu_xmm1;		/* XMM 1  */
-	struct xmm_reg	fpu_xmm2;		/* XMM 2  */
-	struct xmm_reg	fpu_xmm3;		/* XMM 3  */
-	struct xmm_reg	fpu_xmm4;		/* XMM 4  */
-	struct xmm_reg	fpu_xmm5;		/* XMM 5  */
-	struct xmm_reg	fpu_xmm6;		/* XMM 6  */
-	struct xmm_reg	fpu_xmm7;		/* XMM 7  */
-	char			fpu_rsrv4[14*16];	/* reserved */
-	int 			fpu_reserved1;
-} i386_float_state_t;
-
-
-#define FP_old_STATE_BYTES ((mach_msg_type_number_t) \
-	(sizeof (struct i386_fp_save) + sizeof (struct i386_fp_regs)))
-
-struct i386_old_float_state {
-	int		fpkind;			/* FP_NO..FP_387 (readonly) */
-	int		initialized;
-	unsigned char	hw_state[FP_old_STATE_BYTES]; /* actual "hardware" state */
-	int		exc_status;		/* exception status (readonly) */
-};
-#define i386_old_FLOAT_STATE_COUNT ((mach_msg_type_number_t) \
-		(sizeof(struct i386_old_float_state)/sizeof(unsigned int)))
-
-
-#define PORT_MAP_BITS 0x400
-struct i386_isa_port_map_state {
-	unsigned char	pm[PORT_MAP_BITS>>3];
-};
-
-#define i386_ISA_PORT_MAP_STATE_COUNT ((mach_msg_type_number_t) \
-		(sizeof(struct i386_isa_port_map_state)/sizeof(unsigned int)))
 
 /*
- * V8086 assist supplies a pointer to an interrupt
- * descriptor table in task space.
+ * VALID_THREAD_STATE_FLAVOR is a platform specific macro that when passed
+ * an exception flavor will return if that is a defined flavor for that
+ * platform. The macro must be manually updated to include all of the valid
+ * exception flavors as defined above.
  */
-struct i386_v86_assist_state {
-	unsigned int	int_table;	/* interrupt table address */
-	int		int_count;	/* interrupt table size */
-};
+#define VALID_THREAD_STATE_FLAVOR(x)       \
+	((x == x86_THREAD_STATE32)	|| \
+	 (x == x86_FLOAT_STATE32)	|| \
+	 (x == x86_EXCEPTION_STATE32)	|| \
+	 (x == x86_DEBUG_STATE32)	|| \
+	 (x == x86_THREAD_STATE64)	|| \
+	 (x == x86_FLOAT_STATE64)	|| \
+	 (x == x86_EXCEPTION_STATE64)	|| \
+	 (x == x86_DEBUG_STATE64)	|| \
+	 (x == x86_THREAD_STATE)	|| \
+	 (x == x86_FLOAT_STATE)		|| \
+	 (x == x86_EXCEPTION_STATE)	|| \
+	 (x == x86_DEBUG_STATE)		|| \
+	 (x == THREAD_STATE_NONE))
 
-struct v86_interrupt_table {
-	unsigned int	count;	/* count of pending interrupts */
-	unsigned short	mask;	/* ignore this interrupt if true */
-	unsigned short	vec;	/* vector to take */
-};
 
-#define	i386_V86_ASSIST_STATE_COUNT ((mach_msg_type_number_t) \
-	    (sizeof(struct i386_v86_assist_state)/sizeof(unsigned int)))
 
-struct thread_syscall_state {
-	unsigned eax;
-	unsigned edx;
-	unsigned efl;
-	unsigned eip;
-	unsigned esp;
+struct x86_state_hdr {
+    int 	flavor;
+    int		count;
 };
+typedef struct x86_state_hdr x86_state_hdr_t;
 
-#define i386_THREAD_SYSCALL_STATE_COUNT ((mach_msg_type_number_t) \
-		(sizeof(struct thread_syscall_state) / sizeof(unsigned int)))
 
 /*
  * Main thread state consists of
@@ -403,9 +160,7 @@ struct thread_syscall_state {
  * eip and eflags.
  */
 
-#define i386_THREAD_STATE	-1
-
-typedef struct {
+struct i386_thread_state {
     unsigned int	eax;
     unsigned int	ebx;
     unsigned int	ecx;
@@ -422,15 +177,17 @@ typedef struct {
     unsigned int	es;
     unsigned int	fs;
     unsigned int	gs;
-} i386_thread_state_t;
+} ;
 
+/*
+ * to be depecrated in the future
+ */
+typedef struct i386_thread_state i386_thread_state_t;
 #define i386_THREAD_STATE_COUNT	((mach_msg_type_number_t) \
     ( sizeof (i386_thread_state_t) / sizeof (int) ))
 
-/*
- * x86-64 compatibility
- */
-typedef i386_thread_state_t x86_thread_state32_t;
+
+typedef struct i386_thread_state x86_thread_state32_t;
 #define x86_THREAD_STATE32_COUNT ((mach_msg_type_number_t) \
     ( sizeof (x86_thread_state32_t) / sizeof (int) ))
 
@@ -493,27 +250,110 @@ typedef struct x86_thread_state x86_thread_state_t;
 #define KERN_CODE_SELECTOR	0x0008
 #define KERN_DATA_SELECTOR	0x0010
 
+typedef struct fp_control {
+    unsigned short		invalid	:1,
+    				denorm	:1,
+				zdiv	:1,
+				ovrfl	:1,
+				undfl	:1,
+				precis	:1,
+					:2,
+				pc	:2,
+#define FP_PREC_24B		0
+#define	FP_PREC_53B		2
+#define FP_PREC_64B		3
+				rc	:2,
+#define FP_RND_NEAR		0
+#define FP_RND_DOWN		1
+#define FP_RND_UP		2
+#define FP_CHOP			3
+				/*inf*/	:1,
+					:3;
+} fp_control_t;
 /*
- * Thread floating point state
- * includes FPU environment as
- * well as the register stack.
+ * Status word.
  */
- 
-#define i386_THREAD_FPSTATE	-2
 
-typedef struct {
-    fp_env_t		environ;
-    fp_stack_t		stack;
-} i386_thread_fpstate_t;
+typedef struct fp_status {
+    unsigned short		invalid	:1,
+    				denorm	:1,
+				zdiv	:1,
+				ovrfl	:1,
+				undfl	:1,
+				precis	:1,
+				stkflt	:1,
+				errsumm	:1,
+				c0	:1,
+				c1	:1,
+				c2	:1,
+				tos	:3,
+				c3	:1,
+				busy	:1;
+} fp_status_t;
+				
+/* defn of 80bit x87 FPU or MMX register  */
+struct mmst_reg {
+	char	mmst_reg[10];
+	char	mmst_rsrv[6];
+};
+
+
+/* defn of 128 bit XMM regs */
+struct xmm_reg {
+	char		xmm_reg[16];
+};
+
+/* 
+ * Floating point state.
+ */
 
-#define i386_THREAD_FPSTATE_COUNT ((mach_msg_type_number_t)	\
-    ( sizeof (i386_thread_fpstate_t) / sizeof (int) ))
+#define FP_STATE_BYTES		512	/* number of chars worth of data from fpu_fcw */
+
+/* For legacy reasons we need to leave the hw_state as char bytes */
+struct i386_float_state {
+	int 			fpu_reserved[2];
+	fp_control_t		fpu_fcw;			/* x87 FPU control word */
+	fp_status_t		fpu_fsw;			/* x87 FPU status word */
+	uint8_t			fpu_ftw;			/* x87 FPU tag word */
+	uint8_t			fpu_rsrv1;			/* reserved */ 
+	uint16_t		fpu_fop;			/* x87 FPU Opcode */
+	uint32_t		fpu_ip;				/* x87 FPU Instruction Pointer offset */
+	uint16_t		fpu_cs;				/* x87 FPU Instruction Pointer Selector */
+	uint16_t		fpu_rsrv2;			/* reserved */
+	uint32_t		fpu_dp;				/* x87 FPU Instruction Operand(Data) Pointer offset */
+	uint16_t		fpu_ds;				/* x87 FPU Instruction Operand(Data) Pointer Selector */
+	uint16_t		fpu_rsrv3;			/* reserved */
+	uint32_t		fpu_mxcsr;			/* MXCSR Register state */
+	uint32_t		fpu_mxcsrmask;		/* MXCSR mask */
+	struct mmst_reg	fpu_stmm0;		/* ST0/MM0   */
+	struct mmst_reg	fpu_stmm1;		/* ST1/MM1  */
+	struct mmst_reg	fpu_stmm2;		/* ST2/MM2  */
+	struct mmst_reg	fpu_stmm3;		/* ST3/MM3  */
+	struct mmst_reg	fpu_stmm4;		/* ST4/MM4  */
+	struct mmst_reg	fpu_stmm5;		/* ST5/MM5  */
+	struct mmst_reg	fpu_stmm6;		/* ST6/MM6  */
+	struct mmst_reg	fpu_stmm7;		/* ST7/MM7  */
+	struct xmm_reg	fpu_xmm0;		/* XMM 0  */
+	struct xmm_reg	fpu_xmm1;		/* XMM 1  */
+	struct xmm_reg	fpu_xmm2;		/* XMM 2  */
+	struct xmm_reg	fpu_xmm3;		/* XMM 3  */
+	struct xmm_reg	fpu_xmm4;		/* XMM 4  */
+	struct xmm_reg	fpu_xmm5;		/* XMM 5  */
+	struct xmm_reg	fpu_xmm6;		/* XMM 6  */
+	struct xmm_reg	fpu_xmm7;		/* XMM 7  */
+	char			fpu_rsrv4[14*16];	/* reserved */
+	int 			fpu_reserved1;
+};
 
 
 /*
- * x86-64 compatibility
+ * to be depecrated in the future
  */
-typedef i386_float_state_t x86_float_state32_t;
+typedef struct i386_float_state i386_float_state_t;
+#define i386_FLOAT_STATE_COUNT ((mach_msg_type_number_t) \
+		(sizeof(i386_float_state_t)/sizeof(unsigned int)))
+	 
+typedef struct i386_float_state x86_float_state32_t;
 #define x86_FLOAT_STATE32_COUNT ((mach_msg_type_number_t) \
 		(sizeof(x86_float_state32_t)/sizeof(unsigned int)))
 	 
@@ -588,26 +428,21 @@ typedef struct x86_float_state x86_float_state_t;
  * useful to exception handlers.
  */
 
-#define i386_THREAD_EXCEPTSTATE	-3
-
-typedef struct {
-    unsigned int	trapno;
-    err_code_t		err;
-} i386_thread_exceptstate_t;
-
-#define i386_THREAD_EXCEPTSTATE_COUNT	((mach_msg_type_number_t) \
-    ( sizeof (i386_thread_exceptstate_t) / sizeof (int) ))
-
-
-/*
- * x86-64 compatibility
- */
 struct i386_exception_state {
     unsigned int	trapno;
     unsigned int	err;
     unsigned int	faultvaddr;
 };
 
+/*
+ * to be depecrated in the future
+ */
+typedef struct i386_exception_state i386_exception_state_t;
+#define i386_EXCEPTION_STATE_COUNT	((mach_msg_type_number_t) \
+    ( sizeof (i386_exception_state_t) / sizeof (int) ))
+
+#define I386_EXCEPTION_STATE_COUNT i386_EXCEPTION_STATE_COUNT
+
 typedef struct i386_exception_state x86_exception_state32_t;
 #define x86_EXCEPTION_STATE32_COUNT	((mach_msg_type_number_t) \
     ( sizeof (x86_exception_state32_t) / sizeof (int) ))
@@ -687,19 +522,264 @@ typedef struct x86_debug_state x86_debug_state_t;
 #define x86_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
 		(sizeof(x86_debug_state_t)/sizeof(unsigned int)))
 
+/*
+ * Machine-independent way for servers and Mach's exception mechanism to
+ * choose the most efficient state flavor for exception RPC's:
+ */
+#define MACHINE_THREAD_STATE		x86_THREAD_STATE
+#define MACHINE_THREAD_STATE_COUNT	x86_THREAD_STATE_COUNT
+
+
+#ifdef XNU_KERNEL_PRIVATE
+
+#define x86_SAVED_STATE32		THREAD_STATE_NONE + 1
+#define x86_SAVED_STATE64		THREAD_STATE_NONE + 2
+
+#define OLD_i386_THREAD_STATE		-1
+
 
 /*
- * Per-thread variable used
- * to store 'self' id for cthreads.
+ * when reloading the segment registers on
+ * a return out of the kernel, we may take
+ * a GeneralProtection or SegmentNotPresent
+ * fault if one or more of the segment
+ * registers in the saved state was improperly
+ * specified via an x86_THREAD_STATE32 call
+ * the frame we push on top of the existing
+ * save area looks like this... we need to
+ * carry this as part of the save area
+ * in case we get hit so that we have a big
+ * enough stack
+ */
+struct x86_seg_load_fault32 {
+	unsigned int	trapno;
+	unsigned int	err;
+	unsigned int	eip;
+	unsigned int	cs;
+	unsigned int	efl;
+};
+
+
+/*
+ * Subset of saved state stored by processor on kernel-to-kernel
+ * trap.  (Used by ddb to examine state guaranteed to be present
+ * on all traps into debugger.)
+ */
+struct x86_saved_state32_from_kernel {
+	unsigned int	gs;
+	unsigned int	fs;
+	unsigned int	es;
+	unsigned int	ds;
+	unsigned int	edi;
+	unsigned int	esi;
+	unsigned int	ebp;
+	unsigned int	cr2;	/* kernel esp stored by pusha - we save cr2 here later */
+	unsigned int	ebx;
+	unsigned int	edx;
+	unsigned int	ecx;
+	unsigned int	eax;
+	unsigned int	trapno;
+	unsigned int	err;
+	unsigned int	eip;
+	unsigned int	cs;
+	unsigned int	efl;
+};
+
+/*
+ * The format in which thread state is saved by Mach on this machine.  This
+ * state flavor is most efficient for exception RPC's to kernel-loaded
+ * servers, because copying can be avoided:
  */
  
-#define i386_THREAD_CTHREADSTATE	-4
- 
-typedef struct {
-    unsigned int	self;
-} i386_thread_cthreadstate_t;
+struct x86_saved_state32 {
+	unsigned int	gs;
+	unsigned int	fs;
+	unsigned int	es;
+	unsigned int	ds;
+	unsigned int	edi;
+	unsigned int	esi;
+	unsigned int	ebp;
+	unsigned int	cr2;	/* kernel esp stored by pusha - we save cr2 here later */
+	unsigned int	ebx;
+	unsigned int	edx;
+	unsigned int	ecx;
+	unsigned int	eax;
+	unsigned int	trapno;
+	unsigned int	err;
+	unsigned int	eip;
+	unsigned int	cs;
+	unsigned int	efl;
+	unsigned int	uesp;
+	unsigned int	ss;
+};
+typedef struct x86_saved_state32 x86_saved_state32_t;
 
-#define i386_THREAD_CTHREADSTATE_COUNT	((mach_msg_type_number_t) \
-    ( sizeof (i386_thread_cthreadstate_t) / sizeof (int) ))
+#define x86_SAVED_STATE32_COUNT	((mach_msg_type_number_t) \
+	(sizeof (x86_saved_state32_t)/sizeof(unsigned int)))
+
+struct x86_saved_state32_tagged {
+	uint32_t			tag;
+	struct x86_saved_state32	state;
+};
+typedef struct x86_saved_state32_tagged x86_saved_state32_tagged_t;
+
+struct x86_sframe32 {
+        /*
+	 * in case we throw a fault reloading
+	 * segment registers on a return out of
+	 * the kernel... the 'slf' state is only kept
+	 * long enough to rejigger (i.e. restore
+	 * the save area to its original state)
+	 * the save area and throw the appropriate
+	 * kernel trap pointing to the 'ssf' state
+	 */
+        struct x86_seg_load_fault32	slf;
+        struct x86_saved_state32_tagged ssf;
+};
+typedef struct x86_sframe32 x86_sframe32_t;
+
+
+
+/*
+ * This is the state pushed onto the 64-bit interrupt stack
+ * on any exception/trap/interrupt.
+ */
+struct x86_64_intr_stack_frame {
+	uint32_t	trapno;
+	uint32_t	trapfn;
+	uint64_t	err;
+	uint64_t	rip;
+	uint64_t	cs;
+	uint64_t	rflags;
+	uint64_t	rsp;
+	uint64_t	ss;
+};
+typedef struct x86_64_intr_stack_frame x86_64_intr_stack_frame_t;
+
+/*
+ * This defines the state saved before entry into compatibility mode.
+ * The machine state is pushed automatically and the compat state is
+ * synthethized in the exception handling code.
+ */
+struct x86_saved_state_compat32 {
+	struct x86_saved_state32_tagged	iss32;
+	uint32_t			pad_for_16byte_alignment[2];
+	struct	x86_64_intr_stack_frame	isf64;
+};
+typedef struct x86_saved_state_compat32 x86_saved_state_compat32_t;
+
+
+struct x86_sframe_compat32 {
+        struct x86_64_intr_stack_frame  slf;
+	uint32_t			pad_for_16byte_alignment[2];
+        struct x86_saved_state_compat32 ssf;
+	uint32_t			empty[4];
+};
+typedef struct x86_sframe_compat32 x86_sframe_compat32_t;
+
+
+
+/*
+ * thread state format for task running in 64bit long mode
+ * in long mode, the same hardware frame is always pushed regardless
+ * of whether there was a change in privlege level... therefore, there
+ * is no need for an x86_saved_state64_from_kernel variant
+ */
+
+struct x86_saved_state64 {
+        /*
+	 * saved state organized to reflect the
+	 * system call ABI register convention
+	 * so that we can just pass a pointer
+	 * to the saved state when calling through
+	 * to the actual system call functions
+	 * the ABI limits us to 6 args passed in 
+	 * registers... I've add v_arg6 - v_arg8
+	 * to accomodate our most 'greedy' system
+	 * calls (both BSD and MACH)... the individual
+	 * system call handlers will fill these in
+	 * via copyin if needed...
+	 */
+        uint64_t	rdi;		/* arg0 for system call */
+	uint64_t	rsi;
+	uint64_t	rdx;
+        uint64_t	r10;
+        uint64_t	r8;
+        uint64_t	r9;		/* arg5 for system call */
+        uint64_t	v_arg6;
+        uint64_t	v_arg7;
+        uint64_t	v_arg8;
+
+        uint64_t	cr2;
+        uint64_t	r15;
+        uint64_t	r14;
+        uint64_t	r13;
+        uint64_t	r12;
+        uint64_t	r11;
+	uint64_t	rbp;
+	uint64_t	rbx;
+	uint64_t	rcx;
+	uint64_t	rax;
+
+	uint32_t	gs;
+	uint32_t	fs;
+	struct	x86_64_intr_stack_frame	isf;
+};
+typedef struct x86_saved_state64 x86_saved_state64_t;
+#define x86_SAVED_STATE64_COUNT	((mach_msg_type_number_t) \
+	(sizeof (struct x86_saved_state64)/sizeof(unsigned int)))
+
+struct x86_saved_state64_tagged {
+	uint32_t		tag;
+	x86_saved_state64_t	state;
+};
+typedef struct x86_saved_state64_tagged x86_saved_state64_tagged_t;
+
+struct x86_sframe64 {
+        struct x86_64_intr_stack_frame	slf;
+	uint32_t			pad_for_16byte_alignment[3];
+        struct x86_saved_state64_tagged	ssf;
+};
+typedef struct x86_sframe64 x86_sframe64_t;
+
+extern uint32_t get_eflags_exportmask(void);
+/*
+ * Unified, tagged saved state:
+ */
+typedef struct {
+	uint32_t			flavor;
+	union {
+		x86_saved_state32_t	ss_32;
+		x86_saved_state64_t	ss_64;
+	} uss;
+} x86_saved_state_t;
+#define	ss_32	uss.ss_32
+#define	ss_64	uss.ss_64
+
+static inline boolean_t
+is_saved_state64(x86_saved_state_t *iss)
+{
+	return (iss->flavor == x86_SAVED_STATE64);
+}
+
+static inline boolean_t
+is_saved_state32(x86_saved_state_t *iss)
+{
+	return (iss->flavor == x86_SAVED_STATE32);
+}
+
+static inline x86_saved_state32_t *
+saved_state32(x86_saved_state_t *iss)
+{
+	return &iss->ss_32; 
+}
+
+static inline x86_saved_state64_t *
+saved_state64(x86_saved_state_t *iss)
+{
+	return &iss->ss_64; 
+}
+
+#endif /* XNU_KERNEL_PRIVATE */
 
 #endif	/* _MACH_I386_THREAD_STATUS_H_ */
diff --git a/osfmk/mach/i386/vm_param.h b/osfmk/mach/i386/vm_param.h
index 151c6a93b..b7d6931ab 100644
--- a/osfmk/mach/i386/vm_param.h
+++ b/osfmk/mach/i386/vm_param.h
@@ -118,7 +118,27 @@
 					~(I386_PGBYTES-1))
 #define i386_trunc_page(x)	(((pmap_paddr_t)(x)) & ~(I386_PGBYTES-1))
 
-#define VM_MAX_PAGE_ADDRESS     0x00000000C0000000ULL
+
+
+#define VM_MIN_ADDRESS64	((user_addr_t) 0x0000000000000000ULL)
+/*
+ * default top of user stack... it grows down from here
+ */
+#define VM_USRSTACK64		((user_addr_t) 0x00007FFF5FC00000ULL)
+#define VM_DYLD64		((user_addr_t) 0x00007FFF5FC00000ULL)
+#define VM_LIB64_SHR_DATA	((user_addr_t) 0x00007FFF60000000ULL)
+#define VM_LIB64_SHR_TEXT	((user_addr_t) 0x00007FFF80000000ULL)
+/*
+ * the end of the usable user address space , for now about 47 bits.
+ * the 64 bit commpage is past the end of this
+ */
+#define VM_MAX_PAGE_ADDRESS	((user_addr_t) 0x00007FFFFFE00000ULL)
+/*
+ * canonical end of user address space for limits checking
+ */
+#define VM_MAX_USER_PAGE_ADDRESS ((user_addr_t)0x00007FFFFFFFF000ULL)
+
+
 
 /* system-wide values */
 #define MACH_VM_MIN_ADDRESS		((mach_vm_offset_t) 0)
@@ -126,15 +146,30 @@
 
 /* process-relative values (all 32-bit legacy only for now) */
 #define VM_MIN_ADDRESS		((vm_offset_t) 0)
-#define VM_MAX_ADDRESS		((vm_offset_t) (VM_MAX_PAGE_ADDRESS & 0xFFFFFFFF))
+#define VM_USRSTACK32		((vm_offset_t) 0xC0000000)
+#define VM_MAX_ADDRESS		((vm_offset_t) 0xFFE00000)
+
+
+
 
 #ifdef	KERNEL_PRIVATE 
 
 /* Kernel-wide values */
-#define VM_MIN_KERNEL_ADDRESS	((vm_offset_t) 0xC0000000U)
-#define VM_MAX_KERNEL_ADDRESS	((vm_offset_t) 0xFfffffffU)
+#define VM_MIN_KERNEL_ADDRESS	((vm_offset_t) 0x00001000U)
+/*
+ * XXX
+ * The kernel max VM address is limited to 0xFF3FFFFF for now because
+ * some data structures are explicitly allocated at 0xFF400000 without
+ * VM's knowledge (see osfmk/i386/locore.s for the allocation of PTmap and co.).
+ * We can't let VM allocate memory from there.
+ */
+
+#define VM_MAX_KERNEL_ADDRESS	((vm_offset_t) 0xFE7FFFFF)
 #define KERNEL_STACK_SIZE		(I386_PGBYTES*4)
 
+#define VM_MAP_MIN_ADDRESS	MACH_VM_MIN_ADDRESS
+#define VM_MAP_MAX_ADDRESS	MACH_VM_MAX_ADDRESS
+
 /* FIXME  - always leave like this? */
 #define	INTSTACK_SIZE	(I386_PGBYTES*4)
 
@@ -145,11 +180,13 @@
 #define VM32_MIN_ADDRESS		((vm32_offset_t) 0)
 #define VM32_MAX_ADDRESS		((vm32_offset_t) (VM_MAX_PAGE_ADDRESS & 0xFFFFFFFF))
 
-#define LINEAR_KERNEL_ADDRESS	((vm_offset_t) 0xc0000000)
+#define LINEAR_KERNEL_ADDRESS	((vm_offset_t) 0x00000000)
 
-#define VM_MIN_KERNEL_LOADED_ADDRESS	((vm_offset_t) 0x0c000000U)
+#define VM_MIN_KERNEL_LOADED_ADDRESS	((vm_offset_t) 0x00000000U)
 #define VM_MAX_KERNEL_LOADED_ADDRESS	((vm_offset_t) 0x1fffffffU)
 
+#define NCOPY_WINDOWS	4
+
 /*
  *	Conversion between 80386 pages and VM pages
  */
@@ -181,6 +218,9 @@
 	 );						\
 	MACRO_END
 
+#define IS_USERADDR64_CANONICAL(addr)			\
+	((addr) < (VM_MAX_USER_PAGE_ADDRESS + PAGE_SIZE))
+
 #endif	/* MACH_KERNEL_PRIVATE */
 
 #endif	/* KERNEL_PRIVATE */
diff --git a/osfmk/mach/i386/vm_types.h b/osfmk/mach/i386/vm_types.h
index 405e20083..bba48df44 100644
--- a/osfmk/mach/i386/vm_types.h
+++ b/osfmk/mach/i386/vm_types.h
@@ -80,9 +80,9 @@
  *
  * They also had an implicit "same size as pointer" characteristic
  * to them (i.e. Mach's traditional types are very ILP32 or ILP64
- * centric).  We will likely support x86 ABIs that do not follow
- * either ofthese models (specifically LP64).  Therefore, we had to
- * make a choice between making these types scale with pointers or stay
+ * centric).  We support x86 ABIs that do not follow either of
+ * these models (specifically LP64).  Therefore, we had to make a
+ * choice between making these types scale with pointers or stay
  * tied to integers.  Because their use is predominantly tied to
  * to the size of an integer, we are keeping that association and
  * breaking free from pointer size guarantees.
@@ -96,14 +96,22 @@ typedef int			integer_t;
  * A vm_offset_t is a type-neutral pointer,
  * e.g. an offset into a virtual memory space.
  */
+#ifdef __LP64__
+typedef uintptr_t		vm_offset_t;
+#else	/* __LP64__ */
 typedef	natural_t		vm_offset_t;
+#endif	/* __LP64__ */
 
 /*
  * A vm_size_t is the proper type for e.g.
  * expressing the difference between two
  * vm_offset_t entities.
  */
+#ifdef __LP64__
+typedef uintptr_t		vm_size_t;
+#else	/* __LP64__ */
 typedef	natural_t		vm_size_t;
+#endif	/* __LP64__ */
 
 /*
  * This new type is independent of a particular vm map's
@@ -112,23 +120,19 @@ typedef	natural_t		vm_size_t;
  * where the size of the map is not known - or we don't
  * want to have to distinguish.
  */
-typedef uint32_t		mach_vm_address_t;
-typedef uint32_t		mach_vm_offset_t;
-typedef uint32_t		mach_vm_size_t;
+typedef uint64_t		mach_vm_address_t;
+typedef uint64_t		mach_vm_offset_t;
+typedef uint64_t		mach_vm_size_t;
 
 /* LP64todo - convert these over for good */
-#if 0 
+#if 1 
 typedef uint64_t		vm_map_offset_t;
 typedef uint64_t		vm_map_address_t;
 typedef uint64_t		vm_map_size_t;
-#define VM_MAP_MIN_ADDRESS	MACH_VM_MIN_ADDRESS
-#define VM_MAP_MAX_ADDRESS	MACH_VM_MAX_ADDRESS
 #else
 typedef uint32_t		vm_map_offset_t;
 typedef uint32_t		vm_map_address_t;
 typedef uint32_t		vm_map_size_t;
-#define VM_MAP_MIN_ADDRESS	VM_MIN_ADDRESS
-#define VM_MAP_MAX_ADDRESS	VM_MAX_ADDRESS
 #endif
 
 #ifdef	MACH_KERNEL_PRIVATE
diff --git a/osfmk/mach/kmod.h b/osfmk/mach/kmod.h
index 8a14af44f..3acba538e 100644
--- a/osfmk/mach/kmod.h
+++ b/osfmk/mach/kmod.h
@@ -49,9 +49,7 @@ typedef void* kmod_args_t;
 
 #define KMOD_MAX_NAME	64
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 /* LP64todo - not 64-bit safe */
 typedef struct kmod_reference {
@@ -59,9 +57,7 @@ typedef struct kmod_reference {
 	struct kmod_info	*info;
 } kmod_reference_t;
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 /**************************************************************************************/
 /*	 warning any changes to this structure affect the following macros.	      */	
@@ -73,9 +69,7 @@ typedef struct kmod_reference {
 typedef kern_return_t kmod_start_func_t(struct kmod_info *ki, void *data);
 typedef kern_return_t kmod_stop_func_t(struct kmod_info *ki, void *data);
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 /* LP64todo - not 64-bit safe */
 
@@ -94,9 +88,7 @@ typedef struct kmod_info {
         kmod_stop_func_t	*stop;
 } kmod_info_t;
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 typedef kmod_info_t *kmod_info_array_t;
 
diff --git a/osfmk/mach/mach_traps.h b/osfmk/mach/mach_traps.h
index eb982c3d0..2438e7260 100644
--- a/osfmk/mach/mach_traps.h
+++ b/osfmk/mach/mach_traps.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -202,6 +202,11 @@ extern kern_return_t task_for_pid(
 				int pid,
 				mach_port_name_t *t);
 
+extern kern_return_t task_name_for_pid(
+				mach_port_name_t target_tport,
+				int pid,
+				mach_port_name_t *tn);
+
 extern kern_return_t pid_for_task(
 				mach_port_name_t t,
 				int *x);
@@ -223,13 +228,8 @@ extern kern_return_t map_fd(
 #ifdef	XNU_KERNEL_PRIVATE
 
 /* Syscall data translations routines */
-#ifdef __ppc__
 #define	PAD_(t)	(sizeof(uint64_t) <= sizeof(t) \
  		? 0 : sizeof(uint64_t) - sizeof(t))
-#else
-#define	PAD_(t)	(sizeof(register_t) <= sizeof(t) \
- 		? 0 : sizeof(register_t) - sizeof(t))
-#endif
 
 #if BYTE_ORDER == LITTLE_ENDIAN
 #define	PADL_(t)	0
@@ -412,6 +412,14 @@ struct task_for_pid_args {
 extern kern_return_t task_for_pid(
 				struct task_for_pid_args *args);
 
+struct task_name_for_pid_args {
+	PAD_ARG_(mach_port_name_t, target_tport);
+	PAD_ARG_(int, pid);
+	PAD_ARG_(user_addr_t, t);
+};
+extern kern_return_t task_name_for_pid(
+				struct task_name_for_pid_args *args);
+
 struct pid_for_task_args {
 	PAD_ARG_(mach_port_name_t, t);
 	PAD_ARG_(user_addr_t, pid);
@@ -525,17 +533,6 @@ struct mk_timer_cancel_trap_args {
 extern kern_return_t mk_timer_cancel_trap(
 				struct mk_timer_cancel_trap_args *args);
 
-/* no user-level prototype for this one */
-struct mk_timebase_info_trap_args {
-	PAD_ARG_(uint32_t *, delta);
-	PAD_ARG_(uint32_t *, abs_to_ns_numer);
-	PAD_ARG_(uint32_t *, abs_to_ns_denom);
-	PAD_ARG_(uint32_t *, proc_to_abs_numer);
-	PAD_ARG_(uint32_t *, proc_to_abs_denom);
-};
-extern void mk_timebase_info_trap(
-				struct mk_timebase_info_trap_args *args);
-
 /* not published to LP64 clients yet */
 struct iokit_user_client_trap_args {
 	PAD_ARG_(void *, userClientRef);
diff --git a/osfmk/mach/mach_types.defs b/osfmk/mach/mach_types.defs
index 4a02ff803..174c6b9af 100644
--- a/osfmk/mach/mach_types.defs
+++ b/osfmk/mach/mach_types.defs
@@ -104,6 +104,14 @@ type task_t = mach_port_t
 #endif	/* KERNEL_SERVER */
 		;
 
+type task_name_t = mach_port_t
+#if	KERNEL_SERVER
+		intran: task_name_t convert_port_to_task_name(mach_port_t)
+		outtran: mach_port_t convert_task_name_to_port(task_name_t)
+		destructor: task_name_deallocate(task_name_t)
+#endif	/* KERNEL_SERVER */
+		;
+
 type thread_t = mach_port_t
 #if	KERNEL_SERVER
 		intran: thread_t convert_port_to_thread(mach_port_t)
diff --git a/osfmk/mach/mach_types.h b/osfmk/mach/mach_types.h
index f51898bf3..76e6f8af8 100644
--- a/osfmk/mach/mach_types.h
+++ b/osfmk/mach/mach_types.h
@@ -109,7 +109,7 @@
  * If we are in the kernel, then pick up the kernel definitions for
  * the basic mach types.
  */
-typedef struct task			*task_t;
+typedef struct task			*task_t, *task_name_t;
 typedef struct thread		*thread_t, *thread_act_t;
 typedef struct ipc_space		*ipc_space_t;
 typedef struct host			*host_t;
@@ -151,6 +151,7 @@ __END_DECLS
  * ports at user-space.
  */
 typedef mach_port_t		task_t;
+typedef mach_port_t		task_name_t;
 typedef mach_port_t		thread_t;
 typedef	mach_port_t		thread_act_t;
 typedef mach_port_t		ipc_space_t;
@@ -236,6 +237,7 @@ typedef exception_handler_array_t exception_port_arrary_t;
 
 
 #define TASK_NULL		((task_t) 0)
+#define TASK_NAME_NULL		((task_name_t) 0)
 #define THREAD_NULL		((thread_t) 0)
 #define THR_ACT_NULL 		((thread_act_t) 0)
 #define IPC_SPACE_NULL		((ipc_space_t) 0)
diff --git a/osfmk/mach/machine.h b/osfmk/mach/machine.h
index 0f55c1150..e6d82b043 100644
--- a/osfmk/mach/machine.h
+++ b/osfmk/mach/machine.h
@@ -226,7 +226,7 @@ __END_DECLS
 #define	CPU_SUBTYPE_MC68030_ONLY	((cpu_subtype_t) 3)
 
 /*
- *	I386 subtypes.
+ *	I386 subtypes
  */
 
 #define CPU_SUBTYPE_INTEL(f, m)	((cpu_subtype_t) (f) + ((m) << 4))
@@ -333,4 +333,20 @@ __END_DECLS
 #define CPU_SUBTYPE_POWERPC_7450	((cpu_subtype_t) 11)
 #define CPU_SUBTYPE_POWERPC_970		((cpu_subtype_t) 100)
 
+/*
+ *      CPU families (sysctl hw.cpufamily)
+ *
+ * NB: the encodings of the CPU families are intentionally arbitrary.
+ * There is no ordering, and you should never try to deduce whether
+ * or not some feature is available based on the family.
+ * Use feature flags (eg, hw.optional.altivec) to test for optional
+ * functionality.
+ */
+#define CPUFAMILY_UNKNOWN    0
+#define CPUFAMILY_POWERPC_G3 0xcee41549
+#define CPUFAMILY_POWERPC_G4 0x77c184ae
+#define CPUFAMILY_POWERPC_G5 0xed76d8aa
+#define CPUFAMILY_INTEL_6_14 0x73d67300  /* Intel Core Solo and Intel Core Duo (32-bit Pentium-M with SSE3) */
+#define CPUFAMILY_INTEL_6_15 0x426f69ef  /* Intel Core 2 */
+
 #endif	/* _MACH_MACHINE_H_ */
diff --git a/osfmk/mach/machine/asm.h b/osfmk/mach/machine/asm.h
index 6f52457de..677c65f5f 100644
--- a/osfmk/mach/machine/asm.h
+++ b/osfmk/mach/machine/asm.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "mach/ppc/asm.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/asm.h"
 #else
 #error architecture not supported
diff --git a/osfmk/mach/machine/boolean.h b/osfmk/mach/machine/boolean.h
index cc6aa7639..455681fdc 100644
--- a/osfmk/mach/machine/boolean.h
+++ b/osfmk/mach/machine/boolean.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "mach/ppc/boolean.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/boolean.h"
 #else
 #error architecture not supported
diff --git a/osfmk/mach/machine/exception.h b/osfmk/mach/machine/exception.h
index 0d5cc62be..7b648f0dd 100644
--- a/osfmk/mach/machine/exception.h
+++ b/osfmk/mach/machine/exception.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "mach/ppc/exception.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/exception.h"
 #else
 #error architecture not supported
diff --git a/osfmk/mach/machine/kern_return.h b/osfmk/mach/machine/kern_return.h
index b71089823..7a3ab1dfc 100644
--- a/osfmk/mach/machine/kern_return.h
+++ b/osfmk/mach/machine/kern_return.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "mach/ppc/kern_return.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/kern_return.h"
 #else
 #error architecture not supported
diff --git a/osfmk/mach/machine/machine_types.defs b/osfmk/mach/machine/machine_types.defs
index 287849c0e..706163445 100644
--- a/osfmk/mach/machine/machine_types.defs
+++ b/osfmk/mach/machine/machine_types.defs
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "mach/ppc/machine_types.defs"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/machine_types.defs"
 #else
 #error architecture not supported
diff --git a/osfmk/mach/machine/ndr_def.h b/osfmk/mach/machine/ndr_def.h
index 2c530d050..3e68259b7 100644
--- a/osfmk/mach/machine/ndr_def.h
+++ b/osfmk/mach/machine/ndr_def.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "mach/ppc/ndr_def.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/ndr_def.h"
 #else
 #error architecture not supported
diff --git a/osfmk/mach/machine/processor_info.h b/osfmk/mach/machine/processor_info.h
index ba177f5ef..a0a1eec16 100644
--- a/osfmk/mach/machine/processor_info.h
+++ b/osfmk/mach/machine/processor_info.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "mach/ppc/processor_info.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/processor_info.h"
 #else
 #error architecture not supported
diff --git a/osfmk/mach/machine/rpc.h b/osfmk/mach/machine/rpc.h
index 469cc63e6..0eb07a82d 100644
--- a/osfmk/mach/machine/rpc.h
+++ b/osfmk/mach/machine/rpc.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "mach/ppc/rpc.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/rpc.h"
 #else
 #error architecture not supported
diff --git a/osfmk/mach/machine/syscall_sw.h b/osfmk/mach/machine/syscall_sw.h
index 22b78d8e6..ca0e27001 100644
--- a/osfmk/mach/machine/syscall_sw.h
+++ b/osfmk/mach/machine/syscall_sw.h
@@ -33,7 +33,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "mach/ppc/syscall_sw.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/syscall_sw.h"
 #else
 #error architecture not supported
diff --git a/osfmk/mach/machine/thread_state.h b/osfmk/mach/machine/thread_state.h
index b78ab495b..c11399a0d 100644
--- a/osfmk/mach/machine/thread_state.h
+++ b/osfmk/mach/machine/thread_state.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined(__ppc64__)
 #include "mach/ppc/thread_state.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/thread_state.h"
 #else
 #error architecture not supported
diff --git a/osfmk/mach/machine/thread_status.h b/osfmk/mach/machine/thread_status.h
index 0cfdb91fc..cf246ee29 100644
--- a/osfmk/mach/machine/thread_status.h
+++ b/osfmk/mach/machine/thread_status.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "mach/ppc/thread_status.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/thread_status.h"
 #else
 #error architecture not supported
diff --git a/osfmk/mach/machine/vm_param.h b/osfmk/mach/machine/vm_param.h
index 74525f3cc..bebe608fd 100644
--- a/osfmk/mach/machine/vm_param.h
+++ b/osfmk/mach/machine/vm_param.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "mach/ppc/vm_param.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/vm_param.h"
 #else
 #error architecture not supported
diff --git a/osfmk/mach/machine/vm_types.h b/osfmk/mach/machine/vm_types.h
index f086abcfd..aaae76cff 100644
--- a/osfmk/mach/machine/vm_types.h
+++ b/osfmk/mach/machine/vm_types.h
@@ -31,7 +31,7 @@
 
 #if defined (__ppc__) || defined(__ppc64__)
 #include "mach/ppc/vm_types.h"
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/vm_types.h"
 #else
 #error architecture not supported
diff --git a/osfmk/mach/memory_object_types.h b/osfmk/mach/memory_object_types.h
index 063363d8a..5f55f8578 100644
--- a/osfmk/mach/memory_object_types.h
+++ b/osfmk/mach/memory_object_types.h
@@ -72,6 +72,8 @@
 
 #include <mach/port.h>
 #include <mach/message.h>
+#include <mach/vm_prot.h>
+#include <mach/vm_sync.h>
 #include <mach/vm_types.h>
 #include <mach/machine/vm_types.h>
 
@@ -81,6 +83,8 @@
 
 typedef unsigned long long	memory_object_offset_t;
 typedef unsigned long long	memory_object_size_t;
+typedef natural_t		memory_object_cluster_size_t;
+
 
 /*
  * Temporary until real EMMI version gets re-implemented
@@ -88,14 +92,61 @@ typedef unsigned long long	memory_object_size_t;
 
 #ifdef	KERNEL_PRIVATE
 
+struct memory_object_pager_ops;	/* forward declaration */
+
 typedef struct 		memory_object {
-	int		*pager;
+	const struct memory_object_pager_ops	*mo_pager_ops;
 } *memory_object_t;
 
 typedef struct		memory_object_control {
-	struct vm_object *object;
+	struct vm_object *moc_object;
+	unsigned int	moc_ikot; /* XXX fake ip_kotype */
 } *memory_object_control_t;
 
+typedef const struct memory_object_pager_ops {
+	void (*memory_object_reference)(
+		memory_object_t mem_obj);
+	void (*memory_object_deallocate)(
+		memory_object_t mem_obj);
+	kern_return_t (*memory_object_init)(
+		memory_object_t mem_obj,
+		memory_object_control_t mem_control,
+		memory_object_cluster_size_t size);
+	kern_return_t (*memory_object_terminate)(
+		memory_object_t mem_obj);
+	kern_return_t (*memory_object_data_request)(
+		memory_object_t mem_obj,
+		memory_object_offset_t offset,
+		memory_object_cluster_size_t length,
+		vm_prot_t desired_access);
+	kern_return_t (*memory_object_data_return)(
+		memory_object_t mem_obj,
+		memory_object_offset_t offset,
+		vm_size_t size,
+		memory_object_offset_t *resid_offset,
+		int *io_error,
+		boolean_t dirty,
+		boolean_t kernel_copy,
+		int upl_flags);
+	kern_return_t (*memory_object_data_initialize)(
+		memory_object_t mem_obj,
+		memory_object_offset_t offset,
+		vm_size_t size);
+	kern_return_t (*memory_object_data_unlock)(
+		memory_object_t mem_obj,
+		memory_object_offset_t offset,
+		vm_size_t size,
+		vm_prot_t desired_access);
+	kern_return_t (*memory_object_synchronize)(
+		memory_object_t mem_obj,
+		memory_object_offset_t offset,
+		vm_size_t size,
+		vm_sync_t sync_flags);
+	kern_return_t (*memory_object_unmap)(
+		memory_object_t mem_obj);
+	const char *memory_object_pager_name;
+} * memory_object_pager_ops_t;
+
 #else	/* KERNEL_PRIVATE */
 
 typedef mach_port_t	memory_object_t;
@@ -240,8 +291,6 @@ __END_DECLS
 
 #endif	/* PRIVATE */
 
-typedef natural_t memory_object_cluster_size_t;
-
 struct memory_object_perf_info {
 	memory_object_cluster_size_t	cluster_size;
 	boolean_t			may_cache;
@@ -391,12 +440,14 @@ typedef uint32_t	upl_size_t;	/* page-aligned byte size */
 #define UPL_NOZEROFILL		0x00400000
 #define UPL_WILL_MODIFY		0x00800000 /* caller will modify the pages */
 
+#define UPL_NEED_32BIT_ADDR	0x01000000
+
 /* UPL flags known by this kernel */
-#define UPL_VALID_FLAGS		0x00FFFFFF
+#define UPL_VALID_FLAGS		0x01FFFFFF
 
 
 /* upl abort error flags */
-#define UPL_ABORT_RESTART		0x1
+#define UPL_ABORT_RESTART	0x1
 #define UPL_ABORT_UNAVAILABLE	0x2
 #define UPL_ABORT_ERROR		0x4
 #define UPL_ABORT_FREE_ON_EMPTY	0x8  /* only implemented in wrappers */
@@ -556,8 +607,7 @@ extern vm_size_t 	upl_get_internal_pagelist_offset(void);
 __BEGIN_DECLS
 
 extern ppnum_t	upl_phys_page(upl_page_info_t *upl, int index);
-extern void	   	upl_set_dirty(upl_t   upl);
-extern void		upl_clear_dirty(upl_t   upl);
+extern void	upl_clear_dirty(upl_t upl, boolean_t value);
 
 __END_DECLS
 
diff --git a/osfmk/mach/mig.h b/osfmk/mach/mig.h
index c0e114ca4..72ca0dccb 100644
--- a/osfmk/mach/mig.h
+++ b/osfmk/mach/mig.h
@@ -58,13 +58,11 @@
 #endif /* defined(TypeCheck) */
 
 /*
- * Pack MIG message structs if we have Power alignment of structs.
+ * Pack MIG message structs.
  * This is an indicator of the need to view shared structs in a
  * binary-compatible format - and MIG message structs are no different.
  */
-#if __DARWIN_ALIGN_POWER
 #define __MigPackStructs 1
-#endif
 
 /*
  * Definition for MIG-generated server stub routines.  These routines
diff --git a/osfmk/mach/port.h b/osfmk/mach/port.h
index 5fdb3cb86..db2234982 100644
--- a/osfmk/mach/port.h
+++ b/osfmk/mach/port.h
@@ -99,7 +99,7 @@
 typedef natural_t mach_port_name_t;
 typedef mach_port_name_t *mach_port_name_array_t;
 
-#ifdef	KERNEL_PRIVATE
+#ifdef	KERNEL
 
 /* 
  *	mach_port_t - a named port right
@@ -140,7 +140,7 @@ typedef struct ipc_port	        *ipc_port_t;
 
 typedef ipc_port_t 		mach_port_t;
 
-#else	/* KERNEL_PRIVATE */
+#else	/* KERNEL */
 
 /* 
  *	mach_port_t - a named port right
@@ -163,7 +163,7 @@ typedef ipc_port_t 		mach_port_t;
 typedef mach_port_name_t 		mach_port_t;
 #endif
 
-#endif	/* KERNEL_PRIVATE */
+#endif	/* KERNEL */
 
 typedef mach_port_t			*mach_port_array_t;
 
diff --git a/osfmk/mach/ppc/rpc.h b/osfmk/mach/ppc/rpc.h
index 992518209..b3a274a2d 100644
--- a/osfmk/mach/ppc/rpc.h
+++ b/osfmk/mach/ppc/rpc.h
@@ -32,4 +32,4 @@
 #ifndef	_MACH_PPC_RPC_H_
 #define	_MACH_PPC_RPC_H_
 
-#endif	_MACH_PPC_RPC_H_
+#endif	/* _MACH_PPC_RPC_H_ */
diff --git a/osfmk/mach/ppc/syscall_sw.h b/osfmk/mach/ppc/syscall_sw.h
index 95b52988e..a2dda183d 100644
--- a/osfmk/mach/ppc/syscall_sw.h
+++ b/osfmk/mach/ppc/syscall_sw.h
@@ -71,7 +71,7 @@ ppc_trap(CHUDCall,0x6009)
 ppc_trap(ppcNull,0x600A)	
 ppc_trap(perfmon_control,0x600B)	
 ppc_trap(ppcNullinst,0x600C)	
-ppc_trap(pmsCntrl,0x600D)	
+ppc_trap(pmsCPUCntrl,0x600D)	
 #endif /* _MACH_SYSCALL_SW_H_ */
 
 #endif	/* _MACH_PPC_SYSCALL_SW_H_ */
diff --git a/osfmk/mach/syscall_sw.h b/osfmk/mach/syscall_sw.h
index cbbabd329..410d590e4 100644
--- a/osfmk/mach/syscall_sw.h
+++ b/osfmk/mach/syscall_sw.h
@@ -96,6 +96,7 @@ kernel_trap(init_process,-41,0)
 kernel_trap(map_fd,-43,5)
 #endif	/* __LP64__ */
 
+kernel_trap(task_name_for_pid,-44,3)
 kernel_trap(task_for_pid,-45,3)
 kernel_trap(pid_for_task,-46,2)
 
diff --git a/osfmk/mach/task.defs b/osfmk/mach/task.defs
index c4847998e..ae8e6f1c4 100644
--- a/osfmk/mach/task.defs
+++ b/osfmk/mach/task.defs
@@ -117,7 +117,7 @@ routine	mach_ports_lookup(
  *      Returns information about the target task.
  */
 routine task_info(
-                target_task     : task_t;
+                target_task     : task_name_t;
                 flavor          : task_flavor_t;
         out     task_info_out   : task_info_t, CountInOut);
 
diff --git a/osfmk/mach/task_info.h b/osfmk/mach/task_info.h
index 0d2950199..a21ad9e7a 100644
--- a/osfmk/mach/task_info.h
+++ b/osfmk/mach/task_info.h
@@ -87,9 +87,7 @@ typedef	integer_t	task_info_data_t[TASK_INFO_MAX];
  *	Currently defined information structures.
  */
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 #define TASK_BASIC_INFO_32      4       /* basic information */
 
@@ -206,9 +204,7 @@ typedef struct task_absolutetime_info	*task_absolutetime_info_t;
 #define TASK_AUDIT_TOKEN_COUNT	\
 		(sizeof(audit_token_t) / sizeof(natural_t))
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 
 /*
diff --git a/osfmk/mach/task_special_ports.h b/osfmk/mach/task_special_ports.h
index e10d73032..b53d5fa46 100644
--- a/osfmk/mach/task_special_ports.h
+++ b/osfmk/mach/task_special_ports.h
@@ -74,6 +74,8 @@ typedef	int	task_special_port_t;
 
 #define TASK_HOST_PORT		2	/* The host (priv) port for task.  */
 
+#define TASK_NAME_PORT		3	/* the name (unpriv) port for task */
+
 #define TASK_BOOTSTRAP_PORT	4	/* Bootstrap environment for task. */
 
 /*
diff --git a/osfmk/mach/thread_status.h b/osfmk/mach/thread_status.h
index 17478927d..aead09bf9 100644
--- a/osfmk/mach/thread_status.h
+++ b/osfmk/mach/thread_status.h
@@ -83,9 +83,10 @@
 typedef	natural_t	*thread_state_t;	/* Variable-length array */
 
 /* THREAD_STATE_MAX is now defined in <mach/machine/thread_state.h> */
-typedef	int	thread_state_data_t[THREAD_STATE_MAX];
+typedef	natural_t	thread_state_data_t[THREAD_STATE_MAX];
 
 #define	THREAD_STATE_FLAVOR_LIST	0	/* List of valid flavors */
+#define THREAD_STATE_FLAVOR_LIST_NEW	128
 
 typedef	int			thread_state_flavor_t;
 typedef thread_state_flavor_t	*thread_state_flavor_array_t;
diff --git a/osfmk/mach/vm_param.h b/osfmk/mach/vm_param.h
index 861d5501b..b1a33add0 100644
--- a/osfmk/mach/vm_param.h
+++ b/osfmk/mach/vm_param.h
@@ -220,6 +220,9 @@ extern uint64_t		mem_actual;		/* 64-bit size of memory - not limited by maxmem *
 extern uint64_t		sane_size;		/* Memory size to use for defaults calculations */
 extern addr64_t 	vm_last_addr;	/* Highest kernel virtual address known to the VM system */
 
+extern const vm_offset_t	vm_min_kernel_address;
+extern const vm_offset_t	vm_max_kernel_address;
+
 #endif	/* XNU_KERNEL_PRIVATE */
 
 extern vm_size_t	page_size;
diff --git a/osfmk/mach/vm_prot.h b/osfmk/mach/vm_prot.h
index 2801dafd7..ed81997c2 100644
--- a/osfmk/mach/vm_prot.h
+++ b/osfmk/mach/vm_prot.h
@@ -84,6 +84,12 @@ typedef int		vm_prot_t;
 #define VM_PROT_WRITE	((vm_prot_t) 0x02)	/* write permission */
 #define VM_PROT_EXECUTE	((vm_prot_t) 0x04)	/* execute permission */
 
+/*
+ *	For now we can only support no execute on stacks... too many
+ *	entries are marked w/o VM_PROT_EXECUTE that actually have code
+ */
+#define STACK_ONLY_NX	1
+
 /*
  *	The default protection for newly-created virtual memory
  */
@@ -129,4 +135,5 @@ typedef int		vm_prot_t;
 
 #define VM_PROT_WANTS_COPY	((vm_prot_t) 0x10)
 
+
 #endif	/* _MACH_VM_PROT_H_ */
diff --git a/osfmk/mach/vm_region.h b/osfmk/mach/vm_region.h
index 3326507d5..d848aa540 100644
--- a/osfmk/mach/vm_region.h
+++ b/osfmk/mach/vm_region.h
@@ -50,9 +50,7 @@
 
 #include <sys/cdefs.h>
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 /*
  *	Types defined:
@@ -257,8 +255,6 @@ struct vm_read_entry {
 typedef struct mach_vm_read_entry	mach_vm_read_entry_t[VM_MAP_ENTRY_MAX];
 typedef struct vm_read_entry		vm_read_entry_t[VM_MAP_ENTRY_MAX];
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 #endif	/*_MACH_VM_REGION_H_*/
diff --git a/osfmk/mach_debug/vm_info.h b/osfmk/mach_debug/vm_info.h
index 3d840aebd..15134e126 100644
--- a/osfmk/mach_debug/vm_info.h
+++ b/osfmk/mach_debug/vm_info.h
@@ -72,9 +72,7 @@
 #include <mach/vm_prot.h>
 #include <mach/memory_object_types.h>
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=power
-#endif
+#pragma pack(4)
 
 /*
  *	Remember to update the mig type definitions
@@ -148,8 +146,6 @@ typedef struct vm_info_object {
 
 typedef vm_info_object_t *vm_info_object_array_t;
 
-#if __DARWIN_ALIGN_POWER
-#pragma options align=reset
-#endif
+#pragma pack()
 
 #endif	/* _MACH_DEBUG_VM_INFO_H_ */
diff --git a/osfmk/machine/cpu_capabilities.h b/osfmk/machine/cpu_capabilities.h
index 8f7dc84b9..a48f01723 100644
--- a/osfmk/machine/cpu_capabilities.h
+++ b/osfmk/machine/cpu_capabilities.h
@@ -42,7 +42,7 @@
 #else /* !KERNEL_PRIVATE -- System Framework header */
 #if defined (__ppc__) || defined(__ppc64__)
 #include <System/ppc/cpu_capabilities.h>
-#elif defined (__i386__)
+#elif defined (__i386__) || defined(__x86_64__)
 #include <System/i386/cpu_capabilities.h>
 #else
 #error architecture not supported
diff --git a/osfmk/ppc/Diagnostics.h b/osfmk/ppc/Diagnostics.h
index 864b924de..0adb89266 100644
--- a/osfmk/ppc/Diagnostics.h
+++ b/osfmk/ppc/Diagnostics.h
@@ -35,6 +35,7 @@
 /*
  * Here are the Diagnostic interface interfaces
  * Lovingly crafted by Bill Angell using traditional methods
+ *	Keep selectors in sync with the x86 version where possible.	
 */
 #ifdef	KERNEL_PRIVATE
 
diff --git a/osfmk/ppc/Makefile b/osfmk/ppc/Makefile
index 5649d653b..03891582a 100644
--- a/osfmk/ppc/Makefile
+++ b/osfmk/ppc/Makefile
@@ -16,11 +16,11 @@ EXPORT_ONLY_FILES = 		\
 	io_map_entries.h	\
 	lock.h			\
 	locks.h			\
+	pms.h			\
 	proc_reg.h		\
 	machine_routines.h	\
 	mappings.h		\
 	savearea.h		\
-	pms.h			\
 	simple_lock.h
 
 INSTALL_MD_DIR = ppc
diff --git a/osfmk/ppc/PPCcalls.h b/osfmk/ppc/PPCcalls.h
index 1262af9ec..262fe2e91 100644
--- a/osfmk/ppc/PPCcalls.h
+++ b/osfmk/ppc/PPCcalls.h
@@ -55,7 +55,7 @@ PPCcallEnt	PPCcalls[] = {
 	PPCcall(bb_settaskenv),			/* 0x6007 Set the BlueBox per thread task environment data */
 	PPCcall(vmm_stop_vm),			/* 0x6008 Stop a running VM */
 
-	PPCcall(dis),					/* 0x6009 CHUD Interface hook */
+	PPCcall(dis),					/* 0x6009 disabled */
 	
 	PPCcall(ppcNull),				/* 0x600A Null PPC syscall */
 	PPCcall(perfmon_control),		/* 0x600B performance monitor */
diff --git a/osfmk/ppc/ast.h b/osfmk/ppc/ast.h
index 918935f22..a24933948 100644
--- a/osfmk/ppc/ast.h
+++ b/osfmk/ppc/ast.h
@@ -36,9 +36,8 @@
 #ifndef	_PPC_AST_H_
 #define	_PPC_AST_H_
 
-#define AST_PPC_CHUD_URGENT     0x80000000
-#define AST_PPC_CHUD            0x40000000
-
-#define AST_PPC_CHUD_ALL        (AST_PPC_CHUD_URGENT|AST_PPC_CHUD)
+#define AST_PPC_CHUD_URGENT     AST_CHUD_URGENT
+#define AST_PPC_CHUD            AST_CHUD
+#define AST_PPC_CHUD_ALL        AST_CHUD_ALL
 
 #endif	/* _PPC_AST_H_ */
diff --git a/osfmk/ppc/chud/chud_xnu.h b/osfmk/ppc/chud/chud_xnu.h
deleted file mode 100644
index fc4169bc1..000000000
--- a/osfmk/ppc/chud/chud_xnu.h
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- * 
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the License
- * may not be used to create, or enable the creation or redistribution of,
- * unlawful or unlicensed copies of an Apple operating system, or to
- * circumvent, violate, or enable the circumvention or violation of, any
- * terms of an Apple operating system software license agreement.
- * 
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this file.
- * 
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- * 
- * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
- */
-
-#ifndef _PPC_CHUD_XNU_H_
-#define _PPC_CHUD_XNU_H_
-
-
-#include <stdint.h>
-#include <mach/boolean.h>
-#include <mach/mach_types.h>
-
-#pragma mark **** version ****
-uint32_t chudxnu_version(void);
-
-#pragma mark **** task ****
-// ********************************************************************************
-// task
-// ********************************************************************************
-int chudxnu_pid_for_task(task_t task);
-task_t chudxnu_task_for_pid(int pid);
-int chudxnu_current_pid(void);
-
-kern_return_t chudxnu_task_read(task_t task, void *kernaddr, uint64_t usraddr, vm_size_t size);
-kern_return_t chudxnu_task_write(task_t task, uint64_t useraddr, void *kernaddr, vm_size_t size);
-kern_return_t chudxnu_kern_read(void *destaddr, vm_offset_t srcaddr, vm_size_t size);
-kern_return_t chudxnu_kern_write(vm_offset_t destaddr, void *srcaddr, vm_size_t size);
-
-boolean_t chudxnu_is_64bit_task(task_t task);
-
-#pragma mark **** thread ****
-// ********************************************************************************
-// thread
-// ********************************************************************************
-kern_return_t chudxnu_bind_thread(thread_t thread, int cpu);
-
-kern_return_t chudxnu_unbind_thread(thread_t thread);
-
-kern_return_t chudxnu_thread_get_state( thread_t thread, 
-						thread_flavor_t flavor,
-					    thread_state_t tstate,
-					    mach_msg_type_number_t *count,
-					    boolean_t user_only);
-
-kern_return_t chudxnu_thread_set_state( thread_t thread, 
-						thread_flavor_t flavor,
-					    thread_state_t tstate,
-					    mach_msg_type_number_t count,
-					    boolean_t user_only);
-
-kern_return_t chudxnu_thread_user_state_available(thread_t thread);
-
-
-kern_return_t chudxnu_thread_get_callstack( thread_t thread,	
-											uint32_t *callStack,
-											mach_msg_type_number_t *count,
-											boolean_t user_only);
-
-kern_return_t chudxnu_thread_get_callstack64(thread_t thread,
-											uint64_t *callStack,
-						mach_msg_type_number_t *count,
-						boolean_t user_only);
-
-task_t chudxnu_current_task(void);
-thread_t chudxnu_current_thread(void);
-
-task_t chudxnu_task_for_thread(thread_t thread);
-
-kern_return_t chudxnu_all_tasks(task_array_t		*task_list,
-								mach_msg_type_number_t	*count);
-kern_return_t chudxnu_free_task_list(task_array_t	*task_list,
-									 mach_msg_type_number_t	*count);
-									 
-kern_return_t chudxnu_all_threads(	thread_array_t		*thread_list,
-									mach_msg_type_number_t	*count);
-kern_return_t chudxnu_task_threads(	task_t task,
-			 						thread_array_t *thread_list,
-			 						mach_msg_type_number_t *count);
-kern_return_t chudxnu_free_thread_list(thread_array_t	*thread_list,
-			 			mach_msg_type_number_t *count);
-
-kern_return_t chudxnu_thread_info(  thread_t thread,
-        				thread_flavor_t flavor,
-        				thread_info_t thread_info_out,
-        				mach_msg_type_number_t *thread_info_count);
-
-kern_return_t chudxnu_thread_last_context_switch(thread_t thread, uint64_t *timestamp);
-				    
-
-#pragma mark **** memory ****
-// ********************************************************************************
-// memory
-// ********************************************************************************
-
-uint64_t chudxnu_avail_memory_size(void);
-uint64_t chudxnu_phys_memory_size(void);
-
-vm_offset_t chudxnu_io_map(uint64_t phys_addr, vm_size_t size);
-
-uint32_t chudxnu_phys_addr_wimg(uint64_t phys_addr);
-
-#pragma mark **** cpu ****
-// ********************************************************************************
-// cpu
-// ********************************************************************************
-int chudxnu_avail_cpu_count(void);
-int chudxnu_phys_cpu_count(void);
-int chudxnu_cpu_number(void);
-
-kern_return_t chudxnu_enable_cpu(int cpu, boolean_t enable);
-
-kern_return_t chudxnu_enable_cpu_nap(int cpu, boolean_t enable);
-boolean_t chudxnu_cpu_nap_enabled(int cpu);
-
-boolean_t chudxnu_get_interrupts_enabled(void);
-boolean_t chudxnu_set_interrupts_enabled(boolean_t enable);
-boolean_t chudxnu_at_interrupt_context(void);
-void chudxnu_cause_interrupt(void);
-
-kern_return_t chudxnu_set_shadowed_spr(int cpu, int spr, uint32_t val);
-kern_return_t chudxnu_set_shadowed_spr64(int cpu, int spr, uint64_t val);
-
-uint32_t chudxnu_get_orig_cpu_l2cr(int cpu);
-uint32_t chudxnu_get_orig_cpu_l3cr(int cpu);
-
-kern_return_t chudxnu_read_spr(int cpu, int spr, uint32_t *val_p);
-kern_return_t chudxnu_read_spr64(int cpu, int spr, uint64_t *val_p);
-kern_return_t chudxnu_write_spr(int cpu, int spr, uint32_t val);
-kern_return_t chudxnu_write_spr64(int cpu, int spr, uint64_t val);
-
-void chudxnu_flush_caches(void);
-void chudxnu_enable_caches(boolean_t enable);
-
-kern_return_t chudxnu_perfmon_acquire_facility(task_t);
-kern_return_t chudxnu_perfmon_release_facility(task_t);
-
-uint32_t * chudxnu_get_branch_trace_buffer(uint32_t *entries);
-
-typedef struct {
-    uint32_t hwResets;
-    uint32_t hwMachineChecks;
-    uint32_t hwDSIs;
-    uint32_t hwISIs;
-    uint32_t hwExternals;
-    uint32_t hwAlignments;
-    uint32_t hwPrograms;
-    uint32_t hwFloatPointUnavailable;
-    uint32_t hwDecrementers;
-    uint32_t hwIOErrors;
-    uint32_t hwSystemCalls;
-    uint32_t hwTraces;
-    uint32_t hwFloatingPointAssists;
-    uint32_t hwPerformanceMonitors;
-    uint32_t hwAltivecs;
-    uint32_t hwInstBreakpoints;
-    uint32_t hwSystemManagements;
-    uint32_t hwAltivecAssists;
-    uint32_t hwThermal;
-    uint32_t hwSoftPatches;
-    uint32_t hwMaintenances;
-    uint32_t hwInstrumentations;
-} rupt_counters_t;
-
-kern_return_t chudxnu_get_cpu_rupt_counters(int cpu, rupt_counters_t *rupts);
-kern_return_t chudxnu_clear_cpu_rupt_counters(int cpu);
-
-kern_return_t chudxnu_passup_alignment_exceptions(boolean_t enable);
-
-kern_return_t chudxnu_scom_read(uint32_t reg, uint64_t *data);
-kern_return_t chudxnu_scom_write(uint32_t reg, uint64_t data);
-
-#pragma mark **** callbacks ****
-// ********************************************************************************
-// callbacks
-// ********************************************************************************
-
-void chudxnu_cancel_all_callbacks(void);
-
-// cpu timer - each cpu has its own callback 
-typedef kern_return_t (*chudxnu_cpu_timer_callback_func_t)(thread_flavor_t flavor, thread_state_t tstate,  mach_msg_type_number_t count);
-kern_return_t chudxnu_cpu_timer_callback_enter(chudxnu_cpu_timer_callback_func_t func, uint32_t time, uint32_t units); // callback is entered on current cpu
-kern_return_t chudxnu_cpu_timer_callback_cancel(void); // callback is cleared on current cpu
-kern_return_t chudxnu_cpu_timer_callback_cancel_all(void); // callback is cleared on all cpus
-
-// trap callback - one callback for system
-typedef kern_return_t (*chudxnu_trap_callback_func_t)(uint32_t trapentry, thread_flavor_t flavor, thread_state_t tstate,  mach_msg_type_number_t count);
-kern_return_t chudxnu_trap_callback_enter(chudxnu_trap_callback_func_t func);
-kern_return_t chudxnu_trap_callback_cancel(void);
-
-// interrupt callback - one callback for system
-typedef kern_return_t (*chudxnu_interrupt_callback_func_t)(uint32_t trapentry, thread_flavor_t flavor, thread_state_t tstate,  mach_msg_type_number_t count);
-kern_return_t chudxnu_interrupt_callback_enter(chudxnu_interrupt_callback_func_t func);
-kern_return_t chudxnu_interrupt_callback_cancel(void);
-
-// ast callback - one callback for system
-typedef kern_return_t (*chudxnu_perfmon_ast_callback_func_t)(thread_flavor_t flavor, thread_state_t tstate,  mach_msg_type_number_t count);
-kern_return_t chudxnu_perfmon_ast_callback_enter(chudxnu_perfmon_ast_callback_func_t func);
-kern_return_t chudxnu_perfmon_ast_callback_cancel(void);
-kern_return_t chudxnu_perfmon_ast_send(void);
-kern_return_t chudxnu_perfmon_ast_send_urgent(boolean_t urgent);
-
-// cpusig callback - one callback for system
-typedef kern_return_t (*chudxnu_cpusig_callback_func_t)(int request, thread_flavor_t flavor, thread_state_t tstate, mach_msg_type_number_t count);
-kern_return_t chudxnu_cpusig_callback_enter(chudxnu_cpusig_callback_func_t func);
-kern_return_t chudxnu_cpusig_callback_cancel(void);
-kern_return_t chudxnu_cpusig_send(int otherCPU, uint32_t request);
-
-// kdebug callback - one callback for system
-typedef kern_return_t (*chudxnu_kdebug_callback_func_t)(uint32_t debugid, uint32_t arg0, uint32_t arg1, uint32_t arg2, uint32_t arg3, uint32_t arg4);
-kern_return_t chudxnu_kdebug_callback_enter(chudxnu_kdebug_callback_func_t func);
-kern_return_t chudxnu_kdebug_callback_cancel(void);
-
-// timer callback - multiple callbacks
-typedef kern_return_t (*chudxnu_timer_callback_func_t)(uint32_t param0, uint32_t param1);
-typedef void *	chud_timer_t;
-chud_timer_t chudxnu_timer_alloc(chudxnu_timer_callback_func_t func, uint32_t param0);
-kern_return_t chudxnu_timer_callback_enter(chud_timer_t timer, uint32_t param1, uint32_t time, uint32_t units);
-kern_return_t chudxnu_timer_callback_cancel(chud_timer_t timer);
-kern_return_t chudxnu_timer_free(chud_timer_t timer);
-
-// CHUD systemcall callback - one callback for system
-typedef kern_return_t (*chudxnu_syscall_callback_func_t)(thread_flavor_t flavor, thread_state_t tstate, mach_msg_type_number_t count);
-kern_return_t chudxnu_syscall_callback_enter(chudxnu_syscall_callback_func_t func);
-kern_return_t chudxnu_syscall_callback_cancel(void);
-
-// ********************************************************************************
-// DEPRECATED
-// ********************************************************************************
-kern_return_t chudxnu_bind_current_thread(int cpu);
-
-kern_return_t chudxnu_unbind_current_thread(void);
-
-kern_return_t chudxnu_current_thread_get_callstack(uint32_t *callStack,
-                        mach_msg_type_number_t *count,
-                        boolean_t user_only);
-                        
-thread_t chudxnu_current_act(void);
-
-// thread timer callback - one callback for system
-typedef kern_return_t (*chudxnu_thread_timer_callback_func_t)(uint32_t param);
-kern_return_t chudxnu_thread_timer_callback_enter(chudxnu_thread_timer_callback_func_t func, uint32_t param, uint32_t time, uint32_t units);
-kern_return_t chudxnu_thread_timer_callback_cancel(void);
-
-#endif /* _PPC_CHUD_XNU_H_ */
diff --git a/osfmk/ppc/commpage/commpage.h b/osfmk/ppc/commpage/commpage.h
index 5eeb30503..f6ef5d3d6 100644
--- a/osfmk/ppc/commpage/commpage.h
+++ b/osfmk/ppc/commpage/commpage.h
@@ -81,7 +81,7 @@ extern	char	*commPagePtr32;				// virt address of 32-bit commpage in kernel map
 extern	char	*commPagePtr64;				// virt address of 64-bit commpage in kernel map
 
 
-extern	void	commpage_set_timestamp(uint64_t tbr,uint32_t secs,uint32_t usecs,uint32_t ticks_per_sec);
+extern	void	commpage_set_timestamp(uint64_t tbr, uint64_t secs, uint32_t ticks_per_sec);
 extern	int		commpage_time_dcba( void );
 
 #endif	/* __ASSEMBLER__ */
diff --git a/osfmk/ppc/commpage/commpage_asm.s b/osfmk/ppc/commpage/commpage_asm.s
index 92ba18fc3..d3ea83c24 100644
--- a/osfmk/ppc/commpage/commpage_asm.s
+++ b/osfmk/ppc/commpage/commpage_asm.s
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -80,7 +80,7 @@ Ldata:
  *	***********************************************
  *
  *	Update the gettimeofday() shared data on the commpages, as follows:
- *		_COMM_PAGE_TIMESTAMP = a BSD-style pair of uint_32's for secs and usecs
+ *		_COMM_PAGE_TIMESTAMP = the clock offset at timebase (seconds)
  *		_COMM_PAGE_TIMEBASE = the timebase at which the timestamp was valid
  *		_COMM_PAGE_SEC_PER_TICK = multiply timebase ticks by this to get seconds (double)
  *	The convention is that if the timebase is 0, the data is invalid.  Because other
@@ -100,8 +100,8 @@ Ldata:
  *	When called:
  *		r3 = upper half of timebase (timebase is disabled if 0)
  *		r4 = lower half of timebase
- *		r5 = seconds part of timestamp
- *		r6 = useconds part of timestamp
+ *		r5 = upper half of timestamp
+ *		r6 = lower half of timestamp
  *		r7 = divisor (ie, timebase ticks per sec)
  *	We set up:
  *		r8 = ptr to our static data (kkBinary0, kkDouble1, kkTicksPerSec)
@@ -112,7 +112,7 @@ Ldata:
  */
  
         .align	5
-LEXT(commpage_set_timestamp)				// void commpage_set_timestamp(tbr,secs,usecs,divisor)
+LEXT(commpage_set_timestamp)				// void commpage_set_timestamp(tbr,secs,divisor)
         mfmsr	r11							// get MSR
         ori		r2,r11,MASK(MSR_FP)			// turn FP on
         mtmsr	r2
@@ -161,7 +161,7 @@ LEXT(commpage_set_timestamp)				// void commpage_set_timestamp(tbr,secs,usecs,di
         lfd		f3,kkTicksPerSec(r8)		// float new ticks_per_sec + 2**52
         lfd		f4,kkDouble1(r8)			// f4 <- double(1.0)
         mffs	f5							// save caller's FPSCR
-        mtfsfi	7,0							// clear Inexeact Exception bit, set round-to-nearest
+        mtfsfi	7,1							// clear Inexeact Exception bit, set round-to-zero
         fsub	f3,f3,f2					// get ticks_per_sec
         fdiv	f3,f4,f3					// divide 1 by ticks_per_sec to get SEC_PER_TICK
         stfd	f3,_COMM_PAGE_SEC_PER_TICK(r9)
diff --git a/osfmk/ppc/commpage/gettimeofday.s b/osfmk/ppc/commpage/gettimeofday.s
index e92b9f846..8553964d2 100644
--- a/osfmk/ppc/commpage/gettimeofday.s
+++ b/osfmk/ppc/commpage/gettimeofday.s
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -32,9 +32,6 @@
 #include <machine/cpu_capabilities.h>
 #include <machine/commpage.h>
 
-#define	USEC_PER_SEC	1000000
-
-
 /* The red zone is used to move data between GPRs and FPRs: */
 
 #define	rzTicks			-8			// elapsed ticks since timestamp (double)
@@ -54,7 +51,7 @@
 // in user mode, usually without having to make a system call.  We do not deal with
 // the timezone.  The kernel maintains the following values in the comm page:
 //
-//	_COMM_PAGE_TIMESTAMP = a BSD-style pair of uint_32's for seconds and microseconds
+//	_COMM_PAGE_TIMESTAMP = 64 bit seconds timestamp
 //
 //	_COMM_PAGE_TIMEBASE = the timebase at which the timestamp was valid
 //
@@ -75,8 +72,7 @@ gettimeofday_32:								// int gettimeofday(timeval *tp);
 0:
         lwz		r5,_COMM_PAGE_TIMEBASE+0(0)		// r5,r6 = TBR at timestamp
         lwz		r6,_COMM_PAGE_TIMEBASE+4(0)
-        lwz		r7,_COMM_PAGE_TIMESTAMP+0(0)	// r7 = timestamp seconds
-        lwz		r8,_COMM_PAGE_TIMESTAMP+4(0)	// r8 = timestamp microseconds
+        lwz		r8,_COMM_PAGE_TIMESTAMP+4(0)	// r8 = timestamp 32 bit seconds
         lfd		f1,_COMM_PAGE_SEC_PER_TICK(0)
 1:        
         mftbu	r10								// r10,r11 = current timebase
@@ -90,31 +86,28 @@ gettimeofday_32:								// int gettimeofday(timeval *tp);
         
         lwz		r0,_COMM_PAGE_TIMEBASE+0(0)		// then load data a 2nd time
         lwz		r12,_COMM_PAGE_TIMEBASE+4(0)
-        lwz		r2,_COMM_PAGE_TIMESTAMP+0(0)
         lwz		r9,_COMM_PAGE_TIMESTAMP+4(0)
         cmplw	cr6,r5,r0			// did we read a consistent set?
         cmplw	cr7,r6,r12
         beq-	3f					// timestamp is disabled so return bad status
-        cmplw	cr1,r2,r7
         cmplw	cr5,r9,r8
         crand	cr0_eq,cr6_eq,cr7_eq
-        crand	cr1_eq,cr1_eq,cr5_eq
-        crand	cr0_eq,cr0_eq,cr1_eq
+        crand	cr0_eq,cr0_eq,cr5_eq
         bne-	0b					// loop until we have a consistent set of data
         
         subfc	r11,r6,r11			// compute ticks since timestamp
         lwz		r9,_COMM_PAGE_2_TO_52(0)	// get exponent for (2**52)
         subfe	r10,r5,r10			// complete 64-bit subtract
-        lfd		f2,_COMM_PAGE_2_TO_52(0)	// f3 <- (2**52)
+        lfd		f2,_COMM_PAGE_2_TO_52(0)	// f2 <- (2**52)
         srwi.	r0,r10,2			// if more than 2**34 ticks have elapsed...
         stw		r11,rzTicks+4(r1)	// store elapsed ticks into red zone
         or		r10,r10,r9			// convert long-long in (r10,r11) into double
         bne-	3f					// ...call kernel to reprime timestamp
 
         stw		r10,rzTicks(r1)		// complete double
-        lis		r12,hi16(USEC_PER_SEC)
-        ori		r12,r12,lo16(USEC_PER_SEC)
-        
+
+		mffs	f7
+		mtfsfi	7,1        
         lfd		f3,rzTicks(r1)		// get elapsed ticks since timestamp + 2**52
         fsub	f4,f3,f2			// subtract 2**52 and normalize
         fmul	f5,f4,f1			// f5 <- elapsed seconds since timestamp
@@ -128,19 +121,14 @@ gettimeofday_32:								// int gettimeofday(timeval *tp);
         fmul	f6,f6,f3			// f6 <- fractional elapsed useconds
         fctiwz	f6,f6				// convert useconds to integer
         stfd	f6,rzUSeconds(r1)	// store useconds into red zone
+		mtfsf	0xff,f7
         
         lwz		r5,rzSeconds+4(r1)	// r5 <- seconds since timestamp
-        lwz		r6,rzUSeconds+4(r1)	// r6 <- useconds since timestamp
-        add		r7,r7,r5			// add elapsed seconds to timestamp seconds
-        add		r8,r8,r6			// ditto useconds
+        lwz		r7,rzUSeconds+4(r1)	// r7 <- useconds since timestamp
+        add		r6,r8,r5			// add elapsed seconds to timestamp seconds
         
-        cmplw	r8,r12				// r8 >= USEC_PER_SEC ?
-        blt		2f					// no
-        addi	r7,r7,1				// add 1 to secs
-        sub		r8,r8,r12			// subtract USEC_PER_SEC from usecs
-2:
-        stw		r7,0(r3)			// store secs//usecs into user's timeval
-        stw		r8,4(r3)
+        stw		r6,0(r3)			// store secs//usecs into user's timeval
+        stw		r7,4(r3)
         li		r3,0				// return success
         blr
 3:									// too long since last timestamp or this code is disabled
@@ -155,12 +143,12 @@ gettimeofday_32:								// int gettimeofday(timeval *tp);
 // ***************************************
 //
 // This routine is called in 32-bit mode on 64-bit processors.  A timeval is a struct of
-// a long seconds and int useconds, so it's size depends on mode.
+// a long seconds and int useconds, so its size depends on mode.
 
 gettimeofday_g5_32:							// int gettimeofday(timeval *tp);
 0:
         ld		r6,_COMM_PAGE_TIMEBASE(0)	// r6 = TBR at timestamp
-        ld		r8,_COMM_PAGE_TIMESTAMP(0)	// r8 = timestamp (seconds,useconds)
+        ld		r8,_COMM_PAGE_TIMESTAMP(0)	// r8 = timestamp (seconds)
         lfd		f1,_COMM_PAGE_SEC_PER_TICK(0)
         mftb	r10							// r10 = get current timebase
         lwsync								// create a barrier if MP (patched to NOP if UP)
@@ -178,6 +166,8 @@ gettimeofday_g5_32:							// int gettimeofday(timeval *tp);
         std		r11,rzTicks(r1)		// put ticks in redzone where we can "lfd" it
         bne--	3f					// timestamp too old, so reprime
 
+		mffs	f7
+		mtfsfi	7,1
         lfd		f3,rzTicks(r1)		// get elapsed ticks since timestamp (fixed pt)
         fcfid	f4,f3				// float the tick count
         fmul	f5,f4,f1			// f5 <- elapsed seconds since timestamp
@@ -189,22 +179,14 @@ gettimeofday_g5_32:							// int gettimeofday(timeval *tp);
         fmul	f6,f6,f3			// f6 <- fractional elapsed useconds
         fctidz	f6,f6				// convert useconds to fixed pt integer
         stfd	f6,rzUSeconds(r1)	// store useconds into red zone
+		mtfsf	0xff,f7
         
-        lis		r12,hi16(USEC_PER_SEC)	// r12 <- 10**6
-        srdi	r7,r8,32			// extract seconds from doubleword timestamp
         lwz		r5,rzSeconds+4(r1)	// r5 <- seconds since timestamp
-        ori		r12,r12,lo16(USEC_PER_SEC)
-        lwz		r6,rzUSeconds+4(r1)	// r6 <- useconds since timestamp
-        add		r7,r7,r5			// add elapsed seconds to timestamp seconds
-        add		r8,r8,r6			// ditto useconds
+        lwz		r7,rzUSeconds+4(r1)	// r7 <- useconds since timestamp
+        add		r6,r8,r5			// add elapsed seconds to timestamp seconds
         
-        cmplw	r8,r12				// r8 >= USEC_PER_SEC ?
-        blt		2f					// no
-        addi	r7,r7,1				// add 1 to secs
-        sub		r8,r8,r12			// subtract USEC_PER_SEC from usecs
-2:
-        stw		r7,0(r3)			// store secs//usecs into user's timeval
-        stw		r8,4(r3)
+        stw		r6,0(r3)			// store secs//usecs into user's timeval
+        stw		r7,4(r3)
         li		r3,0				// return success
         blr
 3:									// too long since last timestamp or this code is disabled
@@ -219,12 +201,12 @@ gettimeofday_g5_32:							// int gettimeofday(timeval *tp);
 // ***************************************
 //
 // This routine is called in 64-bit mode on 64-bit processors.  A timeval is a struct of
-// a long seconds and int useconds, so it's size depends on mode.
+// a long seconds and int useconds, so its size depends on mode.
 
 gettimeofday_g5_64:							// int gettimeofday(timeval *tp);
 0:
         ld		r6,_COMM_PAGE_TIMEBASE(0)	// r6 = TBR at timestamp
-        ld		r8,_COMM_PAGE_TIMESTAMP(0)	// r8 = timestamp (seconds,useconds)
+        ld		r8,_COMM_PAGE_TIMESTAMP(0)	// r8 = timestamp (seconds)
         lfd		f1,_COMM_PAGE_SEC_PER_TICK(0)
         mftb	r10							// r10 = get current timebase
         lwsync								// create a barrier if MP (patched to NOP if UP)
@@ -242,6 +224,8 @@ gettimeofday_g5_64:							// int gettimeofday(timeval *tp);
         std		r11,rzTicks(r1)		// put ticks in redzone where we can "lfd" it
         bne--	3f					// timestamp too old, so reprime
 
+		mffs	f7
+		mtfsfi	7,1
         lfd		f3,rzTicks(r1)		// get elapsed ticks since timestamp (fixed pt)
         fcfid	f4,f3				// float the tick count
         fmul	f5,f4,f1			// f5 <- elapsed seconds since timestamp
@@ -253,22 +237,14 @@ gettimeofday_g5_64:							// int gettimeofday(timeval *tp);
         fmul	f6,f6,f3			// f6 <- fractional elapsed useconds
         fctidz	f6,f6				// convert useconds to fixed pt integer
         stfd	f6,rzUSeconds(r1)	// store useconds into red zone
+		mtfsf	0xff,f7
         
-        lis		r12,hi16(USEC_PER_SEC)	// r12 <- 10**6
-        srdi	r7,r8,32			// extract seconds from doubleword timestamp
         lwz		r5,rzSeconds+4(r1)	// r5 <- seconds since timestamp
-        ori		r12,r12,lo16(USEC_PER_SEC)
-        lwz		r6,rzUSeconds+4(r1)	// r6 <- useconds since timestamp
-        add		r7,r7,r5			// add elapsed seconds to timestamp seconds
-        add		r8,r8,r6			// ditto useconds
+        lwz		r7,rzUSeconds+4(r1)	// r7 <- useconds since timestamp
+        add		r6,r8,r5			// add elapsed seconds to timestamp seconds
         
-        cmplw	r8,r12				// r8 >= USEC_PER_SEC ?
-        blt		2f					// no
-        addi	r7,r7,1				// add 1 to secs
-        sub		r8,r8,r12			// subtract USEC_PER_SEC from usecs
-2:
-        std		r7,0(r3)			// store secs//usecs into user's timeval
-        stw		r8,8(r3)
+        std		r6,0(r3)			// store secs//usecs into user's timeval
+        stw		r7,8(r3)
         li		r3,0				// return success
         blr
 3:									// too long since last timestamp or this code is disabled
diff --git a/osfmk/ppc/conf.c b/osfmk/ppc/conf.c
index f0900c4d8..395b4770c 100644
--- a/osfmk/ppc/conf.c
+++ b/osfmk/ppc/conf.c
@@ -77,10 +77,10 @@ extern	struct clock_ops	sysclk_ops, calend_ops;
 struct	clock	clock_list[] = {
 
 	/* SYSTEM_CLOCK */
-	{ &sysclk_ops,		0,	0,	0 },
+	{ &sysclk_ops, 0, 0 },
 
 	/* CALENDAR_CLOCK */
-	{ &calend_ops,		0,	0,	0 },
+	{ &calend_ops, 0, 0 },
 };
 int	clock_count = sizeof(clock_list) / sizeof(clock_list[0]);
 
diff --git a/osfmk/ppc/cpu.c b/osfmk/ppc/cpu.c
index 5ad083250..914ea83f3 100644
--- a/osfmk/ppc/cpu.c
+++ b/osfmk/ppc/cpu.c
@@ -37,6 +37,7 @@
 #include <kern/thread.h>
 #include <kern/sched_prim.h>
 #include <kern/processor.h>
+#include <kern/pms.h>
 
 #include <vm/pmap.h>
 #include <IOKit/IOHibernatePrivate.h>
@@ -54,7 +55,6 @@
 #include <ppc/Diagnostics.h>
 #include <ppc/trap.h>
 #include <ppc/machine_cpu.h>
-#include <ppc/pms.h>
 #include <ppc/rtclock.h>
 
 decl_mutex_data(static,ppt_lock);
@@ -71,9 +71,9 @@ static unsigned int	rht_state = 0;
 decl_simple_lock_data(static,SignalReadyLock);
 
 struct SIGtimebase {
-	boolean_t	avail;
-	boolean_t	ready;
-	boolean_t	done;
+	volatile boolean_t	avail;
+	volatile boolean_t	ready;
+	volatile boolean_t	done;
 	uint64_t	abstime;
 };
 
@@ -126,8 +126,9 @@ cpu_init(
 		mttbu(proc_info->save_tbu);
 		mttb(proc_info->save_tbl);
 	}
-	
-	setTimerReq();				/* Now that the time base is sort of correct, request the next timer pop */
+
+	proc_info->rtcPop = EndOfAllTime;			/* forget any existing decrementer setting */
+	etimer_resync_deadlines();				/* Now that the time base is sort of correct, request the next timer pop */
 
 	proc_info->cpu_type = CPU_TYPE_POWERPC;
 	proc_info->cpu_subtype = (cpu_subtype_t)proc_info->pf.rptdProc;
@@ -711,7 +712,7 @@ cpu_sync_timebase(
 							(unsigned int)&syncClkSpot) != KERN_SUCCESS)
 		continue;
 
-	while (*(volatile int *)&(syncClkSpot.avail) == FALSE)
+	while (syncClkSpot.avail == FALSE)
 		continue;
 
 	isync();
@@ -729,11 +730,10 @@ cpu_sync_timebase(
 
 	syncClkSpot.ready = TRUE;
 
-	while (*(volatile int *)&(syncClkSpot.done) == FALSE)
+	while (syncClkSpot.done == FALSE)
 		continue;
 
-	setTimerReq();									/* Start the timer */
-	
+	etimer_resync_deadlines();									/* Start the timer */
 	(void)ml_set_interrupts_enabled(intr);
 }
 
@@ -766,7 +766,8 @@ cpu_timebase_signal_handler(
 						
 	timebaseAddr->avail = TRUE;
 
-	while (*(volatile int *)&(timebaseAddr->ready) == FALSE);
+	while (timebaseAddr->ready == FALSE)
+		continue;
 
 	if(proc_info->time_base_enable !=  (void(*)(cpu_id_t, boolean_t ))NULL)
 		proc_info->time_base_enable(proc_info->cpu_id, TRUE);
diff --git a/osfmk/ppc/cpu_capabilities.h b/osfmk/ppc/cpu_capabilities.h
index e11feacc0..6afde90c3 100644
--- a/osfmk/ppc/cpu_capabilities.h
+++ b/osfmk/ppc/cpu_capabilities.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -97,6 +97,17 @@ static __inline__ int _NumCPUs( void ) { return (_cpu_capabilities & kNumCPUs) >
 #define	_COMM_PAGE_AREA_LENGTH			( 7*4096)						// reserved length of entire comm area
 #define	_COMM_PAGE_AREA_USED			( 2*4096)						// we use two pages so far
 
+/* The following set of definitions are used in the kernel, which needs to distinguish between
+ * the 32 and 64-bit commpage addresses and lengths.  On PPC they are the same, but on Imtel
+ * they are not.
+ */
+#define _COMM_PAGE32_BASE_ADDRESS		( _COMM_PAGE_BASE_ADDRESS )
+#define _COMM_PAGE64_BASE_ADDRESS		( _COMM_PAGE_BASE_ADDRESS )
+#define	_COMM_PAGE32_AREA_LENGTH		( _COMM_PAGE_AREA_LENGTH )
+#define	_COMM_PAGE64_AREA_LENGTH		( _COMM_PAGE_AREA_LENGTH )
+#define	_COMM_PAGE32_AREA_USED			( _COMM_PAGE_AREA_USED )
+#define	_COMM_PAGE64_AREA_USED			( _COMM_PAGE_AREA_USED )
+
 /* The Objective-C runtime fixed address page to optimize message dispatch */
 #define _OBJC_PAGE_BASE_ADDRESS			(-20*4096)						// start at page -20, ie 0xFFFEC000
  
diff --git a/osfmk/ppc/db_interface.c b/osfmk/ppc/db_interface.c
index f7c735697..4963b9a57 100644
--- a/osfmk/ppc/db_interface.c
+++ b/osfmk/ppc/db_interface.c
@@ -591,15 +591,3 @@ void db_reboot(
 	db_printf("Sorry, system can't reboot automatically yet...  You need to do it by hand...\n");
 
 }
-
-/*
- * Switch to gdb
- */
-void
-db_to_gdb(
-	void)
-{
-	extern unsigned int switch_debugger;
-
-	switch_debugger=1;
-}
diff --git a/osfmk/ppc/db_machdep.h b/osfmk/ppc/db_machdep.h
index 80cf7669a..6eb62c17a 100644
--- a/osfmk/ppc/db_machdep.h
+++ b/osfmk/ppc/db_machdep.h
@@ -158,8 +158,6 @@ extern void		db_low_trace(
 				int		have_addr,
 				db_expr_t	count,
 				char 		*modif);
-extern void		db_to_gdb(
-				void);
 
 
 /* macros for printing OS server dependent task name */
diff --git a/osfmk/ppc/exception.h b/osfmk/ppc/exception.h
index f75e5e2db..f4e9da93f 100644
--- a/osfmk/ppc/exception.h
+++ b/osfmk/ppc/exception.h
@@ -47,10 +47,10 @@
 #include <mach/boolean.h>
 #include <kern/ast.h>
 #include <kern/cpu_data.h>
+#include <kern/pms.h>
 #include <pexpert/pexpert.h>
 #include <IOKit/IOInterrupts.h>
 #include <ppc/machine_routines.h>
-#include <ppc/pms.h>
 #include <ppc/rtclock.h>
 
 /*	Per processor CPU features */
@@ -349,7 +349,7 @@ struct per_proc_info {
 	/* PPC cache line boundary here - 140 */
 	void *			pp_cbfr;
 	void *			pp_chud;
-	uint64_t		rtclock_tick_deadline;
+	uint64_t		rtclock_intr_deadline;
 	rtclock_timer_t	rtclock_timer;
 	unsigned int	ppbbTaskEnv;		/* BlueBox Task Environment */
     
diff --git a/osfmk/ppc/hibernate_ppc.c b/osfmk/ppc/hibernate_ppc.c
index e90a52b0e..fad4f440c 100644
--- a/osfmk/ppc/hibernate_ppc.c
+++ b/osfmk/ppc/hibernate_ppc.c
@@ -119,6 +119,14 @@ hibernate_page_list_setall_machine(hibernate_page_list_t * page_list,
     save_snapshot();
 }
 
+// mark pages not to be saved and not for scratch usage during restore
+void
+hibernate_page_list_set_volatile( hibernate_page_list_t * page_list,
+				  hibernate_page_list_t * page_list_wired,
+				  uint32_t * pagesOut)
+{
+}
+
 kern_return_t 
 hibernate_processor_setup(IOHibernateImageHeader * header)
 {
@@ -196,3 +204,7 @@ void ml_ppc_sleep(void)
     }
 }
 
+void
+hibernate_newruntime_map(void * map, vm_size_t map_size, uint32_t runtime_offset)
+{
+}
diff --git a/osfmk/ppc/hw_lock.s b/osfmk/ppc/hw_lock.s
index 609c5b9be..931c5480b 100644
--- a/osfmk/ppc/hw_lock.s
+++ b/osfmk/ppc/hw_lock.s
@@ -38,6 +38,7 @@
 #define	WAIT_FLAG		0x02
 #define	WANT_UPGRADE	0x04
 #define	WANT_EXCL		0x08
+#define	PRIV_EXCL		0x8000
 
 #define TH_FN_OWNED		0x01
 
@@ -2010,8 +2011,10 @@ LEXT(lck_rw_lock_exclusive)
 			.globl	EXT(lock_write)
 LEXT(lock_write)
 #endif
+			lis		r7,0xFFFF
+			ori		r7,r7,(WANT_EXCL|WANT_UPGRADE|ILK_LOCKED)
 rwleloop:	lwarx	r5,RW_DATA,r3					; Grab the lock value
-			rlwinm.	r7,r5,30,1,31					; Can we have it?
+			and.	r8,r5,r7						; Can we have it?
 			ori		r6,r5,WANT_EXCL					; Mark Exclusive
 			bne--	rwlespin						; Branch if cannot be held
 			stwcx.	r6,RW_DATA,r3					; Update lock word
@@ -2044,14 +2047,21 @@ LEXT(lock_read)
 #endif
 rwlsloop:	lwarx	r5,RW_DATA,r3					; Grab the lock value
 			andi.	r7,r5,WANT_EXCL|WANT_UPGRADE|ILK_LOCKED	; Can we have it?
+			bne--	rwlsopt							; Branch if cannot be held
+rwlsloopres:
 			addis	r6,r5,1							; Increment read cnt
-			bne--	rwlsspin						; Branch if cannot be held
 			stwcx.	r6,RW_DATA,r3					; Update lock word
 			bne--	rwlsloop
 			.globl  EXT(rwlsPatch_isync)
 LEXT(rwlsPatch_isync)
 			isync
 			blr
+rwlsopt:
+			andi.	r7,r5,PRIV_EXCL|ILK_LOCKED		; Can we have it?
+			bne--	rwlsspin						; Branch if cannot be held
+			lis		r7,0xFFFF						; Get read cnt mask
+			and.	r8,r5,r7						; Is it shared
+			bne		rwlsloopres						; Branch if can be held
 rwlsspin:
 			li		r4,lgKillResv					; Killing field
 			stwcx.	r4,0,r4							; Kill it
@@ -2198,8 +2208,9 @@ rwtlsloop:	lwarx	r5,RW_DATA,r3					; Grab the lock value
 			andi.	r7,r5,ILK_LOCKED				; Test interlock flag
 			bne--	rwtlsspin						; Branch if interlocked
 			andi.	r7,r5,WANT_EXCL|WANT_UPGRADE	; So, can we have it?
+			bne--	rwtlsopt						; Branch if held exclusive
+rwtlsloopres:
 			addis	r6,r5,1							; Increment read cnt
-			bne--	rwtlsfail						; Branch if held exclusive
 			stwcx.	r6,RW_DATA,r3					; Update lock word
 			bne--	rwtlsloop
 			.globl  EXT(rwtlsPatch_isync)
@@ -2207,6 +2218,12 @@ LEXT(rwtlsPatch_isync)
 			isync
 			li		r3,1							; Return TRUE
 			blr
+rwtlsopt:
+			andi.	r7,r5,PRIV_EXCL					; Can we have it?
+			bne--	rwtlsfail						; Branch if cannot be held
+			lis		r7,0xFFFF						; Get read cnt mask
+			and.	r8,r5,r7						; Is it shared
+			bne		rwtlsloopres					; Branch if can be held
 rwtlsfail:
 			li		r3,0							; Return FALSE
 			blr
diff --git a/osfmk/ppc/hw_vm.s b/osfmk/ppc/hw_vm.s
index 61eb76f51..a85ed3c9f 100644
--- a/osfmk/ppc/hw_vm.s
+++ b/osfmk/ppc/hw_vm.s
@@ -2874,17 +2874,19 @@ hwpSPrtPhy: cmplw	r0,r0						; Make sure we return CR0_EQ
 
 ;			Function 2 - Set protection in mapping
 
+;			NOTE: Changes to no-execute permission are ignored
+
 			.set	.,hwpOpBase+(2*128)			; Generate error if previous function too long
 
 hwpSPrtMap:	lwz		r9,mpFlags(r31)				; Get the mapping flags
 			lwz		r8,mpVAddr+4(r31)			; Get the protection part of mapping
 			rlwinm.	r9,r9,0,mpPermb,mpPermb		; Is the mapping permanent?
-			li		r0,lo16(mpN|mpPP)			; Get no-execute and protection bits
+			li		r0,lo16(mpPP)				; Get protection bits
 			crnot	cr0_eq,cr0_eq				; Change CR0_EQ to true if mapping is permanent
-			rlwinm	r2,r25,0,mpNb-32,mpPPe-32	; Isolate new no-execute and protection bits 
+			rlwinm	r2,r25,0,mpPP				; Isolate new protection bits 
 			beqlr--								; Leave if permanent mapping (before we trash R5)...
-			andc	r5,r5,r0					; Clear the old no-execute and prot bits
-			or		r5,r5,r2					; Move in the new no-execute and prot bits
+			andc	r5,r5,r0					; Clear the old prot bits
+			or		r5,r5,r2					; Move in the new prot bits
 			rlwimi	r8,r5,0,20,31				; Copy into the mapping copy
 			cmpw	r0,r0						; Make sure we return CR0_EQ
 			stw		r8,mpVAddr+4(r31)			; Set the flag part of mapping
diff --git a/osfmk/ppc/interrupt.c b/osfmk/ppc/interrupt.c
index 43860ff26..0ea76f818 100644
--- a/osfmk/ppc/interrupt.c
+++ b/osfmk/ppc/interrupt.c
@@ -35,6 +35,8 @@
 #include <kern/assert.h>
 #include <kern/thread.h>
 #include <kern/counters.h>
+#include <kern/etimer.h>
+#include <kern/pms.h>
 #include <ppc/misc_protos.h>
 #include <ppc/trap.h>
 #include <ppc/proc_reg.h>
@@ -109,7 +111,7 @@ struct savearea * interrupt(
 				}
 			}
 
-			rtclock_intr(ssp);
+			etimer_intr(USER_MODE(ssp->save_srr1), ssp->save_srr0);	/* Handle event timer */
 			break;
 	
 		case T_INTERRUPT:
diff --git a/osfmk/ppc/io_map.c b/osfmk/ppc/io_map.c
index 472063dae..67889dcb2 100644
--- a/osfmk/ppc/io_map.c
+++ b/osfmk/ppc/io_map.c
@@ -52,15 +52,14 @@ extern vm_offset_t	virtual_avail;
  * Note, this will onl
  */
 vm_offset_t
-io_map(phys_addr, size)
-	vm_offset_t	phys_addr;
-	vm_size_t	size;
+io_map(vm_offset_t phys_addr, vm_size_t size, unsigned int flags)
 {
 	vm_offset_t	start;
 	int		i;
-	unsigned int j;
+	unsigned int    j, mflags;
 	vm_page_t 	m;
 
+	mflags = mmFlgBlock | mmFlgUseAttr | (flags & VM_MEM_GUARDED) | ((flags & VM_MEM_NOT_CACHEABLE) >> 1);	/* Convert to our mapping_make flags */
 
 #if DEBUG
 	assert (kernel_map != VM_MAP_NULL);			/* VM must be initialised */
@@ -73,7 +72,7 @@ io_map(phys_addr, size)
 		(void) kmem_alloc_pageable(kernel_map, &start, size);	/* Get some virtual addresses to use */
 		
 		(void)mapping_make(kernel_pmap, (addr64_t)start, (ppnum_t)(phys_addr >> 12), 
-			(mmFlgBlock | mmFlgUseAttr | mmFlgCInhib | mmFlgGuarded),	/* Map as I/O page */
+			mflags,					/* Map with requested cache mode */
 			(size >> 12), VM_PROT_READ|VM_PROT_WRITE);
 
 		return (start + (phys_addr & PAGE_MASK));	/* Pass back the physical address */
@@ -93,7 +92,7 @@ io_map(phys_addr, size)
 			
 			(void)mapping_make(kernel_pmap, 
 				(addr64_t)(start + i), m->phys_page, 
-				(mmFlgBlock | mmFlgUseAttr | mmFlgCInhib | mmFlgGuarded),	/* Map as I/O page */
+				mflags,					/* Map with requested cache mode */
 				1, VM_PROT_READ|VM_PROT_WRITE);	
 			
 		}
@@ -108,21 +107,24 @@ io_map(phys_addr, size)
  * Allocate and map memory for devices before the VM system comes alive.
  */
 
-vm_offset_t io_map_spec(vm_offset_t phys_addr, vm_size_t size)
+vm_offset_t io_map_spec(vm_offset_t phys_addr, vm_size_t size, unsigned int flags)
 {
 	vm_offset_t	start;
+	unsigned int    mflags;
 
 	if(kernel_map != VM_MAP_NULL) {				/* If VM system is up, redirect to normal routine */
 		
-		return io_map(phys_addr, size);			/* Map the address */
+		return io_map(phys_addr, size, flags);			/* Map the address */
 	
 	}
+
+	mflags = mmFlgBlock | mmFlgUseAttr | (flags & VM_MEM_GUARDED) | ((flags & VM_MEM_NOT_CACHEABLE) >> 1);	/* Convert to our mapping_make flags */
 	
 	size = round_page(size + (phys_addr - (phys_addr & -PAGE_SIZE)));	/* Extend the length to include it all */
 	start = pmap_boot_map(size);				/* Get me some virtual address */
 
 	(void)mapping_make(kernel_pmap, (addr64_t)start, (ppnum_t)(phys_addr >> 12), 
-		(mmFlgBlock | mmFlgUseAttr | mmFlgCInhib | mmFlgGuarded),	/* Map as I/O page */
+		mflags,					/* Map with requested cache mode */
 		(size >> 12), VM_PROT_READ|VM_PROT_WRITE);
 
 	return (start + (phys_addr & PAGE_MASK));
diff --git a/osfmk/ppc/io_map_entries.h b/osfmk/ppc/io_map_entries.h
index 5d61a064f..eb0adf48a 100644
--- a/osfmk/ppc/io_map_entries.h
+++ b/osfmk/ppc/io_map_entries.h
@@ -36,8 +36,9 @@
 
 extern vm_offset_t	io_map(
 				vm_offset_t		phys_addr,
-				vm_size_t		size);
-extern vm_offset_t io_map_spec(vm_offset_t phys_addr, vm_size_t size);
+				vm_size_t		size,
+				unsigned int            flags);
+extern vm_offset_t io_map_spec(vm_offset_t phys_addr, vm_size_t size, unsigned int flags);
 
 #endif	/* _PPC_IO_MAP_ENTRIES_H_ */
 
diff --git a/osfmk/ppc/locks.h b/osfmk/ppc/locks.h
index 1e7964aab..366cdb8d2 100644
--- a/osfmk/ppc/locks.h
+++ b/osfmk/ppc/locks.h
@@ -140,7 +140,8 @@ typedef struct {
 	union {
 		struct {
 			unsigned int			lck_rwd_shared_cnt:16,	/* No. of shared granted request */
-									lck_rwd_pad16:12,		/* padding */
+									lck_rwd_priv_excl:1,	/* priority for Writer */
+									lck_rwd_pad17:11,		/* padding */
 									lck_rwd_want_excl:1,	/* Writer is waiting, or locked for write */
 									lck_rwd_want_upgrade:1,	/* Read-to-write upgrade waiting */
 									lck_rwd_waiting:1,		/* Someone is sleeping on lock */
@@ -160,6 +161,7 @@ typedef struct {
 #define	lck_rw_want_upgrade		lck_rw_sw.lck_rwd.lck_rwd_want_upgrade
 #define	lck_rw_want_excl		lck_rw_sw.lck_rwd.lck_rwd_want_excl
 #define	lck_rw_waiting			lck_rw_sw.lck_rwd.lck_rwd_waiting
+#define	lck_rw_priv_excl		lck_rw_sw.lck_rwd.lck_rwd_priv_excl
 #define	lck_rw_shared_cnt		lck_rw_sw.lck_rwd.lck_rwd_shared_cnt
 
 #define lck_rw_tag				lck_rw_sw.lck_rwi.lck_rwi_tag
diff --git a/osfmk/ppc/locks_ppc.c b/osfmk/ppc/locks_ppc.c
index 8f40994ac..8d81cd09b 100644
--- a/osfmk/ppc/locks_ppc.c
+++ b/osfmk/ppc/locks_ppc.c
@@ -710,6 +710,9 @@ lock_init(
 #if	MACH_LDEBUG
 	lck->lck_rw_deb.type = RW_TAG;
 	lck->lck_rw_attr |= (LCK_RW_ATTR_DEBUG|LCK_RW_ATTR_DIS_THREAD|LCK_RW_ATTR_DIS_MYLOCK);
+	lck->lck_rw.lck_rw_priv_excl = TRUE;
+#else
+	lck->lck_rw_priv_excl = TRUE;
 #endif
 
 }
@@ -816,6 +819,10 @@ lck_rw_init(
 		}
 	} else {
 		(void) memset((void *) lck, 0, sizeof(lck_rw_t));
+		if ((lck_attr->lck_attr_val)  & LCK_ATTR_RW_SHARED_PRIORITY)
+			lck->lck_rw_priv_excl = FALSE;
+		else
+			lck->lck_rw_priv_excl = TRUE;
 	}
 
 	lck_grp_reference(grp);
@@ -832,6 +839,10 @@ lck_rw_ext_init(
 	lck_attr_t	*attr) {
 
 	bzero((void *)lck, sizeof(lck_rw_ext_t));
+	if ((attr->lck_attr_val)  & LCK_ATTR_RW_SHARED_PRIORITY)
+		lck->lck_rw.lck_rw_priv_excl = FALSE;
+	else
+		lck->lck_rw.lck_rw_priv_excl = TRUE;
 
 	if ((attr->lck_attr_val) & LCK_ATTR_DEBUG) {
 		lck->lck_rw_deb.type = RW_TAG;
@@ -1089,7 +1100,8 @@ lck_rw_lock_shared_gen(
 
 	lck_rw_ilk_lock(lck);
 
-	while (lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) {
+	while ((lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) &&
+	        ((lck->lck_rw_shared_cnt == 0) || (lck->lck_rw_priv_excl))) {
 		i = lock_wait_time[1];
 
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_CODE) | DBG_FUNC_START,
@@ -1097,12 +1109,15 @@ lck_rw_lock_shared_gen(
 
 		if (i != 0) {
 			lck_rw_ilk_unlock(lck);
-			while (--i != 0 && (lck->lck_rw_want_excl || lck->lck_rw_want_upgrade))
+			while (--i != 0 && 
+			       (lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) &&
+			       ((lck->lck_rw_shared_cnt == 0) || (lck->lck_rw_priv_excl)))
 				continue;
 			lck_rw_ilk_lock(lck);
 		}
 
-		if (lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) {
+		if ((lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) &&
+		    ((lck->lck_rw_shared_cnt == 0) || (lck->lck_rw_priv_excl))) {
 			lck->lck_rw_waiting = TRUE;
 			res = assert_wait((event_t)(((unsigned int*)lck)+((sizeof(lck_rw_t)-1)/sizeof(unsigned int))), THREAD_UNINT);
 			if (res == THREAD_WAITING) {
@@ -1279,7 +1294,8 @@ lck_rw_try_lock_shared_gen(
 {
 	lck_rw_ilk_lock(lck);
 
-	if (lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) {
+	if ((lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) &&
+	    ((lck->lck_rw_shared_cnt == 0) || (lck->lck_rw_priv_excl))) {
 		lck_rw_ilk_unlock(lck);
 		return(FALSE);
 	}
@@ -1515,7 +1531,8 @@ lck_rw_lock_shared_ext(
 	if (lock_stat)
 		lck->lck_rw_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_util_cnt++;
 
-	while (lck->lck_rw.lck_rw_want_excl || lck->lck_rw.lck_rw_want_upgrade) {
+	while ((lck->lck_rw.lck_rw_want_excl || lck->lck_rw.lck_rw_want_upgrade) &&
+	       ((lck->lck_rw.lck_rw_shared_cnt == 0) || (lck->lck_rw.lck_rw_priv_excl))) {
 		i = lock_wait_time[1];
 
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_CODE) | DBG_FUNC_START,
@@ -1528,12 +1545,15 @@ lck_rw_lock_shared_ext(
 
 		if (i != 0) {
 			lck_rw_ilk_unlock(&lck->lck_rw);
-			while (--i != 0 && (lck->lck_rw.lck_rw_want_excl || lck->lck_rw.lck_rw_want_upgrade))
+			while (--i != 0 && 
+			       (lck->lck_rw.lck_rw_want_excl || lck->lck_rw.lck_rw_want_upgrade) &&
+	       		       ((lck->lck_rw.lck_rw_shared_cnt == 0) || (lck->lck_rw.lck_rw_priv_excl)))
 				continue;
 			lck_rw_ilk_lock(&lck->lck_rw);
 		}
 
-		if (lck->lck_rw.lck_rw_want_excl || lck->lck_rw.lck_rw_want_upgrade) {
+		if ((lck->lck_rw.lck_rw_want_excl || lck->lck_rw.lck_rw_want_upgrade)  &&
+		   ((lck->lck_rw.lck_rw_shared_cnt == 0) || (lck->lck_rw.lck_rw_priv_excl))) {
 			lck->lck_rw.lck_rw_waiting = TRUE;
 			res = assert_wait((event_t)(((unsigned int*)rlck)+((sizeof(lck_rw_t)-1)/sizeof(unsigned int))), THREAD_UNINT);
 			if (res == THREAD_WAITING) {
@@ -1790,7 +1810,8 @@ lck_rw_try_lock_shared_ext(
 	if (lock_stat)
 		lck->lck_rw_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_util_cnt++;
 
-	if (lck->lck_rw.lck_rw_want_excl || lck->lck_rw.lck_rw_want_upgrade) {
+	if ((lck->lck_rw.lck_rw_want_excl || lck->lck_rw.lck_rw_want_upgrade) &&
+	    ((lck->lck_rw.lck_rw_shared_cnt == 0) || (lck->lck_rw.lck_rw_priv_excl))) {
 		if (lock_stat) {
 			lck->lck_rw_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_miss_cnt++;
 		}
diff --git a/osfmk/ppc/lowglobals.h b/osfmk/ppc/lowglobals.h
index 5895029fa..fec42c690 100644
--- a/osfmk/ppc/lowglobals.h
+++ b/osfmk/ppc/lowglobals.h
@@ -86,8 +86,12 @@ typedef struct lowglo {
 	unsigned int	lgRsv380[32];			/* 5380 - 5400 reserved  */
 
 	unsigned int	lgRsv400[32];			/* 5400 - 5480 reserved  */
-
-	uint32_t		lgRsv480[704];			/* 5480 reserved - push to 1 page */
+	uint32_t		lgKmodptr;		/* 0x5480 Pointer to kmod, debugging aid */
+	uint32_t		lgTransOff;		/* 0x5484 Pointer to kdp_trans_off, debugging aid */
+	uint32_t		lgReadIO;		/* 0x5488 Pointer to kdp_read_io, debugging aid */
+	uint32_t		lgDevSlot1;		/* 0x548C For developer use */
+	uint32_t		lgDevSlot2;		/* 0x5490 For developer use */
+	uint32_t		lgRsv494[731];		/* 0x5494 reserved - push to 1 page */
 
 } lowglo;
 
diff --git a/osfmk/ppc/lowmem_vectors.s b/osfmk/ppc/lowmem_vectors.s
index a52d7ee57..a63398fd6 100644
--- a/osfmk/ppc/lowmem_vectors.s
+++ b/osfmk/ppc/lowmem_vectors.s
@@ -1554,7 +1554,7 @@ noPerfMonSave32:
 			lwz		r25,traceMask(0)				; Get the trace mask
 			li		r0,SAVgeneral					; Get the savearea type value
 			lhz		r19,PP_CPU_NUMBER(r2)			; Get the logical processor number											
-			rlwinm	r22,r11,30,0,31					; Divide interrupt code by 2
+			rlwinm	r22,r11,30,0,31					; Divide interrupt code by 4
 			stb		r0,SAVflags+2(r13)				; Mark valid context
 			addi	r22,r22,10						; Adjust code so we shift into CR5
 			li		r23,trcWork						; Get the trace work area address
@@ -3965,7 +3965,12 @@ EXT(killresv):
 			.long	0								; 5470 reserved			
 			.long	0								; 5474 reserved			
 			.long	0								; 5478 reserved			
-			.long	0								; 547C reserved	
+			.long	0								; 547C reserved
+			.long	EXT(kmod)							; 5480 Pointer to kmod, debugging aid
+			.long	EXT(kdp_trans_off)						; 5484 Pointer to kdp_trans_off, debugging aid
+			.long	EXT(kdp_read_io)						; 5488 Pointer to kdp_read_io, debugging aid
+			.long	0								; 548C Reserved for developer use
+			.long	0								; 5490 Reserved for developer use
 ;
 ;	The "shared page" is used for low-level debugging
 ;
diff --git a/osfmk/ppc/machine_cpu.h b/osfmk/ppc/machine_cpu.h
index f043f6d3d..88fe14def 100644
--- a/osfmk/ppc/machine_cpu.h
+++ b/osfmk/ppc/machine_cpu.h
@@ -46,4 +46,6 @@ typedef void (*broadcastFunc) (uint32_t);
 
 int32_t cpu_broadcast(uint32_t *, broadcastFunc, uint32_t);
 
+#define cpu_pause()		/* Not for this architecture */
+
 #endif /* _PPC_MACHINE_CPU_H_ */
diff --git a/osfmk/ppc/machine_routines.c b/osfmk/ppc/machine_routines.c
index b6a6ce457..6e2d0bdda 100644
--- a/osfmk/ppc/machine_routines.c
+++ b/osfmk/ppc/machine_routines.c
@@ -45,7 +45,7 @@
 
 #include <vm/vm_page.h>
 
-unsigned int		LockTimeOut = 12500000;
+unsigned int		LockTimeOut = 1250000000;
 unsigned int		MutexSpin = 0;
 
 decl_mutex_data(static,mcpus_lock);
@@ -148,9 +148,17 @@ ml_io_map(
 	vm_offset_t phys_addr, 
 	vm_size_t size)
 {
-	return(io_map(phys_addr,size));
+	return(io_map(phys_addr,size,VM_WIMG_IO));
 }
 
+
+void ml_get_bouncepool_info(vm_offset_t *phys_addr, vm_size_t *size)
+{
+        *phys_addr = 0;
+	*size      = 0;
+}
+
+
 /*
  *	Routine:        ml_static_malloc
  *	Function: 	static memory allocation
diff --git a/osfmk/ppc/machine_routines.h b/osfmk/ppc/machine_routines.h
index 827e1b2c4..f154e0f50 100644
--- a/osfmk/ppc/machine_routines.h
+++ b/osfmk/ppc/machine_routines.h
@@ -218,6 +218,11 @@ extern vm_offset_t		ml_io_map(
 							vm_offset_t			phys_addr, 
 							vm_size_t			size);
 
+void	ml_get_bouncepool_info(
+        vm_offset_t *phys_addr,
+	vm_size_t   *size);
+
+
 /* boot memory allocation */
 extern vm_offset_t		ml_static_malloc(
 							vm_size_t			size);
diff --git a/osfmk/ppc/machine_routines_asm.s b/osfmk/ppc/machine_routines_asm.s
index 938a5e081..6d771f7d6 100644
--- a/osfmk/ppc/machine_routines_asm.s
+++ b/osfmk/ppc/machine_routines_asm.s
@@ -1975,19 +1975,6 @@ LEXT(current_act)
 
 			mfsprg	r3,1
 			blr
-
-			.align	5
-			.globl	EXT(clock_get_uptime)
-LEXT(clock_get_uptime)
-1:			mftbu	r9
-			mftb	r0
-			mftbu	r11
-			cmpw	r11,r9
-			bne--	1b
-			stw		r0,4(r3)
-			stw		r9,0(r3)
-			blr
-
 		
 			.align	5
 			.globl	EXT(mach_absolute_time)
diff --git a/osfmk/ppc/mappings.c b/osfmk/ppc/mappings.c
index bb5d28c94..add0e0eb4 100644
--- a/osfmk/ppc/mappings.c
+++ b/osfmk/ppc/mappings.c
@@ -83,6 +83,8 @@ extern unsigned int DebugWork;						/* (BRINGUP) */
 void mapping_verify(void);
 void mapping_phys_unused(ppnum_t pa);
 
+int nx_enabled = 0;			/* enable no-execute protection */
+
 /*
  *  ppc_prot translates Mach's representation of protections to that of the PPC hardware.
  *  For Virtual Machines (VMM), we also provide translation entries where the output is
@@ -91,15 +93,25 @@ void mapping_phys_unused(ppnum_t pa);
  *	8 table entries; direct translations are placed in the range 8..16, so they fall into
  *  the second half of the table.
  *
- *  ***NOTE*** I've commented out the Mach->PPC translations that would set page-level
- *             no-execute, pending updates to the VM layer that will properly enable its
- *             use.  Bob Abeles 08.02.04
  */
  
-//unsigned char ppc_prot[16] = { 4, 7, 6, 6, 3, 3, 2, 2,		/* Mach -> PPC translations */
-unsigned char ppc_prot[16] = { 0, 3, 2, 2, 3, 3, 2, 2,		/* Mach -> PPC translations */
+unsigned char ppc_prot[16] = { 4, 7, 6, 6, 3, 3, 2, 2,		/* Mach -> PPC translations */
                                0, 1, 2, 3, 4, 5, 6, 7 };	/* VMM direct  translations */
 
+
+
+vm_prot_t getProtPPC(int key, boolean_t disable_NX) {
+        vm_prot_t prot;
+
+	prot = ppc_prot[key & 0xF];
+
+	if (key <= 7 && disable_NX == TRUE)
+	        prot &= ~mpN;
+
+	return (prot);
+}
+
+
 /*
  *			About PPC VSID generation:
  *
@@ -302,6 +314,7 @@ addr64_t mapping_make(pmap_t pmap, addr64_t va, ppnum_t pa, unsigned int flags,
 	unsigned int pindex, mflags, pattr, wimg, rc;
 	phys_entry_t *physent;
 	int nlists, pcf;
+	boolean_t disable_NX = FALSE;
 
 	pindex = 0;
 	
@@ -372,10 +385,12 @@ addr64_t mapping_make(pmap_t pmap, addr64_t va, ppnum_t pa, unsigned int flags,
 	mp->u.mpBSize = size;										/* Set the size */
 	mp->mpPte = 0;												/* Set the PTE invalid */
 	mp->mpPAddr = pa;											/* Set the physical page number */
-	mp->mpVAddr = (va & ~mpHWFlags) | (wimg << 3)				/* Add the protection and attributes to the field */
-		| ((PerProcTable[0].ppe_vaddr->pf.Available & pf64Bit)?
-			getProtPPC(prot) : (getProtPPC(prot) & 0x3));		/* Mask off no-execute control for 32-bit machines */			
-	
+
+	if ( !nx_enabled || (pmap->pmapFlags & pmapNXdisabled) )
+	        disable_NX = TRUE;
+
+	mp->mpVAddr = (va & ~mpHWFlags) | (wimg << 3) | getProtPPC(prot, disable_NX);			/* Add the protection and attributes to the field */
+	  
 	while(1) {													/* Keep trying... */
 		colladdr = hw_add_map(pmap, mp);						/* Go add the mapping to the pmap */
 		rc = colladdr & mapRetCode;								/* Separate return code */
@@ -482,8 +497,12 @@ void
 mapping_protect(pmap_t pmap, addr64_t va, vm_prot_t prot, addr64_t *nextva) {	/* Change protection of a virtual page */
 
 	int	ret;
-	
-	ret = hw_protect(pmap, va, getProtPPC(prot), nextva);	/* Try to change the protect here */
+	boolean_t disable_NX = FALSE;
+
+	if ( !nx_enabled || (pmap->pmapFlags & pmapNXdisabled) )
+	        disable_NX = TRUE;
+
+	ret = hw_protect(pmap, va, getProtPPC(prot, disable_NX), nextva);		/* Try to change the protect here */
 
 	switch (ret) {								/* Decode return code */
 	
@@ -505,8 +524,8 @@ mapping_protect(pmap_t pmap, addr64_t va, vm_prot_t prot, addr64_t *nextva) {	/*
  *
  *		This routine takes a physical entry and runs through all mappings attached to it and changes
  *		the protection.  If there are PTEs associated with the mappings, they will be invalidated before
- *		the protection is changed.  There is no limitation on changes, e.g., 
- *		higher to lower, lower to higher.
+ *		the protection is changed.  There is no limitation on changes, e.g., higher to lower, lower to
+ *		higher; however, changes to execute protection are ignored.
  *
  *		Any mapping that is marked permanent is not changed
  *
@@ -517,16 +536,16 @@ void mapping_protect_phys(ppnum_t pa, vm_prot_t prot) {	/* Change protection of
 	
 	unsigned int pindex;
 	phys_entry_t *physent;
-	
+
 	physent = mapping_phys_lookup(pa, &pindex);					/* Get physical entry */
 	if(!physent) {												/* Did we find the physical page? */
 		panic("mapping_protect_phys: invalid physical page %08X\n", pa);
 	}
 
 	hw_walk_phys(physent, hwpNoop, hwpSPrtMap, hwpNoop,
-	             getProtPPC(prot), hwpPurgePTE);				/* Set the new protection for page and mappings */
+		     getProtPPC(prot, FALSE), hwpPurgePTE);			/* Set the new protection for page and mappings */
 
-	return;														/* Leave... */
+	return;									/* Leave... */
 }
 
 
@@ -1499,52 +1518,18 @@ addr64_t	mapping_p2v(pmap_t pmap, ppnum_t pa) {				/* Finds first virtual mappin
 	
 }
 
-/*
- *	phystokv(addr)
- *
- *	Convert a physical address to a kernel virtual address if
- *	there is a mapping, otherwise return NULL
- */
-
-vm_offset_t phystokv(vm_offset_t pa) {
-
-	addr64_t	va;
-	ppnum_t pp;
-
-	pp = pa >> 12;											/* Convert to a page number */
-	
-	if(!(va = mapping_p2v(kernel_pmap, pp))) {
-		return 0;											/* Can't find it, return 0... */
-	}
-	
-	return (va | (pa & (PAGE_SIZE - 1)));					/* Build and return VADDR... */
-
-}
 
 /*
  *	kvtophys(addr)
  *
  *	Convert a kernel virtual address to a physical address
  */
-vm_offset_t kvtophys(vm_offset_t va) {
+addr64_t kvtophys(vm_offset_t va) {
 
 	return pmap_extract(kernel_pmap, va);					/* Find mapping and lock the physical entry for this mapping */
 
 }
 
-/*
- *	kvtophys64(addr)
- *
- *	Convert a kernel virtual address to a 64-bit physical address
- */
-vm_map_offset_t kvtophys64(vm_map_offset_t va) {
-	ppnum_t pa = pmap_find_phys(kernel_pmap, (addr64_t)va);
-
-	if (!pa)
-		return (vm_map_offset_t)0;
-	return (((vm_map_offset_t)pa) << 12) | (va & 0xfff);
-}
-
 /*
  *		void ignore_zero_fault(boolean_t) - Sets up to ignore or honor any fault on 
  *		page 0 access for the current thread.
@@ -1562,6 +1547,13 @@ void ignore_zero_fault(boolean_t type) {				/* Sets up to ignore or honor any fa
 	return;												/* Return the result or 0... */
 }
 
+/*
+ * nop in current ppc implementation
+ */
+void inval_copy_windows(__unused thread_t t)
+{
+}
+
 
 /* 
  *		Copies data between a physical page and a virtual page, or 2 physical.  This is used to 
diff --git a/osfmk/ppc/mappings.h b/osfmk/ppc/mappings.h
index 4dc8c4e2f..62431b7be 100644
--- a/osfmk/ppc/mappings.h
+++ b/osfmk/ppc/mappings.h
@@ -371,7 +371,7 @@ extern mappingctl_t	mapCtl;				/* Mapping allocation control */
 
 extern unsigned char ppc_prot[];		/* Mach -> PPC protection translation table */
 
-#define getProtPPC(__key) (ppc_prot[(__key) & 0xF])
+vm_prot_t getProtPPC(int, boolean_t);
 										/* Safe Mach -> PPC protection key conversion */
 
 extern addr64_t 	mapping_remove(pmap_t pmap, addr64_t va);	/* Remove a single mapping for this VADDR */
diff --git a/osfmk/ppc/pmap.c b/osfmk/ppc/pmap.c
index 10b8f3b4b..5292ed3c2 100644
--- a/osfmk/ppc/pmap.c
+++ b/osfmk/ppc/pmap.c
@@ -237,8 +237,14 @@ pmap_map(
 	vm_offset_t va,
 	vm_offset_t spa,
 	vm_offset_t epa,
-	vm_prot_t prot)
+	vm_prot_t prot,
+	unsigned int flags)
 {
+	unsigned int mflags;
+	mflags = 0;										/* Make sure this is initialized to nothing special */
+	if(!(flags & VM_WIMG_USE_DEFAULT)) {			/* Are they supplying the attributes? */
+		mflags = mmFlgUseAttr | (flags & VM_MEM_GUARDED) | ((flags & VM_MEM_NOT_CACHEABLE) >> 1);	/* Convert to our mapping_make flags */
+	}
 
 	addr64_t colladr;
 	
@@ -246,7 +252,8 @@ pmap_map(
 
 	assert(epa > spa);
 
-	colladr = mapping_make(kernel_pmap, (addr64_t)va, (ppnum_t)(spa >> 12), (mmFlgBlock | mmFlgPerm), (epa - spa) >> 12, prot & VM_PROT_ALL);
+	colladr = mapping_make(kernel_pmap, (addr64_t)va, (ppnum_t)(spa >> 12),
+			       (mmFlgBlock | mmFlgPerm), (epa - spa) >> 12, (prot & VM_PROT_ALL) );
 
 	if(colladr) {											/* Was something already mapped in the range? */
 		panic("pmap_map: attempt to map previously mapped range - va = %08X, pa = %08X, epa = %08X, collision = %016llX\n",
@@ -363,6 +370,7 @@ pmap_bootstrap(uint64_t msize, vm_offset_t *first_avail, unsigned int kmapsize)
 	kernel_pmap->pmap_link.prev = (queue_t)kernel_pmap;		/* Set up anchor reverse */
 	kernel_pmap->ref_count = 1;
 	kernel_pmap->pmapFlags = pmapKeyDef;					/* Set the default keys */
+	kernel_pmap->pmapFlags |= pmapNXdisabled;
 	kernel_pmap->pmapCCtl = pmapCCtlVal;					/* Initialize cache control */
 	kernel_pmap->space = PPC_SID_KERNEL;
 	kernel_pmap->pmapvr = 0;								/* Virtual = Real  */
@@ -531,7 +539,7 @@ pmap_bootstrap(uint64_t msize, vm_offset_t *first_avail, unsigned int kmapsize)
 
 	/* Map V=R the page tables */
 	pmap_map(first_used_addr, first_used_addr,
-		 round_page(first_used_addr + size), VM_PROT_READ | VM_PROT_WRITE);
+		 round_page(first_used_addr + size), VM_PROT_READ | VM_PROT_WRITE, VM_WIMG_USE_DEFAULT);
 
 	*first_avail = round_page(first_used_addr + size);		/* Set next available page */
 	first_free_virt = *first_avail;							/* Ditto */
@@ -654,7 +662,7 @@ void pmap_virtual_space(
  * only, and is bounded by that size.
  */
 pmap_t
-pmap_create(vm_map_size_t size)
+pmap_create(vm_map_size_t size, __unused boolean_t is_64bit)
 {
 	pmap_t pmap, ckpmap, fore;
 	int s;
@@ -942,7 +950,7 @@ pmap_page_protect(
 	mapping_t			*mp;
 
 
-	switch (prot) {
+	switch (prot & VM_PROT_ALL) {
 		case VM_PROT_READ:
 		case VM_PROT_READ|VM_PROT_EXECUTE:
 			remove = FALSE;
@@ -989,7 +997,7 @@ pmap_page_protect(
  *	physical page.  
  */
  
-	mapping_protect_phys(pa, prot & VM_PROT_ALL);	/* Change protection of all mappings to page. */
+	mapping_protect_phys(pa, (prot & VM_PROT_ALL) );		/* Change protection of all mappings to page. */
 
 }
 
@@ -1067,7 +1075,7 @@ void pmap_protect(
 	endva = eva & -4096LL;						/* Round end down to a page */
 
 	while(1) {									/* Go until we finish the range */
-		mapping_protect(pmap, va, prot & VM_PROT_ALL, &va);	/* Change the protection and see what's next */
+		mapping_protect(pmap, va, (prot & VM_PROT_ALL), &va);	/* Change the protection and see what's next */
 		if((va == 0) || (va >= endva)) break;	/* End loop if we finish range or run off the end */
 	}
 
@@ -1108,7 +1116,7 @@ pmap_enter(pmap_t pmap, vm_map_offset_t va, ppnum_t pa, vm_prot_t prot,
 
 	while(1) {										/* Keep trying the enter until it goes in */
 	
-		colva = mapping_make(pmap, va, pa, mflags, 1, prot & VM_PROT_ALL);	/* Enter the mapping into the pmap */
+		colva = mapping_make(pmap, va, pa, mflags, 1, (prot & VM_PROT_ALL) );		/* Enter the mapping into the pmap */
 		
 		if(!colva) break;							/* If there were no collisions, we are done... */
 		
@@ -1293,6 +1301,29 @@ pmap_attribute(
 
 }
 
+
+
+unsigned int pmap_cache_attributes(ppnum_t pgn) {
+
+        unsigned int	flags;
+	struct phys_entry * pp;
+
+	// Find physical address
+	if ((pp = pmap_find_physentry(pgn))) {
+	        // Use physical attributes as default
+	        // NOTE: DEVICE_PAGER_FLAGS are made to line up
+	        flags = VM_MEM_COHERENT;				/* We only support coherent memory */
+		if (pp->ppLink & ppG) flags |= VM_MEM_GUARDED;		/* Add in guarded if it is */
+		if (pp->ppLink & ppI) flags |= VM_MEM_NOT_CACHEABLE;	/* Add in cache inhibited if so */
+	} else
+	        // If no physical, just hard code attributes
+	        flags = VM_WIMG_IO;
+
+	return (flags);
+}
+
+
+
 /*
  * pmap_attribute_cache_sync(vm_offset_t pa)
  * 
@@ -1962,7 +1993,7 @@ void pmap_init_sharedpage(vm_offset_t cpg){
 	addr64_t cva, cpoff;
 	ppnum_t cpphys;
 	
-	sharedPmap = pmap_create(0);				/* Get a pmap to hold the common segment */
+	sharedPmap = pmap_create(0, FALSE);				/* Get a pmap to hold the common segment */
 	if(!sharedPmap) {							/* Check for errors */
 		panic("pmap_init_sharedpage: couldn't make sharedPmap\n");
 	}
@@ -1975,7 +2006,7 @@ void pmap_init_sharedpage(vm_offset_t cpg){
 		}
 		
 		cva = mapping_make(sharedPmap, (addr64_t)((uint32_t)_COMM_PAGE_BASE_ADDRESS) + cpoff,
-			cpphys, mmFlgPerm, 1, VM_PROT_READ);	/* Map the page read only */
+			cpphys, mmFlgPerm, 1, VM_PROT_READ | VM_PROT_EXECUTE);		/* Map the page read/execute only */
 		if(cva) {								/* Check for errors */
 			panic("pmap_init_sharedpage: couldn't map commpage page - cva = %016llX\n", cva);
 		}
@@ -2062,3 +2093,13 @@ coredumpok(
 {
 	return TRUE;
 }
+
+
+/*
+ * disable no-execute capability on
+ * the specified pmap
+ */
+void pmap_disable_NX(pmap_t pmap) {
+  
+        pmap->pmapFlags |= pmapNXdisabled;
+}
diff --git a/osfmk/ppc/pmap.h b/osfmk/ppc/pmap.h
index 3ec636f54..483726de0 100644
--- a/osfmk/ppc/pmap.h
+++ b/osfmk/ppc/pmap.h
@@ -166,6 +166,7 @@ struct pmap {
 #define pmapKeyDef	0x00000006			/* Default keys - Sup = 1, user = 1, no ex = 0 */
 #define pmapVMhost	0x00000010			/* pmap with Virtual Machines attached to it */
 #define pmapVMgsaa	0x00000020			/* Guest shadow assist active */
+#define pmapNXdisabled	0x00000040			/* no-execute disabled for this pmap */
 	unsigned int	spaceNum;			/* Space number */
 	unsigned int	pmapCCtl;			/* Cache control */
 #define pmapCCtlVal	0xFFFF0000			/* Valid entries */
@@ -282,13 +283,12 @@ extern pmapTransTab *pmapTrans;			/* Space to pmap translate table */
 /* 
  * prototypes.
  */
-extern vm_offset_t phystokv(vm_offset_t pa);					/* Get kernel virtual address from physical */
-extern vm_offset_t kvtophys(vm_offset_t va);					/* Get physical address from kernel virtual */
-extern vm_map_offset_t kvtophys64(vm_map_offset_t va);				/* Get 64-bit physical address from kernel virtual */
+extern addr64_t	   	kvtophys(vm_offset_t va);					/* Get physical address from kernel virtual */
 extern vm_offset_t	pmap_map(vm_offset_t va,
 				 vm_offset_t spa,
 				 vm_offset_t epa,
-				 vm_prot_t prot);
+				 vm_prot_t prot,
+				 unsigned int flags);
 extern kern_return_t    pmap_add_physical_memory(vm_offset_t spa,
 						 vm_offset_t epa,
 						 boolean_t available,
@@ -326,6 +326,10 @@ extern int pmap_list_resident_pages(
 extern void pmap_init_sharedpage(vm_offset_t cpg);
 extern void pmap_map_sharedpage(task_t task, pmap_t pmap);
 extern void pmap_unmap_sharedpage(pmap_t pmap);
+extern void pmap_disable_NX(pmap_t pmap);
+/* Not required for ppc: */
+static inline void pmap_set_4GB_pagezero(__unused pmap_t pmap) {}
+static inline void pmap_clear_4GB_pagezero(__unused pmap_t pmap) {}
 
 
 
diff --git a/osfmk/ppc/pms.h b/osfmk/ppc/pms.h
index 9bd9b5d53..9371152c9 100644
--- a/osfmk/ppc/pms.h
+++ b/osfmk/ppc/pms.h
@@ -30,144 +30,6 @@
 
 #ifndef _PPC_PMS_H_
 #define _PPC_PMS_H_
-
-#define pmsMaxStates 64
-#define HalfwayToForever 0x7FFFFFFFFFFFFFFFULL
-#define century 790560000000000ULL
-
-typedef void (*pmsSetFunc_t)(uint32_t, uint32_t, uint32_t);	/* Function used to set hardware power state */
-typedef uint32_t (*pmsQueryFunc_t)(uint32_t, uint32_t);	/* Function used to query hardware power state */
-
-typedef struct pmsStat {
-	uint64_t	stTime[2];			/* Total time until switch to next step */
-	uint32_t	stCnt[2];			/* Number of times switched to next step */
-} pmsStat;
-
-typedef struct pmsDef {
-	uint64_t	pmsLimit;			/* Max time in this state in microseconds */
-	uint32_t	pmsStepID;			/* Unique ID for this step */
-	uint32_t	pmsSetCmd;			/* Command to select power state */
-#define pmsCngXClk  0x80000000		/* Change external clock */
-#define pmsXUnk   	0x7F			/* External clock unknown  */
-#define pmsXClk     0x7F000000		/* External clock frequency */
-#define pmsCngCPU   0x00800000		/* Change CPU parameters */
-#define pmsSync     0x00400000		/* Make changes synchronously, i.e., spin until delay finished */
-#define pmsMustCmp  0x00200000		/* Delay must complete before next change */
-#define pmsCPU      0x001F0000		/* CPU frequency */
-#define pmsCPUUnk	0x1F			/* CPU frequency unknown */
-#define pmsCngVolt  0x00008000		/* Change voltage */
-#define pmsVoltage  0x00007F00		/* Voltage */
-#define pmsVoltUnk	0x7F			/* Voltage unknown */
-#define pmsPowerID  0x000000FF		/* Identify power state to HW */
-
-/*	Special commands - various things */
-#define pmsDelay    0xFFFFFFFD		/* Delayed step, no processor or platform changes.  Timer expiration causes transition to pmsTDelay */
-#define pmsParkIt	0xFFFFFFFF		/* Enters the parked state.  No processor or platform changes.  Timers cancelled */
-#define pmsCInit	((pmsXUnk << 24) | (pmsCPUUnk << 16) | (pmsVoltUnk << 8))	/* Initial current set command value */
-/*	Note:  pmsSetFuncInd is an index into a table of function pointers and pmsSetFunc is the address
- *	of a function.  Initially, when you create a step table, this field is set as an index into
- *	a table of function addresses that gets passed as a parameter to pmsBuild.  When pmsBuild
- *	internalizes the step and function tables, it converts the index to the function address.
- */
-	union sf {
-		pmsSetFunc_t	pmsSetFunc;	/* Function used to set platform power state */
-		uint32_t	pmsSetFuncInd;	/* Index to function in function table */
-	} sf;
-
-	uint32_t	pmsDown;			/* Next state if going lower */
-	uint32_t	pmsNext;			/* Normal next state */
-	uint32_t	pmsTDelay;			/* State if command was pmsDelay and timer expired */
-} pmsDef;
-
-typedef struct pmsCtl {
-	pmsStat		(*pmsStats)[pmsMaxStates];	/* Pointer to statistics information, 0 if not enabled */
-	pmsDef		*pmsDefs[pmsMaxStates];	/* Indexed pointers to steps */
-} pmsCtl;
-
-/*
- *	Note that this block is in the middle of the per_proc and the size (32 bytes)
- *	can't be changed without moving it.
- */
-
-typedef struct pmsd {
-	uint32_t	pmsState;			/* Current power management state */
-	uint32_t	pmsCSetCmd;			/* Current select command */
-	uint64_t	pmsPop;				/* Time of next step */
-	uint64_t	pmsStamp;			/* Time of transition to current state */
-	uint64_t	pmsTime;			/* Total time in this state */
-} pmsd;
-
-/*
- *	Required power management step programs
- */
- 
-enum {
-	pmsIdle      = 0,				/* Power state in idle loop */
-	pmsNorm      = 1,				/* Normal step - usually low power */
-	pmsNormHigh  = 2,				/* Highest power in normal step */
-	pmsBoost     = 3,				/* Boost/overdrive step */
-	pmsLow       = 4,				/* Lowest non-idle power state, no transitions */
-	pmsHigh      = 5,				/* Power step for full on, no transitions */
-	pmsPrepCng   = 6,				/* Prepare for step table change */
-	pmsPrepSleep = 7,				/* Prepare for sleep */
-	pmsOverTemp  = 8,				/* Machine is too hot */
-	pmsEnterNorm = 9,				/* Enter into the normal step program */
-	pmsFree      = 10,				/* First available empty step */
-	pmsStartUp   = 0xFFFFFFFE,		/* Start stepping */
-	pmsParked    = 0xFFFFFFFF		/* Power parked - used when changing stepping table */
-};
-
-/*
- *	Power Management Stepper Control requests
- */
- 
-enum {
-	pmsCPark = 0,					/* Parks the stepper */
-	pmsCStart = 1,					/* Starts normal steppping */
-	pmsCFLow = 2,					/* Forces low power */
-	pmsCFHigh = 3,					/* Forces high power */
-	pmsCCnfg = 4,					/* Loads new stepper program */
-	pmsCQuery = 5,					/* Query current step and state */
-	pmsCExperimental = 6,			/* Enter experimental mode */
-	pmsCFree = 7					/* Next control command to be assigned */
-};
-
-extern pmsCtl pmsCtls;				/* Power Management Stepper control */
-extern uint32_t pmsCtlp;
-extern uint32_t pmsBroadcastWait;	/* Number of outstanding broadcasts */
-extern pmsDef pmsDefault[];
-extern int pmsInstalled;
-extern int pmsExperimental;
-
-#define pmsSetFuncMax 32
-extern pmsSetFunc_t pmsFuncTab[pmsSetFuncMax];
-extern pmsQueryFunc_t pmsQueryFunc;
-extern uint32_t pmsPlatformData;
-
-extern int pmsCntrl(struct savearea *save);
-extern void pmsInit(void);
-extern void pmsStep(int timer);
-extern void pmsDown(void);
-extern void pmsSetStep(uint32_t nstep, int dir);
-extern void pmsRemote(uint32_t nstep);
-extern void pmsCPUSet(uint32_t sel);
-extern uint32_t pmsCPUquery(void);
-extern void pmsCPUConf(void);
-extern void pmsCPUInit(void);
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern kern_return_t pmsBuild(pmsDef *pd, uint32_t pdsize, pmsSetFunc_t *functab, uint32_t platformData, pmsQueryFunc_t queryFunc);
-extern void pmsRun(uint32_t nstep);
-extern void pmsRunLocal(uint32_t nstep);
-extern void pmsPark(void);
-extern void pmsStart(void);
-
-#ifdef __cplusplus
-}
-#endif
-
+#include <kern/pms.h>
 #endif /* _PPC_PMS_H_ */
 #endif /* KERNEL_PRIVATE */
diff --git a/osfmk/ppc/pmsCPU.c b/osfmk/ppc/pmsCPU.c
index 81a70c9ca..3e67ca5ca 100644
--- a/osfmk/ppc/pmsCPU.c
+++ b/osfmk/ppc/pmsCPU.c
@@ -33,12 +33,15 @@
 #include <ppc/pmap.h>
 #include <ppc/asm.h>
 #include <ppc/proc_reg.h>
-#include <ppc/pms.h>
+#include <kern/pms.h>
 #include <ppc/savearea.h>
 #include <ppc/Diagnostics.h>
 #include <kern/processor.h>
 
 
+static void pmsCPURemote(uint32_t nstep);
+
+
 pmsDef pmsDefault[] = {
 	{
 		.pmsLimit = century,							/* We can normally stay here for 100 years */
@@ -186,6 +189,15 @@ pCCfinish:
 	return;
 }
 
+/*
+ * Machine-dependent initialization
+ */
+void
+pmsCPUMachineInit(void)
+{
+	return;
+}
+
 /*
  *	This function should be called once for each processor to force the
  *	processor to the correct voltage and frequency.
@@ -207,7 +219,7 @@ void pmsCPUInit(void) {
 	return;
 }
 
-uint32_t pmsCPUquery(void) {
+uint32_t pmsCPUQuery(void) {
 
 	uint32_t result;
 	struct per_proc_info *pp;
@@ -236,4 +248,67 @@ uint32_t pmsCPUquery(void) {
 	return result;
 }
 
+/*
+ *	These are not implemented for PPC.
+ */
+void pmsCPUYellowFlag(void) {
+}
+
+void pmsCPUGreenFlag(void) {
+}
+
+uint32_t pmsCPUPackageQuery(void)
+{
+    	/* multi-core CPUs are not supported. */
+    	return(~(uint32_t)0);
+}
+
+/*
+ *	Broadcast a change to all processors including ourselves.
+ *	This must transition before broadcasting because we may block and end up on a different processor.
+ *
+ *	This will block until all processors have transitioned, so
+ *	obviously, this can block.
+ *
+ *	Called with interruptions disabled.
+ *
+ */
+ 
+void pmsCPURun(uint32_t nstep) {
+
+	pmsRunLocal(nstep);								/* If we aren't parking (we are already parked), transition ourselves */
+	(void)cpu_broadcast(&pmsBroadcastWait, pmsCPURemote, nstep);	/* Tell everyone else to do it too */
+
+	return;
+	
+}
+
+/*
+ *	Receive a broadcast and react.
+ *	This is called from the interprocessor signal handler.
+ *	We wake up the initiator after we are finished.
+ *
+ */
+	
+static void pmsCPURemote(uint32_t nstep) {
+
+	pmsRunLocal(nstep);								/* Go set the step */
+	if(!hw_atomic_sub(&pmsBroadcastWait, 1)) {		/* Drop the wait count */
+		thread_wakeup((event_t)&pmsBroadcastWait);	/* If we were the last, wake up the signaller */
+	}
+	return;
+}	
+
+/*
+ *	Control the Power Management Stepper.
+ *	Called from user state by the superuser via a ppc system call.
+ *	Interruptions disabled.
+ *
+ */
+int pmsCntrl(struct savearea *save) {
+	save->save_r3 = pmsControl(save->save_r3, (user_addr_t)(uintptr_t)save->save_r4, save->save_r5);
+	return 1;
+}
+
+
 
diff --git a/osfmk/ppc/ppc_init.c b/osfmk/ppc/ppc_init.c
index b371e79e1..7cc680dd3 100644
--- a/osfmk/ppc/ppc_init.c
+++ b/osfmk/ppc/ppc_init.c
@@ -54,7 +54,7 @@
 #include <ppc/mem.h>
 #include <ppc/mappings.h>
 #include <ppc/locks.h>
-#include <ppc/pms.h>
+#include <kern/pms.h>
 #include <ppc/rtclock.h>
 
 #include <pexpert/pexpert.h>
diff --git a/osfmk/ppc/ppc_vm_init.c b/osfmk/ppc/ppc_vm_init.c
index c8c5ac6fd..ce2205ed8 100644
--- a/osfmk/ppc/ppc_vm_init.c
+++ b/osfmk/ppc/ppc_vm_init.c
@@ -44,6 +44,7 @@
 #include <kern/assert.h>
 #include <kern/cpu_number.h>
 #include <kern/thread.h>
+#include <console/serial_protos.h>
 
 #include <ppc/proc_reg.h>
 #include <ppc/Firmware.h>
@@ -65,7 +66,6 @@ unsigned int hash_table_size;			/* Hash table size */
 int         hash_table_shift;           /* "ht_shift" boot arg, used to scale hash_table_size */
 vm_offset_t taproot_addr;				/* (BRINGUP) */
 unsigned int taproot_size;				/* (BRINGUP) */
-unsigned int serialmode;				/* Serial mode keyboard and console control */
 extern int disableConsoleOutput;
 
 struct shadowBAT shadow_BAT;
@@ -240,13 +240,13 @@ void ppc_vm_init(uint64_t mem_limit, boot_args *args)
 	pmap_bootstrap(max_mem, &first_avail, kmapsize);
 
 	pmap_map(trunc_page(exception_entry), trunc_page(exception_entry), 
-		round_page(exception_end), VM_PROT_READ|VM_PROT_EXECUTE);
+		round_page(exception_end), VM_PROT_READ|VM_PROT_EXECUTE, VM_WIMG_USE_DEFAULT);
 
 	pmap_map(trunc_page(sectTEXTB), trunc_page(sectTEXTB), 
-		round_page(sectTEXTB+sectSizeTEXT), VM_PROT_READ|VM_PROT_EXECUTE);
+		round_page(sectTEXTB+sectSizeTEXT), VM_PROT_READ|VM_PROT_EXECUTE, VM_WIMG_USE_DEFAULT);
 
 	pmap_map(trunc_page(sectDATAB), trunc_page(sectDATAB), 
-		round_page(sectDATAB+sectSizeDATA), VM_PROT_READ|VM_PROT_WRITE);
+		round_page(sectDATAB+sectSizeDATA), VM_PROT_READ|VM_PROT_WRITE, VM_WIMG_USE_DEFAULT);
 
 /* The KLD and LINKEDIT segments are unloaded in toto after boot completes,
 * but via ml_static_mfree(), through IODTFreeLoaderInfo(). Hence, we have
@@ -258,7 +258,7 @@ void ppc_vm_init(uint64_t mem_limit, boot_args *args)
              addr += PAGE_SIZE) {
 
             pmap_enter(kernel_pmap, (vm_map_offset_t)addr, (ppnum_t)(addr>>12), 
-			VM_PROT_READ|VM_PROT_WRITE, 
+			VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, 
 			VM_WIMG_USE_DEFAULT, TRUE);
 
 	}
@@ -268,7 +268,7 @@ void ppc_vm_init(uint64_t mem_limit, boot_args *args)
              addr += PAGE_SIZE) {
 
             pmap_enter(kernel_pmap, (vm_map_offset_t)addr, (ppnum_t)(addr>>12), 
-			VM_PROT_READ|VM_PROT_WRITE, 
+			VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, 
 			VM_WIMG_USE_DEFAULT, TRUE);
 
 	}
@@ -279,7 +279,7 @@ void ppc_vm_init(uint64_t mem_limit, boot_args *args)
 
            pmap_enter(kernel_pmap, (vm_map_offset_t)addr,
 			(ppnum_t)(addr>>12), 
-			VM_PROT_READ|VM_PROT_WRITE, 
+			VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, 
 			VM_WIMG_USE_DEFAULT, TRUE);
 
 	}
@@ -289,7 +289,7 @@ void ppc_vm_init(uint64_t mem_limit, boot_args *args)
              addr += PAGE_SIZE) {
 
             pmap_enter(kernel_pmap, (vm_map_offset_t)addr, (ppnum_t)(addr>>12), 
-			VM_PROT_READ|VM_PROT_WRITE, 
+			VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, 
 			VM_WIMG_USE_DEFAULT, TRUE);
 
 	}
@@ -312,7 +312,7 @@ void ppc_vm_init(uint64_t mem_limit, boot_args *args)
 	for(addr = trunc_page(end); addr < round_page(static_memory_end); addr += PAGE_SIZE) {
 
 		pmap_enter(kernel_pmap, (vm_map_address_t)addr, (ppnum_t)addr>>12, 
-			VM_PROT_READ|VM_PROT_WRITE, 
+			VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, 
 			VM_WIMG_USE_DEFAULT, TRUE);
 
 	}
diff --git a/osfmk/ppc/rtclock.c b/osfmk/ppc/rtclock.c
index bf8e9bd6d..b294ae1b0 100644
--- a/osfmk/ppc/rtclock.c
+++ b/osfmk/ppc/rtclock.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -41,11 +41,10 @@
 
 #include <kern/clock.h>
 #include <kern/thread.h>
+#include <kern/processor.h>
 #include <kern/macro_help.h>
 #include <kern/spl.h>
 
-#include <kern/host_notify.h>
-
 #include <machine/commpage.h>
 #include <machine/machine_routines.h>
 #include <ppc/exception.h>
@@ -53,83 +52,13 @@
 #include <ppc/pms.h>
 #include <ppc/rtclock.h>
 
-#include <IOKit/IOPlatformExpert.h>
-
 #include <sys/kdebug.h>
 
-int		sysclk_config(void);
-
-int		sysclk_init(void);
-
-kern_return_t	sysclk_gettime(
-	mach_timespec_t			*cur_time);
-
-kern_return_t	sysclk_getattr(
-	clock_flavor_t			flavor,
-	clock_attr_t			attr,
-	mach_msg_type_number_t	*count);
-
-void		sysclk_setalarm(
-	mach_timespec_t			*deadline);
-
-struct clock_ops sysclk_ops = {
-	sysclk_config,			sysclk_init,
-	sysclk_gettime,			0,
-	sysclk_getattr,			0,
-	sysclk_setalarm,
-};
-
-int		calend_config(void);
-
-kern_return_t	calend_gettime(
-	mach_timespec_t			*cur_time);
-
-kern_return_t	calend_getattr(
-	clock_flavor_t			flavor,
-	clock_attr_t			attr,
-	mach_msg_type_number_t	*count);
-
-struct clock_ops calend_ops = {
-	calend_config,			0,
-	calend_gettime,			0,
-	calend_getattr,			0,
-	0,
-};
-
-/* local data declarations */
+int		rtclock_config(void);
 
-static struct rtclock_calend {
-	uint32_t			epoch;
-	uint32_t			microepoch;
-
-	uint64_t			epoch1;
-
-	int64_t				adjtotal;
-	int32_t				adjdelta;
-}					rtclock_calend;
-
-static uint32_t		rtclock_boottime;
-
-#define TIME_ADD(rsecs, secs, rfrac, frac, unit)	\
-MACRO_BEGIN											\
-	if (((rfrac) += (frac)) >= (unit)) {			\
-		(rfrac) -= (unit);							\
-		(rsecs) += 1;								\
-	}												\
-	(rsecs) += (secs);								\
-MACRO_END
-
-#define TIME_SUB(rsecs, secs, rfrac, frac, unit)	\
-MACRO_BEGIN											\
-	if ((int32_t)((rfrac) -= (frac)) < 0) {			\
-		(rfrac) += (unit);							\
-		(rsecs) -= 1;								\
-	}												\
-	(rsecs) -= (secs);								\
-MACRO_END
+int		rtclock_init(void);
 
 #define NSEC_PER_HZ		(NSEC_PER_SEC / 100)
-static uint32_t		 	rtclock_tick_interval;
 
 static uint32_t			rtclock_sec_divisor;
 
@@ -137,20 +66,8 @@ static mach_timebase_info_data_t	rtclock_timebase_const;
 
 static boolean_t		rtclock_timebase_initialized;
 
-static clock_timer_func_t	rtclock_timer_expire;
-
-static timer_call_data_t	rtclock_alarm_timer;
-
-static void		nanotime_to_absolutetime(
-					uint32_t		secs,
-					uint32_t		nanosecs,
-					uint64_t		*result);
-
-static void		rtclock_alarm_expire(
-					timer_call_param_t		p0,
-					timer_call_param_t		p1);
-
-/* global data declarations */
+/* XXX this should really be in a header somewhere */
+extern clock_timer_func_t	rtclock_timer_expire;
 
 decl_simple_lock_data(static,rtclock_lock)
 
@@ -187,7 +104,7 @@ timebase_callback(
 
 	LOCK_RTC(s);
 	if (!rtclock_timebase_initialized) {
-		commpage_set_timestamp(0,0,0,0);
+		commpage_set_timestamp(0,0,0);
 
 		rtclock_timebase_const.numer = numer;
 		rtclock_timebase_const.denom = denom;
@@ -211,13 +128,11 @@ timebase_callback(
 }
 
 /*
- * Configure the real-time clock device.
+ * Configure the system clock device.
  */
 int
-sysclk_config(void)
+rtclock_config(void)
 {
-	timer_call_setup(&rtclock_alarm_timer, rtclock_alarm_expire, NULL);
-
 	simple_lock_init(&rtclock_lock, 0);
 
 	PE_register_timebase_callback(timebase_callback);
@@ -229,7 +144,7 @@ sysclk_config(void)
  * Initialize the system clock device.
  */
 int
-sysclk_init(void)
+rtclock_init(void)
 {
 	uint64_t				abstime;
 	struct per_proc_info	*pp;
@@ -237,30 +152,13 @@ sysclk_init(void)
 	pp = getPerProc();
 
 	abstime = mach_absolute_time();
-	pp->rtclock_tick_deadline = abstime + rtclock_tick_interval;	/* Get the time we need to pop */
-	pp->rtcPop = pp->rtclock_tick_deadline;	/* Set the rtc pop time the same for now */
+	pp->rtclock_intr_deadline = abstime + rtclock_tick_interval;	/* Get the time we need to pop */
 	
-	(void)setTimerReq();			/* Start the timers going */
+	etimer_resync_deadlines();			/* Start the timers going */
 
 	return (1);
 }
 
-kern_return_t
-sysclk_gettime(
-	mach_timespec_t		*time)	/* OUT */
-{
-	uint64_t	now, t64;
-	uint32_t	divisor;
-
-	now = mach_absolute_time();
-
-	time->tv_sec = t64 = now / (divisor = rtclock_sec_divisor);
-	now -= (t64 * divisor);
-	time->tv_nsec = (now * NSEC_PER_SEC) / divisor;
-
-	return (KERN_SUCCESS);
-}
-
 void
 clock_get_system_microtime(
 	uint32_t			*secs,
@@ -291,590 +189,29 @@ clock_get_system_nanotime(
 	*nanosecs = (now * NSEC_PER_SEC) / divisor;
 }
 
-/*
- * Get clock device attributes.
- */
-kern_return_t
-sysclk_getattr(
-	clock_flavor_t			flavor,
-	clock_attr_t			attr,		/* OUT */
-	mach_msg_type_number_t	*count)		/* IN/OUT */
-{
-	spl_t		s;
-
-	if (*count != 1)
-		return (KERN_FAILURE);
-
-	switch (flavor) {
-
-	case CLOCK_GET_TIME_RES:	/* >0 res */
-	case CLOCK_ALARM_CURRES:	/* =0 no alarm */
-	case CLOCK_ALARM_MINRES:
-	case CLOCK_ALARM_MAXRES:
-		LOCK_RTC(s);
-		*(clock_res_t *) attr = NSEC_PER_HZ;
-		UNLOCK_RTC(s);
-		break;
-
-	default:
-		return (KERN_INVALID_VALUE);
-	}
-
-	return (KERN_SUCCESS);
-}
-
-/*
- * Set deadline for the next alarm on the clock device. This call
- * always resets the time to deliver an alarm for the clock.
- */
-void
-sysclk_setalarm(
-	mach_timespec_t		*deadline)
-{
-	uint64_t	abstime;
-
-	nanotime_to_absolutetime(deadline->tv_sec, deadline->tv_nsec, &abstime);
-	timer_call_enter(&rtclock_alarm_timer, abstime);
-}
-
-/*
- * Configure the calendar clock.
- */
-int
-calend_config(void)
-{
-	return (1);
-}
-
-/*
- * Get the current clock time.
- */
-kern_return_t
-calend_gettime(
-	mach_timespec_t		*time)	/* OUT */
-{
-	clock_get_calendar_nanotime(
-				&time->tv_sec, &time->tv_nsec);
-
-	return (KERN_SUCCESS);
-}
-
-/*
- * Get clock device attributes.
- */
-kern_return_t
-calend_getattr(
-	clock_flavor_t			flavor,
-	clock_attr_t			attr,		/* OUT */
-	mach_msg_type_number_t	*count)		/* IN/OUT */
-{
-	spl_t		s;
-
-	if (*count != 1)
-		return (KERN_FAILURE);
-
-	switch (flavor) {
-
-	case CLOCK_GET_TIME_RES:	/* >0 res */
-		LOCK_RTC(s);
-		*(clock_res_t *) attr = NSEC_PER_HZ;
-		UNLOCK_RTC(s);
-		break;
-
-	case CLOCK_ALARM_CURRES:	/* =0 no alarm */
-	case CLOCK_ALARM_MINRES:
-	case CLOCK_ALARM_MAXRES:
-		*(clock_res_t *) attr = 0;
-		break;
-
-	default:
-		return (KERN_INVALID_VALUE);
-	}
-
-	return (KERN_SUCCESS);
-}
-
 void
-clock_get_calendar_microtime(
-	uint32_t			*secs,
-	uint32_t			*microsecs)
+clock_gettimeofday_set_commpage(
+	uint64_t				abstime,
+	uint64_t				epoch,
+	uint64_t				offset,
+	uint32_t				*secs,
+	uint32_t				*microsecs)
 {
-	uint32_t		epoch, microepoch;
-	uint64_t		now, t64;
-	spl_t			s = splclock();
+	uint64_t				t64, now = abstime;
 
 	simple_lock(&rtclock_lock);
 
-	if (rtclock_calend.adjdelta >= 0) {
-		uint32_t		divisor;
-
-		now = mach_absolute_time();
-
-		epoch = rtclock_calend.epoch;
-		microepoch = rtclock_calend.microepoch;
+	now += offset;
 
-		simple_unlock(&rtclock_lock);
+	*secs = t64 = now / rtclock_sec_divisor;
+	now -= (t64 * rtclock_sec_divisor);
+	*microsecs = (now * USEC_PER_SEC) / rtclock_sec_divisor;
 
-		*secs = t64 = now / (divisor = rtclock_sec_divisor);
-		now -= (t64 * divisor);
-		*microsecs = (now * USEC_PER_SEC) / divisor;
+	*secs += epoch;
 
-		TIME_ADD(*secs, epoch, *microsecs, microepoch, USEC_PER_SEC);
-	}
-	else {
-		uint32_t	delta, t32;
-
-		delta = -rtclock_calend.adjdelta;
-
-		now = mach_absolute_time();
-
-		*secs = rtclock_calend.epoch;
-		*microsecs = rtclock_calend.microepoch;
-
-		if (now > rtclock_calend.epoch1) {
-			t64 = now - rtclock_calend.epoch1;
-
-			t32 = (t64 * USEC_PER_SEC) / rtclock_sec_divisor;
-
-			if (t32 > delta)
-				TIME_ADD(*secs, 0, *microsecs, (t32 - delta), USEC_PER_SEC);
-		}
-
-		simple_unlock(&rtclock_lock);
-	}
-
-	splx(s);
-}
-
-/* This is only called from the gettimeofday() syscall.  As a side
- * effect, it updates the commpage timestamp.  Otherwise it is
- * identical to clock_get_calendar_microtime().  Because most
- * gettimeofday() calls are handled by the commpage in user mode,
- * this routine should be infrequently used except when slowing down
- * the clock.
- */
-void
-clock_gettimeofday(
-	uint32_t			*secs_p,
-	uint32_t			*microsecs_p)
-{
-	uint32_t		epoch, microepoch;
-    uint32_t		secs, microsecs;
-	uint64_t		now, t64, secs_64, usec_64;
-	spl_t			s = splclock();
-
-	simple_lock(&rtclock_lock);
-
-	if (rtclock_calend.adjdelta >= 0) {
-		now = mach_absolute_time();
-
-		epoch = rtclock_calend.epoch;
-		microepoch = rtclock_calend.microepoch;
-
-		secs = secs_64 = now / rtclock_sec_divisor;
-		t64 = now - (secs_64 * rtclock_sec_divisor);
-		microsecs = usec_64 = (t64 * USEC_PER_SEC) / rtclock_sec_divisor;
-
-		TIME_ADD(secs, epoch, microsecs, microepoch, USEC_PER_SEC);
-        
-        /* adjust "now" to be absolute time at _start_ of usecond */
-        now -= t64 - ((usec_64 * rtclock_sec_divisor) / USEC_PER_SEC);
-        
-        commpage_set_timestamp(now,secs,microsecs,rtclock_sec_divisor);
-	}
-	else {
-		uint32_t	delta, t32;
-
-		delta = -rtclock_calend.adjdelta;
-
-		now = mach_absolute_time();
-
-		secs = rtclock_calend.epoch;
-		microsecs = rtclock_calend.microepoch;
-
-		if (now > rtclock_calend.epoch1) {
-			t64 = now - rtclock_calend.epoch1;
-
-			t32 = (t64 * USEC_PER_SEC) / rtclock_sec_divisor;
-
-			if (t32 > delta)
-				TIME_ADD(secs, 0, microsecs, (t32 - delta), USEC_PER_SEC);
-		}
-
-        /* no need to disable timestamp, it is already off */
-	}
-
-    simple_unlock(&rtclock_lock);
-	splx(s);
-    
-    *secs_p = secs;
-    *microsecs_p = microsecs;
-}
-
-void
-clock_get_calendar_nanotime(
-	uint32_t			*secs,
-	uint32_t			*nanosecs)
-{
-	uint32_t		epoch, nanoepoch;
-	uint64_t		now, t64;
-	spl_t			s = splclock();
-
-	simple_lock(&rtclock_lock);
-
-	if (rtclock_calend.adjdelta >= 0) {
-		uint32_t		divisor;
-
-		now = mach_absolute_time();
-
-		epoch = rtclock_calend.epoch;
-		nanoepoch = rtclock_calend.microepoch * NSEC_PER_USEC;
-
-		simple_unlock(&rtclock_lock);
-
-		*secs = t64 = now / (divisor = rtclock_sec_divisor);
-		now -= (t64 * divisor);
-		*nanosecs = ((now * USEC_PER_SEC) / divisor) * NSEC_PER_USEC;
-
-		TIME_ADD(*secs, epoch, *nanosecs, nanoepoch, NSEC_PER_SEC);
-	}
-	else {
-		uint32_t	delta, t32;
-
-		delta = -rtclock_calend.adjdelta;
-
-		now = mach_absolute_time();
-
-		*secs = rtclock_calend.epoch;
-		*nanosecs = rtclock_calend.microepoch * NSEC_PER_USEC;
-
-		if (now > rtclock_calend.epoch1) {
-			t64 = now - rtclock_calend.epoch1;
-
-			t32 = (t64 * USEC_PER_SEC) / rtclock_sec_divisor;
-
-			if (t32 > delta)
-				TIME_ADD(*secs, 0, *nanosecs, ((t32 - delta) * NSEC_PER_USEC), NSEC_PER_SEC);
-		}
-
-		simple_unlock(&rtclock_lock);
-	}
-
-	splx(s);
-}
-
-void
-clock_set_calendar_microtime(
-	uint32_t			secs,
-	uint32_t			microsecs)
-{
-	uint32_t		sys, microsys;
-	uint32_t		newsecs;
-	spl_t			s;
-
-	newsecs = (microsecs < 500*USEC_PER_SEC)?
-						secs: secs + 1;
-
-	s = splclock();
-	simple_lock(&rtclock_lock);
-
-    commpage_set_timestamp(0,0,0,0);
-
-	/*
-	 *	Cancel any adjustment in progress.
-	 */
-	if (rtclock_calend.adjdelta < 0) {
-		uint64_t	now, t64;
-		uint32_t	delta, t32;
-
-		delta = -rtclock_calend.adjdelta;
-
-		sys = rtclock_calend.epoch;
-		microsys = rtclock_calend.microepoch;
-
-		now = mach_absolute_time();
-
-		if (now > rtclock_calend.epoch1)
-			t64 = now - rtclock_calend.epoch1;
-		else
-			t64 = 0;
-
-		t32 = (t64 * USEC_PER_SEC) / rtclock_sec_divisor;
-
-		if (t32 > delta)
-			TIME_ADD(sys, 0, microsys, (t32 - delta), USEC_PER_SEC);
-
-		rtclock_calend.epoch = sys;
-		rtclock_calend.microepoch = microsys;
-
-		sys = t64 = now / rtclock_sec_divisor;
-		now -= (t64 * rtclock_sec_divisor);
-		microsys = (now * USEC_PER_SEC) / rtclock_sec_divisor;
-
-		TIME_SUB(rtclock_calend.epoch, sys, rtclock_calend.microepoch, microsys, USEC_PER_SEC);
-	}
-
-	rtclock_calend.epoch1 = 0;
-	rtclock_calend.adjdelta = rtclock_calend.adjtotal = 0;
-
-	/*
-	 *	Calculate the new calendar epoch based on
-	 *	the new value and the system clock.
-	 */
-	clock_get_system_microtime(&sys, &microsys);
-	TIME_SUB(secs, sys, microsecs, microsys, USEC_PER_SEC);
-
-	/*
-	 *	Adjust the boottime based on the delta.
-	 */
-	rtclock_boottime += secs - rtclock_calend.epoch;
-
-	/*
-	 *	Set the new calendar epoch.
-	 */
-	rtclock_calend.epoch = secs;
-	rtclock_calend.microepoch = microsecs;
+	commpage_set_timestamp(abstime - now, *secs, rtclock_sec_divisor);
 
 	simple_unlock(&rtclock_lock);
-
-	/*
-	 *	Set the new value for the platform clock.
-	 */
-	PESetGMTTimeOfDay(newsecs);
-
-	splx(s);
-
-	/*
-	 *	Send host notifications.
-	 */
-	host_notify_calendar_change();
-}
-
-#define tickadj		(40)				/* "standard" skew, us / tick */
-#define	bigadj		(USEC_PER_SEC)		/* use 10x skew above bigadj us */
-
-uint32_t
-clock_set_calendar_adjtime(
-	int32_t				*secs,
-	int32_t				*microsecs)
-{
-	int64_t			total, ototal;
-	uint32_t		interval = 0;
-	spl_t			s;
-
-	total = (int64_t)*secs * USEC_PER_SEC + *microsecs;
-
-	LOCK_RTC(s);
-    commpage_set_timestamp(0,0,0,0);
-
-	ototal = rtclock_calend.adjtotal;
-
-	if (rtclock_calend.adjdelta < 0) {
-		uint64_t		now, t64;
-		uint32_t		delta, t32;
-		uint32_t		sys, microsys;
-
-		delta = -rtclock_calend.adjdelta;
-
-		sys = rtclock_calend.epoch;
-		microsys = rtclock_calend.microepoch;
-
-		now = mach_absolute_time();
-
-		if (now > rtclock_calend.epoch1)
-			t64 = now - rtclock_calend.epoch1;
-		else
-			t64 = 0;
-
-		t32 = (t64 * USEC_PER_SEC) / rtclock_sec_divisor;
-
-		if (t32 > delta)
-			TIME_ADD(sys, 0, microsys, (t32 - delta), USEC_PER_SEC);
-
-		rtclock_calend.epoch = sys;
-		rtclock_calend.microepoch = microsys;
-
-		sys = t64 = now / rtclock_sec_divisor;
-		now -= (t64 * rtclock_sec_divisor);
-		microsys = (now * USEC_PER_SEC) / rtclock_sec_divisor;
-
-		TIME_SUB(rtclock_calend.epoch, sys, rtclock_calend.microepoch, microsys, USEC_PER_SEC);
-	}
-
-	if (total != 0) {
-		int32_t		delta = tickadj;
-
-		if (total > 0) {
-			if (total > bigadj)
-				delta *= 10;
-			if (delta > total)
-				delta = total;
-
-			rtclock_calend.epoch1 = 0;
-		}
-		else {
-			uint64_t		now, t64;
-			uint32_t		sys, microsys;
-
-			if (total < -bigadj)
-				delta *= 10;
-			delta = -delta;
-			if (delta < total)
-				delta = total;
-
-			rtclock_calend.epoch1 = now = mach_absolute_time();
-
-			sys = t64 = now / rtclock_sec_divisor;
-			now -= (t64 * rtclock_sec_divisor);
-			microsys = (now * USEC_PER_SEC) / rtclock_sec_divisor;
-
-			TIME_ADD(rtclock_calend.epoch, sys, rtclock_calend.microepoch, microsys, USEC_PER_SEC);
-		}
-
-		rtclock_calend.adjtotal = total;
-		rtclock_calend.adjdelta = delta;
-
-		interval = rtclock_tick_interval;
-	}
-	else {
-		rtclock_calend.epoch1 = 0;
-		rtclock_calend.adjdelta = rtclock_calend.adjtotal = 0;
-	}
-
-	UNLOCK_RTC(s);
-
-	if (ototal == 0)
-		*secs = *microsecs = 0;
-	else {
-		*secs = ototal / USEC_PER_SEC;
-		*microsecs = ototal % USEC_PER_SEC;
-	}
-
-	return (interval);
-}
-
-uint32_t
-clock_adjust_calendar(void)
-{
-	uint32_t		interval = 0;
-	int32_t			delta;
-	spl_t			s;
-
-	LOCK_RTC(s);
-    commpage_set_timestamp(0,0,0,0);
-
-	delta = rtclock_calend.adjdelta;
-
-	if (delta > 0) {
-		TIME_ADD(rtclock_calend.epoch, 0, rtclock_calend.microepoch, delta, USEC_PER_SEC);
-
-		rtclock_calend.adjtotal -= delta;
-		if (delta > rtclock_calend.adjtotal)
-			rtclock_calend.adjdelta = rtclock_calend.adjtotal;
-	}
-	else
-	if (delta < 0) {
-		uint64_t		now, t64;
-		uint32_t		t32;
-
-		now = mach_absolute_time();
-
-		if (now > rtclock_calend.epoch1)
-			t64 = now - rtclock_calend.epoch1;
-		else
-			t64 = 0;
-
-		rtclock_calend.epoch1 = now;
-
-		t32 = (t64 * USEC_PER_SEC) / rtclock_sec_divisor;
-
-		TIME_ADD(rtclock_calend.epoch, 0, rtclock_calend.microepoch, (t32 + delta), USEC_PER_SEC);
-
-		rtclock_calend.adjtotal -= delta;
-		if (delta < rtclock_calend.adjtotal)
-			rtclock_calend.adjdelta = rtclock_calend.adjtotal;
-
-		if (rtclock_calend.adjdelta == 0) {
-			uint32_t		sys, microsys;
-
-			sys = t64 = now / rtclock_sec_divisor;
-			now -= (t64 * rtclock_sec_divisor);
-			microsys = (now * USEC_PER_SEC) / rtclock_sec_divisor;
-
-			TIME_SUB(rtclock_calend.epoch, sys, rtclock_calend.microepoch, microsys, USEC_PER_SEC);
-
-			rtclock_calend.epoch1 = 0;
-		}
-	}
-
-	if (rtclock_calend.adjdelta != 0)
-		interval = rtclock_tick_interval;
-
-	UNLOCK_RTC(s);
-
-	return (interval);
-}
-
-/*
- *	clock_initialize_calendar:
- *
- *	Set the calendar and related clocks
- *	from the platform clock at boot or
- *	wake event.
- */
-void
-clock_initialize_calendar(void)
-{
-	uint32_t		sys, microsys;
-	uint32_t		microsecs = 0, secs = PEGetGMTTimeOfDay();
-	spl_t			s;
-
-	LOCK_RTC(s);
-    commpage_set_timestamp(0,0,0,0);
-
-	if ((int32_t)secs >= (int32_t)rtclock_boottime) {
-		/*
-		 *	Initialize the boot time based on the platform clock.
-		 */
-		if (rtclock_boottime == 0)
-			rtclock_boottime = secs;
-
-		/*
-		 *	Calculate the new calendar epoch based
-		 *	on the platform clock and the system
-		 *	clock.
-		 */
-		clock_get_system_microtime(&sys, &microsys);
-		TIME_SUB(secs, sys, microsecs, microsys, USEC_PER_SEC);
-
-		/*
-		 *	Set the new calendar epoch.
-		 */
-		rtclock_calend.epoch = secs;
-		rtclock_calend.microepoch = microsecs;
-
-		/*
-		 *	 Cancel any adjustment in progress.
-		 */
-		rtclock_calend.epoch1 = 0;
-		rtclock_calend.adjdelta = rtclock_calend.adjtotal = 0;
-	}
-
-	UNLOCK_RTC(s);
-
-	/*
-	 *	Send host notifications.
-	 */
-	host_notify_calendar_change();
-}
-
-void
-clock_get_boottime_nanotime(
-	uint32_t			*secs,
-	uint32_t			*nanosecs)
-{
-	*secs = rtclock_boottime;
-	*nanosecs = 0;
 }
 
 void
@@ -884,37 +221,11 @@ clock_timebase_info(
 	spl_t		s;
 
 	LOCK_RTC(s);
-	rtclock_timebase_initialized = TRUE;
 	*info = rtclock_timebase_const;
+	rtclock_timebase_initialized = TRUE;
 	UNLOCK_RTC(s);
 }	
 
-void
-clock_set_timer_deadline(
-	uint64_t				deadline)
-{
-	int						decr;
-	uint64_t				abstime;
-	rtclock_timer_t			*mytimer;
-	struct per_proc_info	*pp;
-	spl_t					s;
-
-	s = splclock();
-	pp = getPerProc();
-	mytimer = &pp->rtclock_timer;
-	mytimer->deadline = deadline;
-
-	if (!mytimer->has_expired && (deadline < pp->rtclock_tick_deadline)) {		/* Has the timer already expired or is less that set? */
-		pp->rtcPop = deadline;			/* Yes, set the new rtc pop time */
-		decr = setTimerReq();			/* Start the timers going */
-
-		KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 1)
-									| DBG_FUNC_NONE, decr, 2, 0, 0, 0);
-	}
-
-	splx(s);
-}
-
 void
 clock_set_timer_func(
 	clock_timer_func_t		func)
@@ -927,91 +238,20 @@ clock_set_timer_func(
 	UNLOCK_RTC(s);
 }
 
-/*
- * Real-time clock device interrupt.
- */
 void
-rtclock_intr(struct savearea *ssp) {
-	
-	uint64_t				abstime;
-	int						decr;
-	rtclock_timer_t			*mytimer;
-	struct per_proc_info	*pp;
-
-	pp = getPerProc();
-	mytimer = &pp->rtclock_timer;
-
-	abstime = mach_absolute_time();
-	if (pp->rtclock_tick_deadline <= abstime) {	/* Have we passed the pop time? */
-		clock_deadline_for_periodic_event(rtclock_tick_interval, abstime,
-										  		&pp->rtclock_tick_deadline);
-		hertz_tick(USER_MODE(ssp->save_srr1), ssp->save_srr0);
-		abstime = mach_absolute_time();			/* Refresh the current time since we went away */
-	}
-
-	if (mytimer->deadline <= abstime) {			/* Have we expired the deadline? */
-		mytimer->has_expired = TRUE;			/* Remember that we popped */
-		mytimer->deadline = EndOfAllTime;		/* Set timer request to the end of all time in case we have no more events */
-		(*rtclock_timer_expire)(abstime);		/* Process pop */
-		mytimer->has_expired = FALSE;
-	}
-
-	pp->rtcPop = (pp->rtclock_tick_deadline < mytimer->deadline) ?	/* Get shortest pop */
-		pp->rtclock_tick_deadline :				/* It was the periodic timer */
-		mytimer->deadline;						/* Actually, an event request */
-	
-	decr = setTimerReq();						/* Request the timer pop */
-
-	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 1)
-						  | DBG_FUNC_NONE, decr, 3, 0, 0, 0);
-}
-
-/*
- *	Request an interruption at a specific time 
- *
- *	Sets the decrementer to pop at the right time based on the timebase.
- *	The value is chosen by comparing the rtc request with the power management.
- *	request.  We may add other values at a future time.
- *
- */
- 
-int setTimerReq(void) {
-
-	struct per_proc_info *pp;
-	int decr;
-	uint64_t nexttime;
-	
-	pp = getPerProc();							/* Get per_proc */
-
-	nexttime = pp->rtcPop;						/* Assume main timer */
-
-	decr = setPop((pp->pms.pmsPop < nexttime) ? pp->pms.pmsPop : nexttime);	/* Schedule timer pop */
-
-	return decr;								/* Pass back what we actually set */
-}
-
-static void
-rtclock_alarm_expire(
-	__unused void	*p0,
-	__unused void	*p1)
-{
-	mach_timespec_t		timestamp;
-
-	(void) sysclk_gettime(&timestamp);
-
-	clock_alarm_intr(SYSTEM_CLOCK, &timestamp);
-}
-
-static void
-nanotime_to_absolutetime(
-	uint32_t			secs,
-	uint32_t			nanosecs,
+clock_interval_to_absolutetime_interval(
+	uint32_t			interval,
+	uint32_t			scale_factor,
 	uint64_t			*result)
 {
-	uint32_t	divisor = rtclock_sec_divisor;
+	uint64_t		nanosecs = (uint64_t)interval * scale_factor;
+	uint64_t		t64;
+	uint32_t		divisor;
 
-	*result = ((uint64_t)secs * divisor) +
-				((uint64_t)nanosecs * divisor) / NSEC_PER_SEC;
+	*result = (t64 = nanosecs / NSEC_PER_SEC) *
+							(divisor = rtclock_sec_divisor);
+	nanosecs -= (t64 * NSEC_PER_SEC);
+	*result += (nanosecs * divisor) / NSEC_PER_SEC;
 }
 
 void
@@ -1029,44 +269,29 @@ absolutetime_to_microtime(
 }
 
 void
-clock_interval_to_deadline(
-	uint32_t			interval,
-	uint32_t			scale_factor,
-	uint64_t			*result)
-{
-	uint64_t	abstime;
-
-	clock_get_uptime(result);
-
-	clock_interval_to_absolutetime_interval(interval, scale_factor, &abstime);
-
-	*result += abstime;
-}
-
-void
-clock_interval_to_absolutetime_interval(
-	uint32_t			interval,
-	uint32_t			scale_factor,
-	uint64_t			*result)
+absolutetime_to_nanotime(
+	uint64_t			abstime,
+	uint32_t			*secs,
+	uint32_t			*nanosecs)
 {
-	uint64_t		nanosecs = (uint64_t)interval * scale_factor;
-	uint64_t		t64;
-	uint32_t		divisor;
+	uint64_t	t64;
+	uint32_t	divisor;
 
-	*result = (t64 = nanosecs / NSEC_PER_SEC) *
-							(divisor = rtclock_sec_divisor);
-	nanosecs -= (t64 * NSEC_PER_SEC);
-	*result += (nanosecs * divisor) / NSEC_PER_SEC;
+	*secs = t64 = abstime / (divisor = rtclock_sec_divisor);
+	abstime -= (t64 * divisor);
+	*nanosecs = (abstime * NSEC_PER_SEC) / divisor;
 }
 
 void
-clock_absolutetime_interval_to_deadline(
-	uint64_t			abstime,
+nanotime_to_absolutetime(
+	uint32_t			secs,
+	uint32_t			nanosecs,
 	uint64_t			*result)
 {
-	clock_get_uptime(result);
+	uint32_t	divisor = rtclock_sec_divisor;
 
-	*result += abstime;
+	*result = ((uint64_t)secs * divisor) +
+				((uint64_t)nanosecs * divisor) / NSEC_PER_SEC;
 }
 
 void
@@ -1106,4 +331,3 @@ machine_delay_until(
 		now = mach_absolute_time();
 	} while (now < deadline);
 }
-
diff --git a/osfmk/ppc/rtclock.h b/osfmk/ppc/rtclock.h
index e6f2e516c..190174585 100644
--- a/osfmk/ppc/rtclock.h
+++ b/osfmk/ppc/rtclock.h
@@ -40,10 +40,11 @@
 #ifndef _PPC_RTCLOCK_H_
 #define _PPC_RTCLOCK_H_
 
+#include <kern/etimer.h>
+
 #define EndOfAllTime	0xFFFFFFFFFFFFFFFFULL
 
 extern void rtclock_intr(struct savearea *ssp);
-extern int setTimerReq(void);
 
 #pragma pack(push,4)
 struct rtclock_timer_t  {
diff --git a/osfmk/ppc/serial_io.c b/osfmk/ppc/serial_io.c
index 425570b12..9c546c5c8 100644
--- a/osfmk/ppc/serial_io.c
+++ b/osfmk/ppc/serial_io.c
@@ -107,8 +107,6 @@ extern unsigned int disableSerialOuput;
 int	serial_initted = 0;
 unsigned int scc_parm_done = 0;
 
-extern unsigned int serialmode;
-
 static struct scc_byte {
 	unsigned char	reg;
 	unsigned char	val;
@@ -405,6 +403,17 @@ again:
 	return c;
 }
 
+
+/*
+ *	This front-ends scc_getc to make some intel changes easier
+ */
+ 
+int _serial_getc(int unit, int line, boolean_t wait, boolean_t raw) {
+
+	return(scc_getc(unit, line, wait, raw));
+
+}
+
 /*
  * Put a char on a specific SCC line
  * use splhigh since we might be doing a printf in high spl'd code
@@ -653,55 +662,5 @@ scc_param(struct scc_tty *tp)
 
 }
 
-/*
- *  This routine will start a thread that polls the serial port, listening for
- *  characters that have been typed.
- */
-
-void
-serial_keyboard_init(void)
-{
-	kern_return_t	result;
-	thread_t		thread;
-
-	if(!(serialmode & 2)) return;		/* Leave if we do not want a serial console */
-
-	kprintf("Serial keyboard started\n");
-	result = kernel_thread_start_priority((thread_continue_t)serial_keyboard_start, NULL, MAXPRI_KERNEL, &thread);
-	if (result != KERN_SUCCESS)
-		panic("serial_keyboard_init");
-
-	thread_deallocate(thread);
-}
-
-void
-serial_keyboard_start(void)
-{
-	serial_keyboard_poll();			/* Go see if there are any characters pending now */
-	panic("serial_keyboard_start: we can't get back here\n");
-}
-
-static int ptestxxx = 0;
-
-void
-serial_keyboard_poll(void)
-{
-	int chr;
-	uint64_t next;
-	extern void cons_cinput(char ch);	/* The BSD routine that gets characters */
-
-
-	while(1) {				/* Do this for a while */
-		chr = scc_getc(0, 1, 0, 1);	/* Get a character if there is one */
-		if(chr < 0) break;		/* The serial buffer is empty */
-		cons_cinput((char)chr);		/* Buffer up the character */
-	}
-
-	clock_interval_to_deadline(16, 1000000, &next);	/* Get time of pop */
-
-	assert_wait_deadline((event_t)serial_keyboard_poll, THREAD_UNINT, next);	/* Show we are "waiting" */
-	thread_block((thread_continue_t)serial_keyboard_poll);	/* Wait for it */
-	panic("serial_keyboard_poll: Shouldn't never ever get here...\n");
-}
 
 #endif	/* NSCC > 0 */
diff --git a/osfmk/ppc/serial_io.h b/osfmk/ppc/serial_io.h
index 7f794c565..34af7efc7 100644
--- a/osfmk/ppc/serial_io.h
+++ b/osfmk/ppc/serial_io.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -118,21 +118,6 @@ extern int		scc_getc(
 				boolean_t		wait,
 				boolean_t		raw);
 
-/* Functions in serial_console.c for switching between serial and video
-   consoles.  */
-extern boolean_t	console_is_serial(void);
-extern int		switch_to_serial_console(
-				void);
-
-extern int		switch_to_video_console(
-				void);
-
-extern void		switch_to_old_console(
-				int			old_console);
-
-void serial_keyboard_init(void);
-void serial_keyboard_start(void);
-void serial_keyboard_poll(void);
 
 
 /*
diff --git a/osfmk/ppc/skiplists.s b/osfmk/ppc/skiplists.s
index 31d733166..69a9dccbb 100644
--- a/osfmk/ppc/skiplists.s
+++ b/osfmk/ppc/skiplists.s
@@ -368,7 +368,7 @@ LEXT(mapSearchFull)
             ;	r7 = current skip list number * 8
             ;	r8 = ptr to skip list vector of mapping pointed to by r9
             ;	r9 = prev ptr, ie highest mapping that comes before search target (initially the pmap)
-            ;  r10 = lowest expected next va, 0 at the beginning of the search 
+            ;  r10 = lowest expected next va, 0 at the beginning of the search
             ;  r12 = ptr to the skipListPrev vector in the per-proc
             
             .align	5
@@ -451,7 +451,7 @@ mapSrchFull64Found:							; WARNING: can drop down to here
             ;	r7 = current skip list number * 8
             ;	r8 = ptr to skip list vector of mapping pointed to by r9
             ;	r9 = prev ptr, ie highest mapping that comes before search target (initially the pmap)
-            ;  r10 = lowest expected next va, 0 at the beginning of the search 
+            ;  r10 = lowest expected next va, 0 at the beginning of the search
             ;  r12 = ptr to the skipListPrev vector in the per-proc
             
             .align	4
@@ -480,7 +480,7 @@ mapSrchFull32b:
             la		r8,mpList0+4(r3)		; point to skip list vector in this mapping
             mr		r9,r3					; current becomes previous
             lwzx	r3,r7,r8				; get ptr to next mapping in current list
-            addi	r10,r4,0x1000			; Get the lowest VA we can get next
+            addi	r10,r4,0x1000					; Get the lowest VA we can get next
 mapSrchFull32c:
             mr.		r3,r3					; next becomes current
             bne+	mapSrchFull32a			; was another, so loop
diff --git a/osfmk/ppc/status.c b/osfmk/ppc/status.c
index a78b121a7..1910f6adc 100644
--- a/osfmk/ppc/status.c
+++ b/osfmk/ppc/status.c
@@ -1330,6 +1330,19 @@ thread_adjuserstack(thread_t thread, int adjust)
 	
 }    
 
+void
+thread_setsinglestep(thread_t thread, int on)
+{
+	savearea *sv;
+	
+	sv = get_user_regs(thread);	/* Get the user state registers */
+	
+	if (on)
+	        sv->save_srr1 |= MASK(MSR_SE);
+	else
+	        sv->save_srr1 &= ~MASK(MSR_SE);
+}    
+
 /*
  * thread_setentrypoint:
  *
diff --git a/osfmk/ppc/trap.c b/osfmk/ppc/trap.c
index 1182e0e0e..1b992c852 100644
--- a/osfmk/ppc/trap.c
+++ b/osfmk/ppc/trap.c
@@ -88,6 +88,7 @@ extern int not_in_kdp;
 #define PROT_RO		(VM_PROT_READ)
 #define PROT_RW		(VM_PROT_READ|VM_PROT_WRITE)
 
+
 /* A useful macro to update the ppc_exception_state in the PCB
  * before calling doexception
  */
@@ -134,11 +135,11 @@ struct savearea *trap(int trapno,
 
 	myast = ast_pending();
 	if(perfASTHook) {
-		if(*myast & AST_PPC_CHUD_ALL) {
+		if(*myast & AST_CHUD_ALL) {
 			perfASTHook(trapno, ssp, dsisr, (unsigned int)dar);
 		}
 	} else {
-		*myast &= ~AST_PPC_CHUD_ALL;
+		*myast &= ~AST_CHUD_ALL;
 	}
 
 	if(perfTrapHook) {							/* Is there a hook? */
@@ -375,7 +376,7 @@ struct savearea *trap(int trapno,
 			map = kernel_map;
 			
 			code = vm_fault(map, vm_map_trunc_page(ssp->save_srr0),
-					PROT_EXEC, FALSE, THREAD_UNINT, NULL, vm_map_trunc_page(0));
+					(PROT_EXEC | PROT_RO), FALSE, THREAD_UNINT, NULL, vm_map_trunc_page(0));
 
 			if (code != KERN_SUCCESS) {
 				unresolved_kernel_trap(trapno, ssp, dsisr, dar, NULL);
@@ -604,7 +605,7 @@ struct savearea *trap(int trapno,
 			map = thread->map;
 			
 			code = vm_fault(map, vm_map_trunc_page(ssp->save_srr0),
-				PROT_EXEC, FALSE, THREAD_ABORTSAFE, NULL, vm_map_trunc_page(0));
+					(PROT_EXEC | PROT_RO), FALSE, THREAD_ABORTSAFE, NULL, vm_map_trunc_page(0));
 
 			if ((code != KERN_SUCCESS) && (code != KERN_ABORTED)) {
 				UPDATE_PPC_EXCEPTION_STATE;
diff --git a/osfmk/ppc/vmachmon.c b/osfmk/ppc/vmachmon.c
index 362eb864c..a501ac663 100644
--- a/osfmk/ppc/vmachmon.c
+++ b/osfmk/ppc/vmachmon.c
@@ -622,7 +622,7 @@ int vmm_init_context(struct savearea *save)
 	hw_atomic_add((int *)&saveanchor.savetarget, 2);	/* Account for the number of extra saveareas we think we might "need" */
 
 	pmap_t hpmap = act->map->pmap;						/* Get host pmap */
-	pmap_t gpmap = pmap_create(0);						/* Make a fresh guest pmap */
+	pmap_t gpmap = pmap_create(0, FALSE);					/* Make a fresh guest pmap */
 	if (gpmap) {										/* Did we succeed ? */
 		CTable->vmmAdsp[cvi] = gpmap;					/* Remember guest pmap for new context */
 		if (lowGlo.lgVMMforcedFeats & vmmGSA) {			/* Forcing on guest shadow assist ? */
@@ -981,7 +981,7 @@ kern_return_t vmm_map_page(
 	map = current_thread()->map;				/* Get the host's map */
 
 	if (pmap->pmapFlags & pmapVMgsaa) {			/* Guest shadow assist active ? */
-		ret = hw_res_map_gv(map->pmap, pmap, cva, ava, getProtPPC(prot));
+		ret = hw_res_map_gv(map->pmap, pmap, cva, ava, getProtPPC(prot, TRUE));
 												/* Attempt to resume an existing gv->phys mapping */
 		if (mapRtOK != ret) {					/* Nothing to resume, construct a new mapping */
 			
@@ -1023,7 +1023,7 @@ kern_return_t vmm_map_page(
 			if (pattr & mmFlgCInhib)  wimg |= 0x4;
 			if (pattr & mmFlgGuarded) wimg |= 0x1;
 			unsigned int mflags = (pindex << 16) | mpGuest;
-			addr64_t	 gva = ((ava & ~mpHWFlags) | (wimg << 3) | getProtPPC(prot));
+			addr64_t	 gva = ((ava & ~mpHWFlags) | (wimg << 3) | getProtPPC(prot, TRUE));
 			
 			hw_add_map_gv(map->pmap, pmap, gva, mflags, mp->mpPAddr);
 												/* Construct new guest->phys mapping */
diff --git a/osfmk/vm/bsd_vm.c b/osfmk/vm/bsd_vm.c
index 5c777b715..b61fd9077 100644
--- a/osfmk/vm/bsd_vm.c
+++ b/osfmk/vm/bsd_vm.c
@@ -49,7 +49,6 @@
 #include <default_pager/default_pager_object_server.h>
 
 #include <vm/vm_map.h>
-#include <vm/vm_kern.h>
 #include <vm/vm_pageout.h>
 #include <vm/memory_object.h>
 #include <vm/vm_pageout.h>
@@ -114,11 +113,22 @@ get_vm_end(
  * BSD VNODE PAGER 
  */
 
-/* until component support available */
-int	vnode_pager_workaround;
+const struct memory_object_pager_ops vnode_pager_ops = {
+	vnode_pager_reference,
+	vnode_pager_deallocate,
+	vnode_pager_init,
+	vnode_pager_terminate,
+	vnode_pager_data_request,
+	vnode_pager_data_return,
+	vnode_pager_data_initialize,
+	vnode_pager_data_unlock,
+	vnode_pager_synchronize,
+	vnode_pager_unmap,
+	"vnode pager"
+};
 
 typedef struct vnode_pager {
-	int 			*pager;		/* pager workaround pointer  */
+	memory_object_pager_ops_t pager_ops;	/* == &vnode_pager_ops	     */
 	unsigned int		pager_ikot;	/* JMM: fake ip_kotype()     */
 	unsigned int		ref_count;	/* reference count	     */
 	memory_object_control_t control_handle;	/* mem object control handle */
@@ -209,6 +219,11 @@ macx_triggers(
 		return EINVAL;
 	}
 
+       if (default_pager_init_flag == 0) {
+               start_def_pager(NULL);
+               default_pager_init_flag = 1;
+       }
+
 	if (flags & SWAP_ENCRYPT_ON) {
 		/* ENCRYPTED SWAP: tell default_pager to encrypt */
 		default_pager_triggers(default_pager,
@@ -430,6 +445,9 @@ vnode_pager_bootstrap(void)
 	size = (vm_size_t) sizeof(struct vnode_pager);
 	vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size,
 				PAGE_SIZE, "vnode pager structures");
+#ifdef __i386__
+	apple_protect_pager_bootstrap();
+#endif	/* __i386__ */
 	return;
 }
 
@@ -543,12 +561,53 @@ vnode_pager_get_object_size(
 {
 	vnode_pager_t	vnode_object;
 
+	if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
+		*length = 0;
+		return KERN_INVALID_ARGUMENT;
+	}
+
 	vnode_object = vnode_pager_lookup(mem_obj);
 
 	*length = vnode_pager_get_filesize(vnode_object->vnode_handle);
 	return KERN_SUCCESS;
 }
 
+kern_return_t
+vnode_pager_get_object_pathname(
+	memory_object_t		mem_obj,
+	char			*pathname,
+	vm_size_t		*length_p)
+{
+	vnode_pager_t	vnode_object;
+
+	if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	vnode_object = vnode_pager_lookup(mem_obj);
+
+	return vnode_pager_get_pathname(vnode_object->vnode_handle,
+					pathname,
+					length_p);
+}
+
+kern_return_t
+vnode_pager_get_object_filename(
+	memory_object_t	mem_obj,
+	char		**filename)
+{
+	vnode_pager_t	vnode_object;
+
+	if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	vnode_object = vnode_pager_lookup(mem_obj);
+
+	return vnode_pager_get_filename(vnode_object->vnode_handle,
+					filename);
+}
+
 /*
  *
  */
@@ -813,7 +872,7 @@ vnode_pager_cluster_read(
 	if (kret == 1) {
 		int	uplflags;
 		upl_t	upl = NULL;
-		int	count = 0;
+		unsigned int	count = 0;
 		kern_return_t	kr;
 
 		uplflags = (UPL_NO_SYNC |
@@ -852,7 +911,7 @@ vnode_pager_release_from_cache(
 		int	*cnt)
 {
 	memory_object_free_from_cache(
-			&realhost, &vnode_pager_workaround, cnt);
+			&realhost, &vnode_pager_ops, cnt);
 }
 
 /*
@@ -875,7 +934,7 @@ vnode_object_create(
 	 * we reserve the second word in the object for a fake ip_kotype
 	 * setting - that will tell vm_map to use it as a memory object.
 	 */
-	vnode_object->pager = &vnode_pager_workaround;
+	vnode_object->pager_ops = &vnode_pager_ops;
 	vnode_object->pager_ikot = IKOT_MEMORY_OBJECT;
 	vnode_object->ref_count = 1;
 	vnode_object->control_handle = MEMORY_OBJECT_CONTROL_NULL;
@@ -894,7 +953,193 @@ vnode_pager_lookup(
 	vnode_pager_t	vnode_object;
 
 	vnode_object = (vnode_pager_t)name;
-	assert(vnode_object->pager == &vnode_pager_workaround);
+	assert(vnode_object->pager_ops == &vnode_pager_ops);
 	return (vnode_object);
 }
 
+
+/*********************** proc_info implementation *************/
+
+#include <sys/bsdtask_info.h>
+
+static int fill_vnodeinfoforaddr( vm_map_entry_t entry, uint32_t * vnodeaddr, uint32_t * vid);
+
+
+int
+fill_procregioninfo(task_t task, uint64_t arg, struct proc_regioninfo_internal *pinfo, uint32_t  *vnodeaddr, uint32_t  *vid)
+{
+
+	vm_map_t map = task->map;
+	vm_map_offset_t	address = (vm_map_offset_t )arg;
+	vm_map_entry_t		tmp_entry;
+	vm_map_entry_t		entry;
+	vm_map_offset_t		start;
+	vm_region_extended_info_data_t extended;
+	vm_region_top_info_data_t top;
+
+
+	if (map == VM_MAP_NULL) 
+		return(0);
+
+	    vm_map_lock_read(map);
+
+	    start = address;
+	    if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
+		if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
+			vm_map_unlock_read(map);
+		   	return(0);
+		}
+	    } else {
+		entry = tmp_entry;
+	    }
+
+	    start = entry->vme_start;
+
+	    pinfo->pri_offset = entry->offset;
+	    pinfo->pri_protection = entry->protection;
+	    pinfo->pri_max_protection = entry->max_protection;
+	    pinfo->pri_inheritance = entry->inheritance;
+	    pinfo->pri_behavior = entry->behavior;
+	    pinfo->pri_user_wired_count = entry->user_wired_count;
+	    pinfo->pri_user_tag = entry->alias;
+
+	    if (entry->is_sub_map) {
+		pinfo->pri_flags |= PROC_REGION_SUBMAP;
+	    } else {
+		if (entry->is_shared)
+			pinfo->pri_flags |= PROC_REGION_SHARED;
+	    }
+
+
+	    extended.protection = entry->protection;
+	    extended.user_tag = entry->alias;
+	    extended.pages_resident = 0;
+	    extended.pages_swapped_out = 0;
+	    extended.pages_shared_now_private = 0;
+	    extended.pages_dirtied = 0;
+	    extended.external_pager = 0;
+	    extended.shadow_depth = 0;
+
+	    vm_map_region_walk(map, start, entry, entry->offset, entry->vme_end - start, &extended);
+
+	    if (extended.external_pager && extended.ref_count == 2 && extended.share_mode == SM_SHARED)
+	            extended.share_mode = SM_PRIVATE;
+
+	    top.private_pages_resident = 0;
+	    top.shared_pages_resident = 0;
+	    vm_map_region_top_walk(entry, &top);
+
+	
+	    pinfo->pri_pages_resident = extended.pages_resident;
+	    pinfo->pri_pages_shared_now_private = extended.pages_shared_now_private;
+	    pinfo->pri_pages_swapped_out = extended.pages_swapped_out;
+	    pinfo->pri_pages_dirtied = extended.pages_dirtied;
+	    pinfo->pri_ref_count = extended.ref_count;
+	    pinfo->pri_shadow_depth = extended.shadow_depth;
+	    pinfo->pri_share_mode = extended.share_mode;
+
+	    pinfo->pri_private_pages_resident = top.private_pages_resident;
+	    pinfo->pri_shared_pages_resident = top.shared_pages_resident;
+	    pinfo->pri_obj_id = top.obj_id;
+		
+	    pinfo->pri_address = (uint64_t)start;
+	    pinfo->pri_size = (uint64_t)(entry->vme_end - start);
+	    pinfo->pri_depth = 0;
+	
+	    if ((vnodeaddr != 0) && (entry->is_sub_map == 0)) {
+		*vnodeaddr = (uint32_t)0;
+
+		if (fill_vnodeinfoforaddr(entry, vnodeaddr, vid) ==0) {
+			vm_map_unlock_read(map);
+			return(1);
+		}
+	    }
+
+	    vm_map_unlock_read(map);
+	    return(1);
+}
+
+static int
+fill_vnodeinfoforaddr(
+	vm_map_entry_t			entry,
+	uint32_t * vnodeaddr,
+	uint32_t * vid)
+{
+	vm_object_t	top_object, object;
+	memory_object_t memory_object;
+	memory_object_pager_ops_t pager_ops;
+	kern_return_t	kr;
+	int		shadow_depth;
+
+
+	if (entry->is_sub_map) {
+		return(0);
+	} else {
+		/*
+		 * The last object in the shadow chain has the
+		 * relevant pager information.
+		 */
+		top_object = entry->object.vm_object;
+		if (top_object == VM_OBJECT_NULL) {
+			object = VM_OBJECT_NULL;
+			shadow_depth = 0;
+		} else {
+			vm_object_lock(top_object);
+			for (object = top_object, shadow_depth = 0;
+			     object->shadow != VM_OBJECT_NULL;
+			     object = object->shadow, shadow_depth++) {
+				vm_object_lock(object->shadow);
+				vm_object_unlock(object);
+			}
+		}
+	}
+
+	if (object == VM_OBJECT_NULL) {
+		return(0);
+	} else if (object->internal) {
+		vm_object_unlock(object);
+		return(0);
+	} else if (! object->pager_ready ||
+		   object->terminating ||
+		   ! object->alive) {
+		vm_object_unlock(object);
+		return(0);
+	} else {
+		memory_object = object->pager;
+		pager_ops = memory_object->mo_pager_ops;
+		if (pager_ops == &vnode_pager_ops) {
+			kr = vnode_pager_get_object_vnode(
+				memory_object,
+				vnodeaddr, vid);
+			if (kr != KERN_SUCCESS) {
+				vm_object_unlock(object);
+				return(0);
+			}
+		} else {
+			vm_object_unlock(object);
+			return(0);
+		}
+	}
+	vm_object_unlock(object);
+	return(1);
+}
+
+kern_return_t 
+vnode_pager_get_object_vnode (
+	memory_object_t		mem_obj,
+	uint32_t * vnodeaddr,
+	uint32_t * vid)
+{
+	vnode_pager_t	vnode_object;
+
+	vnode_object = vnode_pager_lookup(mem_obj);
+	if (vnode_object->vnode_handle)  {
+		*vnodeaddr = (uint32_t)vnode_object->vnode_handle;
+		*vid = (uint32_t)vnode_vid((void *)vnode_object->vnode_handle);	
+
+		return(KERN_SUCCESS);
+	}
+	
+	return(KERN_FAILURE);
+}
+
diff --git a/osfmk/vm/device_vm.c b/osfmk/vm/device_vm.c
index fa7e48456..4fe6ef023 100644
--- a/osfmk/vm/device_vm.c
+++ b/osfmk/vm/device_vm.c
@@ -63,12 +63,27 @@
 
 
 /* until component support available */
-int	device_pager_workaround;
+const struct memory_object_pager_ops device_pager_ops = {
+	device_pager_reference,
+	device_pager_deallocate,
+	device_pager_init,
+	device_pager_terminate,
+	device_pager_data_request,
+	device_pager_data_return,
+	device_pager_data_initialize,
+	device_pager_data_unlock,
+	device_pager_synchronize,
+	device_pager_unmap,
+	"device pager"
+};
 
 typedef int device_port_t;
 
+/*
+ * The start of "struct device_pager" MUST match a "struct memory_object".
+ */
 typedef struct device_pager {
-	int 		*pager;		/* pager workaround pointer	*/
+	memory_object_pager_ops_t pager_ops; /* == &device_pager_ops	*/
 	unsigned int	pager_ikot;	/* fake ip_kotype() 		*/
 	unsigned int	ref_count;	/* reference count		*/
 	memory_object_control_t	control_handle;	/* mem object's cntrl handle */
@@ -167,7 +182,7 @@ device_pager_populate_object(
 		return kr;
 
 	if(!vm_object->phys_contiguous) {
-		int null_size = 0;
+		unsigned int null_size = 0;
         	kr = vm_object_upl_request(vm_object,
              		(vm_object_offset_t)offset, size, &upl,  NULL,
 			&null_size, (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE)); 
@@ -193,7 +208,7 @@ device_pager_lookup(
 	device_pager_t	device_object;
 
 	device_object = (device_pager_t)name;
-	assert(device_object->pager == &device_pager_workaround);
+	assert(device_object->pager_ops == &device_pager_ops);
 	return (device_object);
 }
 
@@ -260,9 +275,11 @@ device_pager_init(
 /*ARGSUSED6*/
 kern_return_t
 device_pager_data_return(
-	memory_object_t		mem_obj,
-	memory_object_offset_t	offset,
-	vm_size_t		data_cnt,
+	memory_object_t			mem_obj,
+	memory_object_offset_t		offset,
+	vm_size_t			data_cnt,
+	__unused memory_object_offset_t	*resid_offset,
+	__unused int			*io_error,
 	__unused boolean_t		dirty,
 	__unused boolean_t		kernel_copy,
 	__unused int			upl_flags)  
@@ -342,7 +359,7 @@ device_pager_deallocate(
 			 * We still have to release the "memory object control"
 			 * handle.
 			 */
-			assert(device_control->object == VM_OBJECT_NULL);
+			assert(device_control->moc_object == VM_OBJECT_NULL);
 			memory_object_control_deallocate(device_control);
 			device_object->control_handle =
 				MEMORY_OBJECT_CONTROL_NULL;
@@ -425,7 +442,7 @@ device_object_create()
 	device_object = (struct device_pager *) zalloc(device_pager_zone);
 	if (device_object == DEVICE_PAGER_NULL)
 		return(DEVICE_PAGER_NULL);
-	device_object->pager = &device_pager_workaround;
+	device_object->pager_ops = &device_pager_ops;
 	device_object->pager_ikot = IKOT_MEMORY_OBJECT;
 	device_object->ref_count = 1;
 	device_object->control_handle = MEMORY_OBJECT_CONTROL_NULL;
diff --git a/osfmk/vm/memory_object.c b/osfmk/vm/memory_object.c
index 94c912a86..7aef4fc18 100644
--- a/osfmk/vm/memory_object.c
+++ b/osfmk/vm/memory_object.c
@@ -1547,15 +1547,17 @@ memory_object_iopl_request(
 			vm_object_reference(object);
 			named_entry_unlock(named_entry);
 		}
-	} else  {
+	} else if (ip_kotype(port) == IKOT_MEM_OBJ_CONTROL) {
 		memory_object_control_t	control;
-		control = (memory_object_control_t)port->ip_kobject;
+		control = (memory_object_control_t) port;
 		if (control == NULL)
 			return (KERN_INVALID_ARGUMENT);
 		object = memory_object_control_to_vm_object(control);
 		if (object == VM_OBJECT_NULL)
 			return (KERN_INVALID_ARGUMENT);
 		vm_object_reference(object);
+	} else {
+		return KERN_INVALID_ARGUMENT;
 	}
 	if (object == VM_OBJECT_NULL)
 		return (KERN_INVALID_ARGUMENT);
@@ -1817,152 +1819,12 @@ memory_object_page_op(
 	int			*flags)
 {
 	vm_object_t		object;
-	vm_page_t		dst_page;
-
 
 	object = memory_object_control_to_vm_object(control);
 	if (object == VM_OBJECT_NULL)
 		return (KERN_INVALID_ARGUMENT);
 
-	vm_object_lock(object);
-
-	if(ops & UPL_POP_PHYSICAL) {
-		if(object->phys_contiguous) {
-			if (phys_entry) {
-				*phys_entry = (ppnum_t)
-					(object->shadow_offset >> 12);
-			}
-			vm_object_unlock(object);
-			return KERN_SUCCESS;
-		} else {
-			vm_object_unlock(object);
-			return KERN_INVALID_OBJECT;
-		}
-	}
-	if(object->phys_contiguous) {
-		vm_object_unlock(object);
-		return KERN_INVALID_OBJECT;
-	}
-
-	while(TRUE) {
-		if((dst_page = vm_page_lookup(object,offset)) == VM_PAGE_NULL) {
-			vm_object_unlock(object);
-			return KERN_FAILURE;
-		}
-
-		/* Sync up on getting the busy bit */
-		if((dst_page->busy || dst_page->cleaning) && 
-			   (((ops & UPL_POP_SET) && 
-			   (ops & UPL_POP_BUSY)) || (ops & UPL_POP_DUMP))) {
-			/* someone else is playing with the page, we will */
-			/* have to wait */
-			PAGE_SLEEP(object, dst_page, THREAD_UNINT);
-			continue;
-		}
-
-		if (ops & UPL_POP_DUMP) {
-		        vm_page_lock_queues();
-
-			if (dst_page->no_isync == FALSE)
-			        pmap_disconnect(dst_page->phys_page);
-			vm_page_free(dst_page);
-
-			vm_page_unlock_queues();
-			break;
-		}
-
-		if (flags) {
-		        *flags = 0;
-
-			/* Get the condition of flags before requested ops */
-			/* are undertaken */
-
-			if(dst_page->dirty) *flags |= UPL_POP_DIRTY;
-			if(dst_page->pageout) *flags |= UPL_POP_PAGEOUT;
-			if(dst_page->precious) *flags |= UPL_POP_PRECIOUS;
-			if(dst_page->absent) *flags |= UPL_POP_ABSENT;
-			if(dst_page->busy) *flags |= UPL_POP_BUSY;
-		}
-
-		/* The caller should have made a call either contingent with */
-		/* or prior to this call to set UPL_POP_BUSY */
-		if(ops & UPL_POP_SET) {
-			/* The protection granted with this assert will */
-			/* not be complete.  If the caller violates the */
-			/* convention and attempts to change page state */
-			/* without first setting busy we may not see it */
-			/* because the page may already be busy.  However */
-			/* if such violations occur we will assert sooner */
-			/* or later. */
-			assert(dst_page->busy || (ops & UPL_POP_BUSY));
-			if (ops & UPL_POP_DIRTY) dst_page->dirty = TRUE;
-			if (ops & UPL_POP_PAGEOUT) dst_page->pageout = TRUE;
-			if (ops & UPL_POP_PRECIOUS) dst_page->precious = TRUE;
-			if (ops & UPL_POP_ABSENT) dst_page->absent = TRUE;
-			if (ops & UPL_POP_BUSY) dst_page->busy = TRUE;
-		}
-
-		if(ops & UPL_POP_CLR) {
-			assert(dst_page->busy);
-			if (ops & UPL_POP_DIRTY) dst_page->dirty = FALSE;
-			if (ops & UPL_POP_PAGEOUT) dst_page->pageout = FALSE;
-			if (ops & UPL_POP_PRECIOUS) dst_page->precious = FALSE;
-			if (ops & UPL_POP_ABSENT) dst_page->absent = FALSE;
-			if (ops & UPL_POP_BUSY) {
-			        dst_page->busy = FALSE;
-				PAGE_WAKEUP(dst_page);
-			}
-		}
-
-		if (dst_page->encrypted) {
-			/*
-			 * ENCRYPTED SWAP:
-			 * We need to decrypt this encrypted page before the
-			 * caller can access its contents.
-			 * But if the caller really wants to access the page's
-			 * contents, they have to keep the page "busy".
-			 * Otherwise, the page could get recycled or re-encrypted
-			 * at any time.
-			 */
-			if ((ops & UPL_POP_SET) && (ops & UPL_POP_BUSY) &&
-			    dst_page->busy) {
-				/*
-				 * The page is stable enough to be accessed by
-				 * the caller, so make sure its contents are
-				 * not encrypted.
-				 */
-				vm_page_decrypt(dst_page, 0);
-			} else {
-				/*
-				 * The page is not busy, so don't bother
-				 * decrypting it, since anything could
-				 * happen to it between now and when the
-				 * caller wants to access it.
-				 * We should not give the caller access
-				 * to this page.
-				 */
-				assert(!phys_entry);
-			}
-		}
-
-		if (phys_entry) {
-			/*
-			 * The physical page number will remain valid
-			 * only if the page is kept busy.
-			 * ENCRYPTED SWAP: make sure we don't let the
-			 * caller access an encrypted page.
-			 */
-			assert(dst_page->busy);
-			assert(!dst_page->encrypted);
-			*phys_entry = dst_page->phys_page;
-		}
-
-		break;
-	}
-
-	vm_object_unlock(object);
-	return KERN_SUCCESS;
-				
+	return vm_object_page_op(object, offset, ops, phys_entry, flags);
 }
 
 /*
@@ -1983,71 +1845,17 @@ memory_object_range_op(
 	int                     ops,
 	int                     *range)
 {
-        memory_object_offset_t	offset;
 	vm_object_t		object;
-	vm_page_t		dst_page;
 
 	object = memory_object_control_to_vm_object(control);
 	if (object == VM_OBJECT_NULL)
 		return (KERN_INVALID_ARGUMENT);
 
-	if (object->resident_page_count == 0) {
-	        if (range) {
-		        if (ops & UPL_ROP_PRESENT)
-			        *range = 0;
-			else
-			        *range = offset_end - offset_beg;
-		}
-		return KERN_SUCCESS;
-	}
-	vm_object_lock(object);
-
-	if (object->phys_contiguous) {
-		vm_object_unlock(object);
-	        return KERN_INVALID_OBJECT;
-	}
-	
-	offset = offset_beg;
-
-	while (offset < offset_end) {
-		dst_page = vm_page_lookup(object, offset);
-		if (dst_page != VM_PAGE_NULL) {
-			if (ops & UPL_ROP_DUMP) {
-				if (dst_page->busy || dst_page->cleaning) {
-				        /*
-					 * someone else is playing with the 
-					 * page, we will have to wait
-					 */
-				        PAGE_SLEEP(object, 
-						dst_page, THREAD_UNINT);
-					/*
-					 * need to relook the page up since it's
-					 * state may have changed while we slept
-					 * it might even belong to a different object
-					 * at this point
-					 */
-					continue;
-				}
-				vm_page_lock_queues();
-
-				if (dst_page->no_isync == FALSE)
-				        pmap_disconnect(dst_page->phys_page);
-				vm_page_free(dst_page);
-
-				vm_page_unlock_queues();
-			} else if (ops & UPL_ROP_ABSENT)
-			        break;
-		} else if (ops & UPL_ROP_PRESENT)
-		        break;
-
-		offset += PAGE_SIZE;
-	}
-	vm_object_unlock(object);
-
-	if (range)
-	        *range = offset - offset_beg;
-
-	return KERN_SUCCESS;
+	return vm_object_range_op(object,
+				  offset_beg,
+				  offset_end,
+				  ops,
+				  range);
 }
 
 
@@ -2090,8 +1898,10 @@ memory_object_control_allocate(
 	memory_object_control_t control;
 
 	control = (memory_object_control_t)zalloc(mem_obj_control_zone);
-	if (control != MEMORY_OBJECT_CONTROL_NULL)
-		control->object = object;
+	if (control != MEMORY_OBJECT_CONTROL_NULL) {
+		control->moc_object = object;
+		control->moc_ikot = IKOT_MEM_OBJ_CONTROL; /* fake ip_kotype */
+	}
 	return (control);
 }
 
@@ -2100,19 +1910,20 @@ memory_object_control_collapse(
 	memory_object_control_t control,		       
 	vm_object_t		object)
 {		       
-	assert((control->object != VM_OBJECT_NULL) &&
-	       (control->object != object));
-	control->object = object;
+	assert((control->moc_object != VM_OBJECT_NULL) &&
+	       (control->moc_object != object));
+	control->moc_object = object;
 }
 
 __private_extern__ vm_object_t
 memory_object_control_to_vm_object(
 	memory_object_control_t	control)
 {
-	if (control == MEMORY_OBJECT_CONTROL_NULL)
+	if (control == MEMORY_OBJECT_CONTROL_NULL ||
+	    control->moc_ikot != IKOT_MEM_OBJ_CONTROL)
 		return VM_OBJECT_NULL;
 
-	return (control->object);
+	return (control->moc_object);
 }
 
 memory_object_control_t
@@ -2153,8 +1964,8 @@ void
 memory_object_control_disable(
 	memory_object_control_t	control)
 {
-	assert(control->object != VM_OBJECT_NULL);
-	control->object = VM_OBJECT_NULL;
+	assert(control->moc_object != VM_OBJECT_NULL);
+	control->moc_object = VM_OBJECT_NULL;
 }
 
 void
@@ -2191,30 +2002,16 @@ convert_memory_object_to_port(
 void memory_object_reference(
 	memory_object_t memory_object)
 {
-
-#ifdef	MACH_BSD
-	if (memory_object->pager == &vnode_pager_workaround) {
-		vnode_pager_reference(memory_object);
-	} else if (memory_object->pager == &device_pager_workaround) {
-		device_pager_reference(memory_object);
-	} else
-#endif
-		dp_memory_object_reference(memory_object);
+	(memory_object->mo_pager_ops->memory_object_reference)(
+		memory_object);
 }
 
 /* Routine memory_object_deallocate */
 void memory_object_deallocate(
 	memory_object_t memory_object)
 {
-
-#ifdef	MACH_BSD
-	if (memory_object->pager == &vnode_pager_workaround) {
-		vnode_pager_deallocate(memory_object);
-	} else if (memory_object->pager == &device_pager_workaround) {
-		device_pager_deallocate(memory_object);
-	} else
-#endif
-		dp_memory_object_deallocate(memory_object);
+	(memory_object->mo_pager_ops->memory_object_deallocate)(
+		 memory_object);
 }
 
 
@@ -2226,20 +2023,10 @@ kern_return_t memory_object_init
 	memory_object_cluster_size_t memory_object_page_size
 )
 {
-#ifdef	MACH_BSD
-	if (memory_object->pager == &vnode_pager_workaround) {
-		return vnode_pager_init(memory_object,
-					memory_control,
-					memory_object_page_size);
-	} else if (memory_object->pager == &device_pager_workaround) {
-		return device_pager_init(memory_object,
-					 memory_control,
-					 memory_object_page_size);
-	} else
-#endif
-		return dp_memory_object_init(memory_object,
-					     memory_control,
-					     memory_object_page_size);
+	return (memory_object->mo_pager_ops->memory_object_init)(
+		memory_object,
+		memory_control,
+		memory_object_page_size);
 }
 
 /* Routine memory_object_terminate */
@@ -2248,14 +2035,8 @@ kern_return_t memory_object_terminate
 	memory_object_t memory_object
 )
 {
-#ifdef	MACH_BSD
-	if (memory_object->pager == &vnode_pager_workaround) {
-		return vnode_pager_terminate(memory_object);
-	} else if (memory_object->pager == &device_pager_workaround) {
-		return device_pager_terminate(memory_object);
-	} else
-#endif
-		return dp_memory_object_terminate(memory_object);
+	return (memory_object->mo_pager_ops->memory_object_terminate)(
+		memory_object);
 }
 
 /* Routine memory_object_data_request */
@@ -2267,23 +2048,11 @@ kern_return_t memory_object_data_request
 	vm_prot_t desired_access
 )
 {
-#ifdef	MACH_BSD
-	if (memory_object->pager == &vnode_pager_workaround) {
-		return vnode_pager_data_request(memory_object, 
-						offset, 
-						length,
-						desired_access);
-	} else if (memory_object->pager == &device_pager_workaround) {
-		return device_pager_data_request(memory_object, 
-						 offset, 
-						 length,
-						 desired_access);
-	} else
-#endif
-		return dp_memory_object_data_request(memory_object, 
-						     offset, 
-						     length,
-						     desired_access);
+	return (memory_object->mo_pager_ops->memory_object_data_request)(
+		memory_object,
+		offset, 
+		length,
+		desired_access);
 }
 
 /* Routine memory_object_data_return */
@@ -2299,37 +2068,15 @@ kern_return_t memory_object_data_return
 	int	upl_flags
 )
 {
-#ifdef MACH_BSD
-	if (memory_object->pager == &vnode_pager_workaround) {
-		return vnode_pager_data_return(memory_object,
-					       offset,
-					       size,
-					       resid_offset,
-					       io_error,
-					       dirty,
-					       kernel_copy,
-					       upl_flags);
-	} else if (memory_object->pager == &device_pager_workaround) {
-
-		return device_pager_data_return(memory_object,
-						offset,
-						size,
-						dirty,
-						kernel_copy,
-						upl_flags);
-	}
-	else 
-#endif
-	{
-		return dp_memory_object_data_return(memory_object,
-						    offset,
-						    size,
-						    NULL,
-						    NULL,
-						    dirty,
-						    kernel_copy,
-						    upl_flags);
-	}
+	return (memory_object->mo_pager_ops->memory_object_data_return)(
+		memory_object,
+		offset,
+		size,
+		resid_offset,
+		io_error,
+		dirty,
+		kernel_copy,
+		upl_flags);
 }
 
 /* Routine memory_object_data_initialize */
@@ -2340,20 +2087,10 @@ kern_return_t memory_object_data_initialize
 	vm_size_t size
 )
 {
-#ifdef MACH_BSD
-	if (memory_object->pager == &vnode_pager_workaround) {
-		return vnode_pager_data_initialize(memory_object,
-						   offset,
-						   size);
-	} else if (memory_object->pager == &device_pager_workaround) {
-		return device_pager_data_initialize(memory_object,
-						    offset,
-						    size);
-	} else
-#endif
-		return dp_memory_object_data_initialize(memory_object,
-							offset,
-							size);
+	return (memory_object->mo_pager_ops->memory_object_data_initialize)(
+		memory_object,
+		offset,
+		size);
 }
 
 /* Routine memory_object_data_unlock */
@@ -2365,23 +2102,11 @@ kern_return_t memory_object_data_unlock
 	vm_prot_t desired_access
 )
 {
-#ifdef MACH_BSD
-	if (memory_object->pager == &vnode_pager_workaround) {
-		return vnode_pager_data_unlock(memory_object,
-					       offset,
-					       size,
-					       desired_access);
-	} else if (memory_object->pager == &device_pager_workaround) {
-		return device_pager_data_unlock(memory_object,
-						offset,
-						size,
-						desired_access);
-	} else
-#endif
-		return dp_memory_object_data_unlock(memory_object,
-						    offset,
-						    size,
-						    desired_access);
+	return (memory_object->mo_pager_ops->memory_object_data_unlock)(
+		memory_object,
+		offset,
+		size,
+		desired_access);
 }
 
 /* Routine memory_object_synchronize */
@@ -2393,23 +2118,11 @@ kern_return_t memory_object_synchronize
 	vm_sync_t sync_flags
 )
 {
-#ifdef MACH_BSD
-	if (memory_object->pager == &vnode_pager_workaround) {
-		return vnode_pager_synchronize(memory_object,
-					       offset,
-					       size,
-					       sync_flags);
-	} else if (memory_object->pager == &device_pager_workaround) {
-		return device_pager_synchronize(memory_object,
-						offset,
-						size,
-						sync_flags);
-	} else
-#endif
-		return dp_memory_object_synchronize(memory_object,
-						    offset,
-						    size,
-						    sync_flags);
+	return (memory_object->mo_pager_ops->memory_object_synchronize)(
+		memory_object,
+		offset,
+		size,
+		sync_flags);
 }
 
 /* Routine memory_object_unmap */
@@ -2418,14 +2131,8 @@ kern_return_t memory_object_unmap
 	memory_object_t memory_object
 )
 {
-#ifdef MACH_BSD
-	if (memory_object->pager == &vnode_pager_workaround) {
-		return vnode_pager_unmap(memory_object);
-	} else if (memory_object->pager == &device_pager_workaround) {
-		return device_pager_unmap(memory_object);
-	} else
-#endif
-		return dp_memory_object_unmap(memory_object);
+	return (memory_object->mo_pager_ops->memory_object_unmap)(
+		memory_object);
 }
 
 /* Routine memory_object_create */
diff --git a/osfmk/vm/memory_object.h b/osfmk/vm/memory_object.h
index 933fdba28..181019879 100644
--- a/osfmk/vm/memory_object.h
+++ b/osfmk/vm/memory_object.h
@@ -118,9 +118,9 @@ extern ipc_port_t convert_upl_to_port( upl_t );
 __private_extern__ void upl_no_senders(ipc_port_t, mach_port_mscount_t);
 
 extern kern_return_t	memory_object_free_from_cache(
-				host_t		host,
-				int		*pager_id,
-				int		*count);
+	host_t				host,
+	memory_object_pager_ops_t	pager_ops,
+	int				*count);
 
 extern kern_return_t	memory_object_iopl_request(
 	ipc_port_t		port,
diff --git a/osfmk/vm/pmap.h b/osfmk/vm/pmap.h
index f8aa247ec..06e064fec 100644
--- a/osfmk/vm/pmap.h
+++ b/osfmk/vm/pmap.h
@@ -184,7 +184,9 @@ extern void		pmap_virtual_space(
 /*
  *	Routines to manage the physical map data structure.
  */
-extern pmap_t		pmap_create(vm_map_size_t size);	/* Create a pmap_t. */
+extern pmap_t		pmap_create(	/* Create a pmap_t. */
+				vm_map_size_t	size,
+				boolean_t	is_64bit);
 extern pmap_t		(pmap_kernel)(void);	/* Return the kernel's pmap */
 extern void		pmap_reference(pmap_t pmap);	/* Gain a reference. */
 extern void		pmap_destroy(pmap_t pmap); /* Release a reference. */
@@ -254,6 +256,9 @@ extern kern_return_t	(pmap_attribute_cache_sync)(  /* Flush appropriate
 				vm_machine_attribute_t attribute, 
 				vm_machine_attribute_val_t* value);
 
+extern unsigned int	(pmap_cache_attributes)(
+				ppnum_t		pn);
+
 /*
  * debug/assertions. pmap_verify_free returns true iff
  * the given physical page is mapped into no pmap.
diff --git a/osfmk/vm/task_working_set.c b/osfmk/vm/task_working_set.c
index bd3ca3a68..c799dc3c6 100644
--- a/osfmk/vm/task_working_set.c
+++ b/osfmk/vm/task_working_set.c
@@ -1077,9 +1077,13 @@ tws_build_cluster(
 		return;
 
 	if (!object->internal) {
+		/* XXX FBDP !internal doesn't mean vnode pager */
 		kret = vnode_pager_get_object_size(
        		 	object->pager,
 			&object_size);
+		if (kret != KERN_SUCCESS) {
+			object_size = object->size;
+		}
 	} else {
 		object_size = object->size;
 	}
diff --git a/osfmk/vm/vm_apple_protect.c b/osfmk/vm/vm_apple_protect.c
new file mode 100644
index 000000000..41f70d9c9
--- /dev/null
+++ b/osfmk/vm/vm_apple_protect.c
@@ -0,0 +1,1077 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <sys/errno.h>
+
+#include <mach/mach_types.h>
+#include <mach/mach_traps.h>
+#include <mach/host_priv.h>
+#include <mach/kern_return.h>
+#include <mach/memory_object_control.h>
+#include <mach/memory_object_types.h>
+#include <mach/port.h>
+#include <mach/policy.h>
+#include <mach/upl.h>
+#include <mach/thread_act.h>
+#include <mach/mach_vm.h>
+
+#include <kern/host.h>
+#include <kern/kalloc.h>
+#include <kern/page_decrypt.h>
+#include <kern/queue.h>
+#include <kern/thread.h>
+
+#include <ipc/ipc_port.h>
+#include <ipc/ipc_space.h>
+
+#include <default_pager/default_pager_types.h>
+#include <default_pager/default_pager_object_server.h>
+
+#include <vm/vm_map.h>
+#include <vm/vm_pageout.h>
+#include <vm/memory_object.h>
+#include <vm/vm_pageout.h>
+#include <vm/vm_protos.h>
+
+
+/* 
+ * APPLE PROTECT MEMORY PAGER 
+ *
+ * This external memory manager (EMM) handles memory from the encrypted
+ * sections of some executables protected by the DSMOS kernel extension.
+ * 
+ * It mostly handles page-in requests (from memory_object_data_request()) by
+ * getting the encrypted data from its backing VM object, itself backed by
+ * the encrypted file, decrypting it and providing it to VM.
+ *
+ * The decrypted pages will never be dirtied, so the memory manager doesn't
+ * need to handle page-out requests (from memory_object_data_return()).  The
+ * pages need to be mapped copy-on-write, so that the originals stay clean.
+ *
+ * We don't expect to have to handle a large number of apple-protected
+ * binaries, so the data structures are very simple (simple linked list)
+ * for now.
+ */
+
+/* forward declarations */
+void apple_protect_pager_reference(memory_object_t mem_obj);
+void apple_protect_pager_deallocate(memory_object_t mem_obj);
+kern_return_t apple_protect_pager_init(memory_object_t mem_obj,
+				       memory_object_control_t control,
+				       vm_size_t pg_size);
+kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj);
+kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj,
+					       memory_object_offset_t offset,
+					       vm_size_t length,
+					       vm_prot_t protection_required);
+kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj,
+					      memory_object_offset_t offset,
+					      vm_size_t	data_cnt,
+					      memory_object_offset_t *resid_offset,
+					      int *io_error,
+					      boolean_t dirty,
+					      boolean_t kernel_copy,
+					      int upl_flags);
+kern_return_t apple_protect_pager_data_initialize(memory_object_t mem_obj,
+						  memory_object_offset_t offset,
+						  vm_size_t data_cnt);
+kern_return_t apple_protect_pager_data_unlock(memory_object_t mem_obj,
+					      memory_object_offset_t offset,
+					      vm_size_t size,
+					      vm_prot_t desired_access);
+kern_return_t apple_protect_pager_synchronize(memory_object_t mem_obj,
+					      memory_object_offset_t offset,
+					      vm_size_t length,
+					      vm_sync_t sync_flags);
+kern_return_t apple_protect_pager_unmap(memory_object_t mem_obj);
+
+/*
+ * Vector of VM operations for this EMM.
+ * These routines are invoked by VM via the memory_object_*() interfaces.
+ */
+const struct memory_object_pager_ops apple_protect_pager_ops = {
+	apple_protect_pager_reference,
+	apple_protect_pager_deallocate,
+	apple_protect_pager_init,
+	apple_protect_pager_terminate,
+	apple_protect_pager_data_request,
+	apple_protect_pager_data_return,
+	apple_protect_pager_data_initialize,
+	apple_protect_pager_data_unlock,
+	apple_protect_pager_synchronize,
+	apple_protect_pager_unmap,
+	"apple protect pager"
+};
+
+/*
+ * The "apple_protect_pager" describes a memory object backed by
+ * the "apple protect" EMM.
+ */
+typedef struct apple_protect_pager {
+	memory_object_pager_ops_t pager_ops; /* == &apple_protect_pager_ops */
+	unsigned int		pager_ikot;	/* JMM: fake ip_kotype() */
+	queue_chain_t		pager_queue;	/* next & prev pagers */
+	unsigned int		ref_count;	/* reference count */
+	boolean_t		is_ready;	/* is this pager ready ? */
+	boolean_t		is_mapped;	/* is this mem_obj mapped ? */
+	memory_object_control_t pager_control;	/* mem object control handle */
+	vm_object_t		backing_object; /* VM obj w/ encrypted data */
+} *apple_protect_pager_t;
+#define	APPLE_PROTECT_PAGER_NULL	((apple_protect_pager_t) NULL)
+
+/*
+ * List of memory objects managed by this EMM.
+ * The list is protected by the "apple_protect_pager_lock" lock.
+ */
+int apple_protect_pager_count = 0;		/* number of pagers */
+int apple_protect_pager_count_mapped = 0;	/* number of unmapped pagers */
+queue_head_t apple_protect_pager_queue;
+decl_mutex_data(,apple_protect_pager_lock)
+
+/*
+ * Maximum number of unmapped pagers we're willing to keep around.
+ */
+int apple_protect_pager_cache_limit = 10;
+
+/*
+ * Statistics & counters.
+ */
+int apple_protect_pager_count_max = 0;
+int apple_protect_pager_count_unmapped_max = 0;
+int apple_protect_pager_num_trim_max = 0;
+int apple_protect_pager_num_trim_total = 0;
+
+/* internal prototypes */
+apple_protect_pager_t apple_protect_pager_create(vm_object_t backing_object);
+apple_protect_pager_t apple_protect_pager_lookup(memory_object_t mem_obj);
+void apple_protect_pager_dequeue(apple_protect_pager_t pager);
+void apple_protect_pager_deallocate_internal(apple_protect_pager_t pager,
+					     boolean_t locked);
+void apple_protect_pager_terminate_internal(apple_protect_pager_t pager);
+void apple_protect_pager_trim(void);
+
+
+#if DEBUG
+int apple_protect_pagerdebug = 0;
+#define PAGER_ALL		0xffffffff
+#define	PAGER_INIT		0x00000001
+#define	PAGER_PAGEIN		0x00000002
+
+#define PAGER_DEBUG(LEVEL, A)						\
+	MACRO_BEGIN							\
+	if ((apple_protect_pagerdebug & LEVEL)==LEVEL) {		\
+		printf A;						\
+	}								\
+	MACRO_END
+#else
+#define PAGER_DEBUG(LEVEL, A)
+#endif
+
+
+void
+apple_protect_pager_bootstrap(void)
+{
+	mutex_init(&apple_protect_pager_lock, 0);
+	queue_init(&apple_protect_pager_queue);
+}
+
+/*
+ * apple_protect_pager_init()
+ *
+ * Initialize the memory object and makes it ready to be used and mapped.
+ */
+kern_return_t
+apple_protect_pager_init(
+	memory_object_t		mem_obj, 
+	memory_object_control_t	control, 
+#if !DEBUG
+	__unused
+#endif
+	vm_size_t pg_size)
+{
+	apple_protect_pager_t	pager;
+	kern_return_t   	kr;
+	memory_object_attr_info_data_t  attributes;
+
+	PAGER_DEBUG(PAGER_ALL,
+		    ("apple_protect_pager_init: %p, %p, %x\n",
+		     mem_obj, control, pg_size));
+
+	if (control == MEMORY_OBJECT_CONTROL_NULL)
+		return KERN_INVALID_ARGUMENT;
+
+	pager = apple_protect_pager_lookup(mem_obj);
+
+	memory_object_control_reference(control);
+
+	pager->pager_control = control;
+
+	attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
+	/* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
+	attributes.cluster_size = (1 << (PAGE_SHIFT));
+	attributes.may_cache_object = FALSE;
+	attributes.temporary = TRUE;
+
+	kr = memory_object_change_attributes(
+					control,
+					MEMORY_OBJECT_ATTRIBUTE_INFO,
+					(memory_object_info_t) &attributes,
+					MEMORY_OBJECT_ATTR_INFO_COUNT);
+	if (kr != KERN_SUCCESS)
+		panic("apple_protect_pager_init: "
+		      "memory_object_change_attributes() failed");
+
+	return KERN_SUCCESS;
+}
+
+/*
+ * apple_protect_data_return()
+ *
+ * Handles page-out requests from VM.  This should never happen since
+ * the pages provided by this EMM are not supposed to be dirty or dirtied
+ * and VM should simply discard the contents and reclaim the pages if it
+ * needs to.
+ */
+kern_return_t
+apple_protect_pager_data_return(
+        __unused memory_object_t	mem_obj,
+        __unused memory_object_offset_t	offset,
+        __unused vm_size_t		data_cnt,
+        __unused memory_object_offset_t	*resid_offset,
+	__unused int			*io_error,
+	__unused boolean_t		dirty,
+	__unused boolean_t		kernel_copy,
+	__unused int			upl_flags)  
+{
+	panic("apple_protect_pager_data_return: should never get called");
+	return KERN_FAILURE;
+}
+
+kern_return_t
+apple_protect_pager_data_initialize(
+	__unused memory_object_t	mem_obj,
+	__unused memory_object_offset_t	offset,
+	__unused vm_size_t		data_cnt)
+{
+	panic("apple_protect_pager_data_initialize: should never get called");
+	return KERN_FAILURE;
+}
+
+kern_return_t
+apple_protect_pager_data_unlock(
+	__unused memory_object_t	mem_obj,
+	__unused memory_object_offset_t	offset,
+	__unused vm_size_t		size,
+	__unused vm_prot_t		desired_access)
+{
+	return KERN_FAILURE;
+}
+
+/*
+ * apple_protect_pager_data_request()
+ *
+ * Handles page-in requests from VM.
+ */
+kern_return_t	
+apple_protect_pager_data_request(
+	memory_object_t		mem_obj,
+	memory_object_offset_t	offset,
+	vm_size_t		length,
+#if !DEBUG
+	__unused
+#endif
+	vm_prot_t		protection_required)
+{
+	apple_protect_pager_t	pager;
+	memory_object_control_t	mo_control;
+	upl_t			upl = NULL;
+	int			upl_flags;
+	upl_size_t		upl_size;
+	upl_page_info_t		*upl_pl;
+	vm_object_t		src_object, dst_object;
+	kern_return_t		kr, retval;
+	vm_map_offset_t		src_mapping = 0, dst_mapping = 0;
+	vm_offset_t		src_vaddr, dst_vaddr;
+	vm_offset_t		cur_offset;
+	boolean_t		src_map_page_by_page;
+	vm_map_entry_t		map_entry;
+
+	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %x, %llx, %llxx, %x\n", mem_obj, offset, length, protection_required));
+
+	pager = apple_protect_pager_lookup(mem_obj);
+	assert(pager->is_ready);
+	assert(pager->ref_count > 1); /* pager is alive and mapped */
+
+	PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %x, %llx, %llx, %x, pager %x\n", mem_obj, offset, length, protection_required, pager));
+
+	/*
+	 * Map the encrypted data in the kernel address space from the 
+	 * backing VM object (itself backed by the encrypted file via
+	 * the vnode pager).
+	 */
+	src_object = pager->backing_object;
+	assert(src_object != VM_OBJECT_NULL);
+	vm_object_reference(src_object); /* ref. for the mapping */
+	src_mapping = 0;
+	kr = vm_map_enter(kernel_map,
+			  &src_mapping,
+			  length,
+			  0,
+			  VM_FLAGS_ANYWHERE,
+			  src_object,
+			  offset,
+			  FALSE,
+			  VM_PROT_READ,
+			  VM_PROT_READ,
+			  VM_INHERIT_NONE);
+	switch (kr) {
+	case KERN_SUCCESS:
+		/* wire the memory to make sure it is available */
+		kr = vm_map_wire(kernel_map,
+				 src_mapping,
+				 src_mapping + length,
+				 VM_PROT_READ,
+				 FALSE);
+		if (kr != KERN_SUCCESS) {
+			/*
+			 * Wiring failed, so unmap source and fall back
+			 * to page by page mapping of the source.
+			 */
+			kr = vm_map_remove(kernel_map,
+					   src_mapping,
+					   src_mapping + length,
+					   VM_MAP_NO_FLAGS);
+			assert(kr == KERN_SUCCESS);
+			src_mapping = 0;
+			src_vaddr = 0;
+			src_map_page_by_page = TRUE;
+			break;
+		}
+		/* source region is now fully mapped and wired */
+		src_map_page_by_page = FALSE;
+		src_vaddr = CAST_DOWN(vm_offset_t, src_mapping);
+		break;
+	case KERN_NO_SPACE:
+		/* we couldn't map the entire source, so map it page by page */
+		src_map_page_by_page = TRUE;
+		/* release the reference for the failed mapping */
+		vm_object_deallocate(src_object);
+		break;
+	default:
+		vm_object_deallocate(src_object);
+		retval = kr;
+		goto done;
+	}
+
+
+	/*
+	 * Gather in a UPL all the VM pages requested by VM.
+	 */
+	mo_control = pager->pager_control;
+
+	upl_size = length;
+	upl_flags =
+		UPL_RET_ONLY_ABSENT |
+		UPL_SET_LITE |
+		UPL_NO_SYNC |
+		UPL_CLEAN_IN_PLACE |	/* triggers UPL_CLEAR_DIRTY */
+		UPL_SET_INTERNAL;
+	kr = memory_object_upl_request(mo_control,
+				       offset, upl_size,
+				       &upl, NULL, NULL, upl_flags);
+	if (kr != KERN_SUCCESS) {
+		retval = kr;
+		goto done;
+	}
+
+	/*
+	 * Reserve a virtual page in the kernel address space to map each
+	 * destination physical page when it's its turn to be filled.
+	 */
+	dst_object = mo_control->moc_object;
+	assert(dst_object != VM_OBJECT_NULL);
+	dst_mapping = 0;
+	vm_object_reference(kernel_object);	/* ref. for mapping */
+	kr = vm_map_find_space(kernel_map,
+			       &dst_mapping,
+			       PAGE_SIZE_64,
+			       0,
+			       0,
+			       &map_entry);
+	if (kr != KERN_SUCCESS) {
+		vm_object_deallocate(kernel_object);
+		retval = kr;
+		goto done;
+	}
+	map_entry->object.vm_object = kernel_object;
+	map_entry->offset = dst_mapping - VM_MIN_KERNEL_ADDRESS;
+	vm_map_unlock(kernel_map);
+	dst_vaddr = CAST_DOWN(vm_offset_t, dst_mapping);
+
+	/*
+	 * Fill in the contents of the pages requested by VM.
+	 */
+	upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
+	for (cur_offset = 0; cur_offset < length; cur_offset += PAGE_SIZE) {
+		ppnum_t dst_pnum;
+
+		if (!upl_page_present(upl_pl, cur_offset / PAGE_SIZE)) {
+			/* this page is not in the UPL: skip it */
+			continue;
+		}
+
+		/*
+		 * Map the source (encrypted) page in the kernel's
+		 * virtual address space.
+		 */
+		if (src_map_page_by_page) {
+			vm_object_reference(src_object); /* ref. for mapping */
+			kr = vm_map_enter(kernel_map,
+					  &src_mapping,
+					  PAGE_SIZE_64,
+					  0,
+					  VM_FLAGS_ANYWHERE,
+					  src_object,
+					  offset + cur_offset,
+					  FALSE,
+					  VM_PROT_READ,
+					  VM_PROT_READ,
+					  VM_INHERIT_NONE);
+			if (kr != KERN_SUCCESS) {
+				vm_object_deallocate(src_object);
+				retval = kr;
+				goto done;
+			}
+			kr = vm_map_wire(kernel_map,
+					 src_mapping,
+					 src_mapping + PAGE_SIZE_64,
+					 VM_PROT_READ,
+					 FALSE);
+			if (kr != KERN_SUCCESS) {
+				retval = kr;
+				kr = vm_map_remove(kernel_map,
+						   src_mapping,
+						   src_mapping + PAGE_SIZE_64,
+						   VM_MAP_NO_FLAGS);
+				assert(kr == KERN_SUCCESS);
+				src_mapping = 0;
+				src_vaddr = 0;
+				printf("apple_protect_pager_data_request: "
+				       "failed to resolve page fault for src "
+				       "object %p offset 0x%llx "
+				       "preempt %d error 0x%x\n",
+				       src_object, offset + cur_offset,
+				       get_preemption_level(), retval);
+				goto done;
+			}
+			src_vaddr = CAST_DOWN(vm_offset_t, src_mapping);
+		} else {
+			src_vaddr = src_mapping + cur_offset;
+		}
+
+		/*
+		 * Establish an explicit pmap mapping of the destination
+		 * physical page.
+		 * We can't do a regular VM mapping because the VM page
+		 * is "busy".
+		 */
+		dst_pnum = (addr64_t)
+			upl_phys_page(upl_pl, cur_offset / PAGE_SIZE);
+		assert(dst_pnum != 0);
+		pmap_enter(kernel_pmap, dst_mapping, dst_pnum,
+			   VM_PROT_READ | VM_PROT_WRITE,
+			   dst_object->wimg_bits & VM_WIMG_MASK,
+			   FALSE);
+
+		/*
+		 * Decrypt the encrypted contents of the source page
+		 * into the destination page.
+		 */
+		dsmos_page_transform((const void *) src_vaddr,
+				     (void *) dst_vaddr);
+
+		/*
+		 * Remove the pmap mapping of the destination page
+		 * in the kernel.
+		 */
+		pmap_remove(kernel_pmap,
+			    (addr64_t) dst_mapping,
+			    (addr64_t) (dst_mapping + PAGE_SIZE_64));
+
+		if (src_map_page_by_page) {
+			/*
+			 * Remove the wired kernel mapping of the source page.
+			 * This releases the extra reference we took on
+			 * src_object.
+			 */
+			kr = vm_map_remove(kernel_map,
+					   src_mapping,
+					   src_mapping + PAGE_SIZE_64,
+					   VM_MAP_REMOVE_KUNWIRE);
+			assert(kr == KERN_SUCCESS);
+			src_mapping = 0;
+			src_vaddr = 0;
+		}
+	}
+
+	retval = KERN_SUCCESS;
+done:
+	if (src_mapping != 0) {
+		/* remove the wired mapping of the source pages */
+		kr = vm_map_remove(kernel_map,
+				   src_mapping,
+				   src_mapping + length,
+				   VM_MAP_REMOVE_KUNWIRE);
+		assert(kr == KERN_SUCCESS);
+		src_mapping = 0;
+		src_vaddr = 0;
+	}
+	if (upl != NULL) {
+		/* clean up the UPL */
+
+		/*
+		 * The pages are currently dirty because we've just been
+		 * writing on them, but as far as we're concerned, they're
+		 * clean since they contain their "original" contents as
+		 * provided by us, the pager.
+		 * Tell the UPL to mark them "clean".
+		 */
+		upl_clear_dirty(upl, TRUE);
+
+		/* abort or commit the UPL */
+		if (retval != KERN_SUCCESS) {
+			upl_abort(upl, 0);
+		} else {
+			upl_commit(upl, NULL, 0);
+		}
+
+		/* and deallocate the UPL */
+		upl_deallocate(upl);
+		upl = NULL;
+	}
+	if (dst_mapping != 0) {
+		/* clean up the mapping of the destination pages */
+		kr = vm_map_remove(kernel_map,
+				   dst_mapping,
+				   dst_mapping + PAGE_SIZE_64,
+				   VM_MAP_NO_FLAGS);
+		assert(kr == KERN_SUCCESS);
+		dst_mapping = 0;
+		dst_vaddr = 0;
+	}
+
+	return retval;
+}
+
+/*
+ * apple_protect_pager_reference()
+ *
+ * Get a reference on this memory object.
+ * For external usage only.  Assumes that the initial reference count is not 0,
+ * i.e one should not "revive" a dead pager this way.
+ */
+void
+apple_protect_pager_reference(
+	memory_object_t		mem_obj)
+{	
+	apple_protect_pager_t	pager;
+
+	pager = apple_protect_pager_lookup(mem_obj);
+
+	mutex_lock(&apple_protect_pager_lock);
+	assert(pager->ref_count > 0);
+	pager->ref_count++;
+	mutex_unlock(&apple_protect_pager_lock);
+}
+
+
+/*
+ * apple_protect_pager_dequeue:
+ *
+ * Removes a pager from the list of pagers.
+ *
+ * The caller must hold "apple_protect_pager_lock".
+ */
+void
+apple_protect_pager_dequeue(
+	apple_protect_pager_t pager)
+{
+	assert(!pager->is_mapped);
+
+	queue_remove(&apple_protect_pager_queue,
+		     pager,
+		     apple_protect_pager_t,
+		     pager_queue);
+	pager->pager_queue.next = NULL;
+	pager->pager_queue.prev = NULL;
+	
+	apple_protect_pager_count--;
+}
+
+/*
+ * apple_protect_pager_terminate_internal:
+ *
+ * Trigger the asynchronous termination of the memory object associated
+ * with this pager.
+ * When the memory object is terminated, there will be one more call
+ * to memory_object_deallocate() (i.e. apple_protect_pager_deallocate())
+ * to finish the clean up.
+ *
+ * "apple_protect_pager_lock" should not be held by the caller.
+ * We don't need the lock because the pager has already been removed from
+ * the pagers' list and is now ours exclusively.
+ */
+void
+apple_protect_pager_terminate_internal(
+	apple_protect_pager_t pager)
+{
+	assert(pager->is_ready);
+	assert(!pager->is_mapped);
+
+	if (pager->backing_object != VM_OBJECT_NULL) {
+		vm_object_deallocate(pager->backing_object);
+		pager->backing_object = VM_OBJECT_NULL;
+	}
+
+	/* trigger the destruction of the memory object */
+	memory_object_destroy(pager->pager_control, 0);
+}
+
+/*
+ * apple_protect_pager_deallocate_internal()
+ *
+ * Release a reference on this pager and free it when the last
+ * reference goes away.
+ * Can be called with apple_protect_pager_lock held or not but always returns
+ * with it unlocked.
+ */
+void
+apple_protect_pager_deallocate_internal(
+	apple_protect_pager_t	pager,
+	boolean_t		locked)
+{
+	boolean_t	needs_trimming;
+	int		count_unmapped;
+
+	if (! locked) {
+		mutex_lock(&apple_protect_pager_lock);
+	}
+
+	count_unmapped = (apple_protect_pager_count - 
+			  apple_protect_pager_count_mapped);
+	if (count_unmapped > apple_protect_pager_cache_limit) {
+		/* we have too many unmapped pagers:  trim some */
+		needs_trimming = TRUE;
+	} else {
+		needs_trimming = FALSE;
+	}
+
+	/* drop a reference on this pager */
+	pager->ref_count--;
+
+	if (pager->ref_count == 1) {
+		/*
+		 * Only the "named" reference is left, which means that
+		 * no one is realy holding on to this pager anymore.
+		 * Terminate it.
+		 */
+		apple_protect_pager_dequeue(pager);
+		/* the pager is all ours: no need for the lock now */
+		mutex_unlock(&apple_protect_pager_lock);
+		apple_protect_pager_terminate_internal(pager);
+	} else if (pager->ref_count == 0) {
+		/*
+		 * Dropped the existence reference;  the memory object has
+		 * been terminated.  Do some final cleanup and release the
+		 * pager structure.
+		 */
+		mutex_unlock(&apple_protect_pager_lock);
+		if (pager->pager_control != MEMORY_OBJECT_CONTROL_NULL) {
+			memory_object_control_deallocate(pager->pager_control);
+			pager->pager_control = MEMORY_OBJECT_CONTROL_NULL;
+		}
+		kfree(pager, sizeof (*pager));
+		pager = APPLE_PROTECT_PAGER_NULL;
+	} else {
+		/* there are still plenty of references:  keep going... */
+		mutex_unlock(&apple_protect_pager_lock);
+	}
+
+	if (needs_trimming) {
+		apple_protect_pager_trim();
+	}
+	/* caution: lock is not held on return... */
+}
+
+/*
+ * apple_protect_pager_deallocate()
+ *
+ * Release a reference on this pager and free it when the last
+ * reference goes away.
+ */
+void
+apple_protect_pager_deallocate(
+	memory_object_t		mem_obj)
+{
+	apple_protect_pager_t	pager;
+
+	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %x\n", mem_obj));
+	pager = apple_protect_pager_lookup(mem_obj);
+	apple_protect_pager_deallocate_internal(pager, FALSE);
+}
+
+/*
+ *
+ */
+kern_return_t
+apple_protect_pager_terminate(
+#if !DEBUG
+	__unused
+#endif
+	memory_object_t	mem_obj)
+{
+	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %x\n", mem_obj));
+
+	return KERN_SUCCESS;
+}
+
+/*
+ *
+ */
+kern_return_t
+apple_protect_pager_synchronize(
+	memory_object_t		mem_obj,
+	memory_object_offset_t	offset,
+	vm_size_t		length,
+	__unused vm_sync_t		sync_flags)
+{
+	apple_protect_pager_t	pager;
+
+	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_synchronize: %x\n", mem_obj));
+
+	pager = apple_protect_pager_lookup(mem_obj);
+
+	memory_object_synchronize_completed(pager->pager_control,
+					    offset, length);
+
+	return KERN_SUCCESS;
+}
+
+/*
+ * apple_protect_pager_map()
+ *
+ * This allows VM to let us, the EMM, know that this memory object
+ * is currently mapped one or more times.  This is called by VM only the first
+ * time the memory object gets mapped and we take one extra reference on the
+ * memory object to account for all its mappings.
+ */
+void
+apple_protect_pager_map(
+	memory_object_t		mem_obj)
+{
+	apple_protect_pager_t	pager;
+
+	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %x\n", mem_obj));
+
+	pager = apple_protect_pager_lookup(mem_obj);
+
+	mutex_lock(&apple_protect_pager_lock);
+	assert(pager->is_ready);
+	assert(pager->ref_count > 0); /* pager is alive */
+	if (pager->is_mapped == FALSE) {
+		/*
+		 * First mapping of this pager:  take an extra reference
+		 * that will remain until all the mappings of this pager
+		 * are removed.
+		 */
+		pager->is_mapped = TRUE;
+		pager->ref_count++;
+		apple_protect_pager_count_mapped++;
+	}
+	mutex_unlock(&apple_protect_pager_lock);
+}
+
+/*
+ * apple_protect_pager_unmap()
+ *
+ * This is called by VM when this memory object is no longer mapped anywhere.
+ */
+kern_return_t
+apple_protect_pager_unmap(
+	memory_object_t		mem_obj)
+{
+	apple_protect_pager_t	pager;
+	int			count_unmapped;
+
+	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_unmap: %x\n", mem_obj));
+
+	pager = apple_protect_pager_lookup(mem_obj);
+
+	mutex_lock(&apple_protect_pager_lock);
+	if (pager->is_mapped) {
+		/*
+		 * All the mappings are gone, so let go of the one extra
+		 * reference that represents all the mappings of this pager.
+		 */
+		apple_protect_pager_count_mapped--;
+		count_unmapped = (apple_protect_pager_count -
+				  apple_protect_pager_count_mapped);
+		if (count_unmapped > apple_protect_pager_count_unmapped_max) {
+			apple_protect_pager_count_unmapped_max = count_unmapped;
+		}
+		pager->is_mapped = FALSE;
+		apple_protect_pager_deallocate_internal(pager, TRUE);
+		/* caution: deallocate_internal() released the lock ! */
+	} else {
+		mutex_unlock(&apple_protect_pager_lock);
+	}
+	
+	return KERN_SUCCESS;
+}
+
+
+/*
+ *
+ */
+apple_protect_pager_t
+apple_protect_pager_lookup(
+	memory_object_t	 mem_obj)
+{
+	apple_protect_pager_t	pager;
+
+	pager = (apple_protect_pager_t) mem_obj;
+	assert(pager->pager_ops == &apple_protect_pager_ops);
+	assert(pager->ref_count > 0);
+	return pager;
+}
+
+apple_protect_pager_t
+apple_protect_pager_create(
+	vm_object_t	backing_object)
+{
+	apple_protect_pager_t	pager, pager2;
+	memory_object_control_t	control;
+	kern_return_t		kr;
+
+	pager = (apple_protect_pager_t) kalloc(sizeof (*pager));
+	if (pager == APPLE_PROTECT_PAGER_NULL) {
+		return APPLE_PROTECT_PAGER_NULL;
+	}
+
+	/*
+	 * The vm_map call takes both named entry ports and raw memory
+	 * objects in the same parameter.  We need to make sure that
+	 * vm_map does not see this object as a named entry port.  So,
+	 * we reserve the second word in the object for a fake ip_kotype
+	 * setting - that will tell vm_map to use it as a memory object.
+	 */
+	pager->pager_ops = &apple_protect_pager_ops;
+	pager->pager_ikot = IKOT_MEMORY_OBJECT;
+	pager->is_ready = FALSE;/* not ready until it has a "name" */
+	pager->ref_count = 2;	/* existence + setup reference */
+	pager->is_mapped = FALSE;
+	pager->pager_control = MEMORY_OBJECT_CONTROL_NULL;
+	pager->backing_object = backing_object;
+	vm_object_reference(backing_object);
+
+	mutex_lock(&apple_protect_pager_lock);
+	/* see if anyone raced us to create a pager for the same object */
+	queue_iterate(&apple_protect_pager_queue,
+		      pager2,
+		      apple_protect_pager_t,
+		      pager_queue) {
+		if (pager2->backing_object == backing_object) {
+			break;
+		}
+	}
+	if (! queue_end(&apple_protect_pager_queue,
+			(queue_entry_t) pager2)) {
+		/* while we hold the lock, transfer our setup ref to winner */
+		pager2->ref_count++;
+		/* we lost the race, down with the loser... */
+		mutex_unlock(&apple_protect_pager_lock);
+		vm_object_deallocate(pager->backing_object);
+		pager->backing_object = VM_OBJECT_NULL;
+		kfree(pager, sizeof (*pager));
+		/* ... and go with the winner */
+		pager = pager2;
+		/* let the winner make sure the pager gets ready */
+		return pager;
+	}
+
+	/* enter new pager at the head of our list of pagers */
+	queue_enter_first(&apple_protect_pager_queue,
+			  pager,
+			  apple_protect_pager_t,
+			  pager_queue);
+	apple_protect_pager_count++;
+	if (apple_protect_pager_count > apple_protect_pager_count_max) {
+		apple_protect_pager_count_max = apple_protect_pager_count;
+	}
+	mutex_unlock(&apple_protect_pager_lock);
+
+	kr = memory_object_create_named((memory_object_t) pager,
+					0,
+					&control);
+	assert(kr == KERN_SUCCESS);
+
+	mutex_lock(&apple_protect_pager_lock);
+	/* the new pager is now ready to be used */
+	pager->is_ready = TRUE;
+	mutex_unlock(&apple_protect_pager_lock);
+
+	/* wakeup anyone waiting for this pager to be ready */
+	thread_wakeup(&pager->is_ready);
+
+	return pager;
+}
+
+/*
+ * apple_protect_pager_setup()
+ *
+ * Provide the caller with a memory object backed by the provided
+ * "backing_object" VM object.  If such a memory object already exists,
+ * re-use it, otherwise create a new memory object.
+ */
+memory_object_t
+apple_protect_pager_setup(
+	vm_object_t	backing_object)
+{
+	apple_protect_pager_t	pager;
+
+	mutex_lock(&apple_protect_pager_lock);
+
+	queue_iterate(&apple_protect_pager_queue,
+		      pager,
+		      apple_protect_pager_t,
+		      pager_queue) {
+		if (pager->backing_object == backing_object) {
+			break;
+		}
+	}
+	if (queue_end(&apple_protect_pager_queue,
+		      (queue_entry_t) pager)) {
+		/* no existing pager for this backing object */
+		pager = APPLE_PROTECT_PAGER_NULL;
+	} else {
+		/* make sure pager doesn't disappear */
+		pager->ref_count++;
+	}
+
+	mutex_unlock(&apple_protect_pager_lock);
+
+	if (pager == APPLE_PROTECT_PAGER_NULL) {
+		pager = apple_protect_pager_create(backing_object);
+		if (pager == APPLE_PROTECT_PAGER_NULL) {
+			return MEMORY_OBJECT_NULL;
+		}
+	}
+
+	mutex_lock(&apple_protect_pager_lock);
+	while (!pager->is_ready) {
+		thread_sleep_mutex(&pager->is_ready,
+				   &apple_protect_pager_lock,
+				   THREAD_UNINT);
+	}
+	mutex_unlock(&apple_protect_pager_lock);
+
+	return (memory_object_t) pager;
+}	
+
+void
+apple_protect_pager_trim(void)
+{
+	apple_protect_pager_t	pager, prev_pager;
+	queue_head_t		trim_queue;
+	int			num_trim;
+	int			count_unmapped;
+
+	mutex_lock(&apple_protect_pager_lock);
+
+	/*
+	 * We have too many pagers, try and trim some unused ones,
+	 * starting with the oldest pager at the end of the queue.
+	 */
+	queue_init(&trim_queue);
+	num_trim = 0;
+
+	for (pager = (apple_protect_pager_t)
+		     queue_last(&apple_protect_pager_queue);
+	     !queue_end(&apple_protect_pager_queue,
+			(queue_entry_t) pager);
+	     pager = prev_pager) {
+		/* get prev elt before we dequeue */
+		prev_pager = (apple_protect_pager_t)
+			queue_prev(&pager->pager_queue);
+
+		if (pager->ref_count == 2 &&
+		    pager->is_ready &&
+		    !pager->is_mapped) {
+			/* this pager can be trimmed */
+			num_trim++;
+			/* remove this pager from the main list ... */
+			apple_protect_pager_dequeue(pager);
+			/* ... and add it to our trim queue */
+			queue_enter_first(&trim_queue,
+					  pager,
+					  apple_protect_pager_t,
+					  pager_queue);
+
+			count_unmapped = (apple_protect_pager_count -
+					  apple_protect_pager_count_mapped);
+			if (count_unmapped <= apple_protect_pager_cache_limit) {
+				/* we have enough pagers to trim */
+				break;
+			}
+		}
+	}
+	if (num_trim > apple_protect_pager_num_trim_max) {
+		apple_protect_pager_num_trim_max = num_trim;
+	}
+	apple_protect_pager_num_trim_total += num_trim;
+
+	mutex_unlock(&apple_protect_pager_lock);
+
+	/* terminate the trimmed pagers */
+	while (!queue_empty(&trim_queue)) {
+		queue_remove_first(&trim_queue,
+				   pager,
+				   apple_protect_pager_t,
+				   pager_queue);
+		pager->pager_queue.next = NULL;
+		pager->pager_queue.prev = NULL;
+		assert(pager->ref_count == 2);
+		/*
+		 * We can't call deallocate_internal() because the pager
+		 * has already been dequeued, but we still need to remove
+		 * a reference.
+		 */
+		pager->ref_count--;
+		apple_protect_pager_terminate_internal(pager);
+	}
+}
diff --git a/osfmk/vm/vm_fault.c b/osfmk/vm/vm_fault.c
index 963f6345e..b873e3d17 100644
--- a/osfmk/vm/vm_fault.c
+++ b/osfmk/vm/vm_fault.c
@@ -1681,7 +1681,7 @@ no_clustering:
 			 */     
 
 			vm_object_paging_end(object); 
-			vm_object_collapse(object, offset);
+			vm_object_collapse(object, offset, TRUE);
 			vm_object_paging_begin(object);
 
 		}
@@ -2157,10 +2157,8 @@ vm_fault(
 	unsigned int		cache_attr;
 	int			write_startup_file = 0;
 	boolean_t		need_activation;
-	vm_prot_t		full_fault_type;
+	vm_prot_t		original_fault_type;
 
-	if (get_preemption_level() != 0)
-		return (KERN_FAILURE);
 
 	KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 0)) | DBG_FUNC_START,
 			      vaddr,
@@ -2169,13 +2167,15 @@ vm_fault(
 			      0,
 			      0);
 
-	/* at present we do not fully check for execute permission */
-	/* we generally treat it is read except in certain device  */
-	/* memory settings */
-	full_fault_type = fault_type;
-	if(fault_type & VM_PROT_EXECUTE) {
-		fault_type &= ~VM_PROT_EXECUTE;
-		fault_type |= VM_PROT_READ;
+	if (get_preemption_level() != 0) {
+	        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 0)) | DBG_FUNC_END,
+				      vaddr,
+				      0,
+				      KERN_FAILURE,
+				      0,
+				      0);
+
+		return (KERN_FAILURE);
 	}
 
 	interruptible_state = thread_interrupt_level(interruptible);
@@ -2190,12 +2190,15 @@ vm_fault(
 	VM_STAT(faults++);
 	current_task()->faults++;
 
+	original_fault_type = fault_type;
+
     RetryFault: ;
 
 	/*
 	 *	Find the backing store object and offset into
 	 *	it to begin the search.
 	 */
+	fault_type = original_fault_type;
 	map = original_map;
 	vm_map_lock_read(map);
 	kr = vm_map_lookup_locked(&map, vaddr, fault_type, &version,
@@ -2373,8 +2376,6 @@ vm_fault(
 FastMapInFault:
 				m->busy = TRUE;
 
-				vm_object_paging_begin(object);
-
 FastPmapEnter:
 				/*
 				 *	Check a couple of global reasons to
@@ -2444,35 +2445,27 @@ FastPmapEnter:
 				 *	move active page to back of active
 				 *	queue.  This code doesn't.
 				 */
-				vm_page_lock_queues();
-
 				if (m->clustered) {
 				        vm_pagein_cluster_used++;
 					m->clustered = FALSE;
 				}
-				m->reference = TRUE;
-
 				if (change_wiring) {
+				        vm_page_lock_queues();
+
 					if (wired)
 						vm_page_wire(m);
 					else
 						vm_page_unwire(m);
+
+					vm_page_unlock_queues();
 				}
-#if VM_FAULT_STATIC_CONFIG
 				else {
-				        if ((!m->active && !m->inactive) || ((need_activation == TRUE) && !m->active))
+				        if ((!m->active && !m->inactive) || ((need_activation == TRUE) && !m->active)) {
+					        vm_page_lock_queues();
 					        vm_page_activate(m);
+						vm_page_unlock_queues();
+					}
 				}
-#else				
-				else if (software_reference_bits) {
-					if (!m->active && !m->inactive)
-						vm_page_activate(m);
-				}
-				else if (!m->active) {
-					vm_page_activate(m);
-				}
-#endif
-				vm_page_unlock_queues();
 
 				/*
 				 *	That's it, clean up and return.
@@ -2489,12 +2482,14 @@ FastPmapEnter:
 				 * normal clustering behavior.
 				 */
 				if (!sequential && !object->private) {
+				        vm_object_paging_begin(object);
+
 					write_startup_file = 
 						vm_fault_tws_insert(map, real_map, vaddr, 
 					                        object, cur_offset);
-				}
 
-				vm_object_paging_end(object);
+					vm_object_paging_end(object);
+				}
 				vm_object_unlock(object);
 
 				vm_map_unlock_read(map);
@@ -2586,8 +2581,7 @@ FastPmapEnter:
 			 */     
  
 			vm_object_paging_end(object); 
-			vm_object_collapse(object, offset);
-			vm_object_paging_begin(object);
+			vm_object_collapse(object, offset, TRUE);
 
 			goto FastPmapEnter;
 		}
@@ -2664,9 +2658,6 @@ FastPmapEnter:
 			  	if (cur_object != object)
 					vm_object_unlock(cur_object);
 
-				vm_object_paging_begin(object);
-				vm_object_unlock(object);
-
 				/*
 				 *	Now zero fill page and map it.
 				 *	the page is probably going to 
@@ -2715,7 +2706,6 @@ FastPmapEnter:
 				m->inactive = TRUE;
 				vm_page_inactive_count++;
 				vm_page_unlock_queues();
-				vm_object_lock(object);
 
 				goto FastPmapEnter;
 		        }
@@ -3008,7 +2998,6 @@ FastPmapEnter:
 		}
 	} else {
 
-#ifndef i386
 		vm_map_entry_t		entry;
 		vm_map_offset_t		laddr;
 		vm_map_offset_t		ldelta, hdelta;
@@ -3018,12 +3007,13 @@ FastPmapEnter:
 		 * in the object 
 		 */
 
+#ifndef i386
 		/* While we do not worry about execution protection in   */
 		/* general, certian pages may have instruction execution */
 		/* disallowed.  We will check here, and if not allowed   */
 		/* to execute, we return with a protection failure.      */
 
-		if((full_fault_type & VM_PROT_EXECUTE) &&
+		if((fault_type & VM_PROT_EXECUTE) &&
 			(!pmap_eligible_for_execute((ppnum_t)
 				(object->shadow_offset >> 12)))) {
 
@@ -3035,6 +3025,7 @@ FastPmapEnter:
 			kr = KERN_PROTECTION_FAILURE;
 			goto done;
 		}
+#endif	/* !i386 */
 
 		if(real_map != map) {
 			vm_map_unlock(real_map);
@@ -3075,45 +3066,38 @@ FastPmapEnter:
 		}
 
 		if(vm_map_lookup_entry(map, laddr, &entry) && 
-					(entry->object.vm_object != NULL) &&
-					(entry->object.vm_object == object)) {
+		   (entry->object.vm_object != NULL) &&
+		   (entry->object.vm_object == object)) {
 
+			vm_map_offset_t	phys_offset;
 
+			phys_offset = (entry->object.vm_object->shadow_offset
+				       + entry->offset
+				       + laddr
+				       - entry->vme_start);
+			phys_offset -= ldelta;
 			if(caller_pmap) {
 				/* Set up a block mapped area */
-				pmap_map_block(caller_pmap, 
+				pmap_map_block(
+					caller_pmap, 
 					(addr64_t)(caller_pmap_addr - ldelta), 
-					(((vm_map_offset_t)
-				    (entry->object.vm_object->shadow_offset)) 
-					+ entry->offset + 
-					(laddr - entry->vme_start) 
-							- ldelta) >> 12,
-				((ldelta + hdelta) >> 12), prot, 
-				(VM_WIMG_MASK & (int)object->wimg_bits), 0);
+					phys_offset >> 12,
+					(ldelta + hdelta) >> 12,
+					prot,
+					(VM_WIMG_MASK &	(int)object->wimg_bits),
+					0);
 			} else { 
 				/* Set up a block mapped area */
-				pmap_map_block(real_map->pmap, 
-				   (addr64_t)(vaddr - ldelta), 
-				   (((vm_map_offset_t)
-				    (entry->object.vm_object->shadow_offset)) 
-				       + entry->offset + 
-				       (laddr - entry->vme_start) - ldelta) >> 12,
-				   ((ldelta + hdelta) >> 12), prot, 
-				   (VM_WIMG_MASK & (int)object->wimg_bits), 0);
+				pmap_map_block(
+					real_map->pmap, 
+					(addr64_t)(vaddr - ldelta), 
+					phys_offset >> 12,
+					(ldelta + hdelta) >> 12,
+					prot,
+					(VM_WIMG_MASK & (int)object->wimg_bits),
+					0);
 			}
 		}
-#else
-#ifdef notyet
-		if(caller_pmap) {
-               		pmap_enter(caller_pmap, caller_pmap_addr, 
-				object->shadow_offset>>12, prot, 0, TRUE);
-		} else {
-               		pmap_enter(pmap, vaddr, 
-				object->shadow_offset>>12, prot, 0, TRUE);
-		}
-        		/* Map it in */
-#endif
-#endif
 
 	}
 
diff --git a/osfmk/vm/vm_init.c b/osfmk/vm/vm_init.c
index 2ee659818..88f922eda 100644
--- a/osfmk/vm/vm_init.c
+++ b/osfmk/vm/vm_init.c
@@ -79,7 +79,11 @@
 #include <vm/vm_protos.h>
 
 #define ZONE_MAP_MIN (12 * 1024 * 1024) 
-#define ZONE_MAP_MAX (768 * 1024 * 1024) 
+/* Maximum Zone size is 1G */
+#define ZONE_MAP_MAX (1024 * 1024 * 1024) 
+
+const vm_offset_t vm_min_kernel_address = VM_MIN_KERNEL_ADDRESS;
+const vm_offset_t vm_max_kernel_address = VM_MAX_KERNEL_ADDRESS;
 
 /*
  *	vm_mem_bootstrap initializes the virtual memory system.
@@ -90,7 +94,8 @@ void
 vm_mem_bootstrap(void)
 {
 	vm_offset_t	start, end;
-	vm_size_t zsize;
+	vm_size_t zsizearg;
+	mach_vm_size_t zsize;
 
 	/*
 	 *	Initializes resident memory structures.
@@ -110,8 +115,8 @@ vm_mem_bootstrap(void)
 	kmem_init(start, end);
 	pmap_init();
 	
-	if (PE_parse_boot_arg("zsize", &zsize))
-		zsize = zsize * 1024 * 1024;
+	if (PE_parse_boot_arg("zsize", &zsizearg))
+		zsize = zsizearg * 1024ULL * 1024ULL;
 	else {
 		zsize = sane_size >> 2;				/* Get target zone size as 1/4 of physical memory */
 	}
diff --git a/osfmk/vm/vm_kern.c b/osfmk/vm/vm_kern.c
index d5ed1f659..b81d47b22 100644
--- a/osfmk/vm/vm_kern.c
+++ b/osfmk/vm/vm_kern.c
@@ -139,7 +139,7 @@ kmem_alloc_contig(
 		object = vm_object_allocate(map_size);
 	}
 
-	kr = vm_map_find_space(map, &map_addr, map_size, map_mask, &entry);
+	kr = vm_map_find_space(map, &map_addr, map_size, map_mask, 0, &entry);
 	if (KERN_SUCCESS != kr) {
 		vm_object_deallocate(object);
 		return kr;
@@ -167,6 +167,7 @@ kmem_alloc_contig(
 	for (i = 0; i < map_size; i += PAGE_SIZE) {
 		m = pages;
 		pages = NEXT_PAGE(m);
+		*(NEXT_PAGE_PTR(m)) = VM_PAGE_NULL;
 		m->busy = FALSE;
 		vm_page_insert(m, object, offset + i);
 	}
@@ -205,6 +206,10 @@ kmem_alloc_contig(
  *		  KMA_HERE		*addrp is base address, else "anywhere"
  *		  KMA_NOPAGEWAIT	don't wait for pages if unavailable
  *		  KMA_KOBJECT		use kernel_object
+ *		  KMA_LOMEM		support for 32 bit devices in a 64 bit world
+ *					if set and a lomemory pool is available
+ *					grab pages from it... this also implies
+ *					KMA_NOPAGEWAIT
  */
 
 kern_return_t
@@ -228,6 +233,12 @@ kernel_memory_allocate(
 		*addrp = 0;
 		return KERN_INVALID_ARGUMENT;
 	}
+	if (flags & KMA_LOMEM) {
+	        if ( !(flags & KMA_NOPAGEWAIT) ) {
+		        *addrp = 0;
+		        return KERN_INVALID_ARGUMENT;
+		}
+	}
 
 	map_size = vm_map_round_page(size);
 	map_mask = (vm_map_offset_t) mask;
@@ -243,12 +254,11 @@ kernel_memory_allocate(
 		object = vm_object_allocate(map_size);
 	}
 
-	kr = vm_map_find_space(map, &map_addr, map_size, map_mask, &entry);
+	kr = vm_map_find_space(map, &map_addr, map_size, map_mask, 0, &entry);
 	if (KERN_SUCCESS != kr) {
 		vm_object_deallocate(object);
 		return kr;
 	}
-
 	entry->object.vm_object = object;
 	entry->offset = offset = (object == kernel_object) ? 
 		        map_addr - VM_MIN_KERNEL_ADDRESS : 0;
@@ -260,8 +270,15 @@ kernel_memory_allocate(
 	for (i = 0; i < map_size; i += PAGE_SIZE) {
 		vm_page_t	mem;
 
-		while (VM_PAGE_NULL == 
-		       (mem = vm_page_alloc(object, offset + i))) {
+		for (;;) {
+		        if (flags & KMA_LOMEM)
+			        mem = vm_page_alloclo(object, offset + i);
+			else
+			        mem = vm_page_alloc(object, offset + i);
+
+		        if (mem != VM_PAGE_NULL)
+			        break;
+
 			if (flags & KMA_NOPAGEWAIT) {
 				if (object == kernel_object)
 					vm_object_page_remove(object, offset, offset + i);
@@ -389,7 +406,7 @@ kmem_realloc(
 	 */
 
 	kr = vm_map_find_space(map, &newmapaddr, newmapsize,
-			       (vm_map_offset_t) 0, &newentry);
+			       (vm_map_offset_t) 0, 0, &newentry);
 	if (kr != KERN_SUCCESS) {
 		vm_object_lock(object);
 		for(offset = oldmapsize; 
@@ -723,25 +740,24 @@ kmem_init(
 	map_end = vm_map_round_page(end);
 
 	kernel_map = vm_map_create(pmap_kernel(),VM_MIN_KERNEL_ADDRESS,
-				   map_end, FALSE);
-
+			    map_end, FALSE);
 	/*
 	 *	Reserve virtual memory allocated up to this time.
 	 */
 
 	if (start != VM_MIN_KERNEL_ADDRESS) {
 		vm_map_offset_t map_addr;
-
+ 
 		map_addr = VM_MIN_KERNEL_ADDRESS;
 		(void) vm_map_enter(kernel_map,
-				    &map_addr, 
-				    (vm_map_size_t)(map_start - VM_MIN_KERNEL_ADDRESS),
-				    (vm_map_offset_t) 0,
-				    VM_FLAGS_ANYWHERE | VM_FLAGS_NO_PMAP_CHECK,
-				    VM_OBJECT_NULL, 
-				    (vm_object_offset_t) 0, FALSE,
-				    VM_PROT_DEFAULT, VM_PROT_ALL,
-				    VM_INHERIT_DEFAULT);
+			    &map_addr, 
+			    (vm_map_size_t)(map_start - VM_MIN_KERNEL_ADDRESS),
+			    (vm_map_offset_t) 0,
+			    VM_FLAGS_ANYWHERE | VM_FLAGS_NO_PMAP_CHECK,
+			    VM_OBJECT_NULL, 
+			    (vm_object_offset_t) 0, FALSE,
+			    VM_PROT_NONE, VM_PROT_NONE,
+			    VM_INHERIT_DEFAULT);
 	}
 
         /*
diff --git a/osfmk/vm/vm_kern.h b/osfmk/vm/vm_kern.h
index 2b705c494..d71773991 100644
--- a/osfmk/vm/vm_kern.h
+++ b/osfmk/vm/vm_kern.h
@@ -83,6 +83,7 @@ extern kern_return_t	kernel_memory_allocate(
 #define KMA_HERE	0x01
 #define KMA_NOPAGEWAIT	0x02
 #define KMA_KOBJECT	0x04
+#define KMA_LOMEM	0x08
 
 extern kern_return_t kmem_alloc_contig(
 				vm_map_t	map,
diff --git a/osfmk/vm/vm_map.c b/osfmk/vm/vm_map.c
index 267ca6b37..a55583c4a 100644
--- a/osfmk/vm/vm_map.c
+++ b/osfmk/vm/vm_map.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -74,6 +74,7 @@
 #include <mach/vm_behavior.h>
 #include <mach/vm_statistics.h>
 #include <mach/memory_object.h>
+#include <mach/mach_vm.h>
 #include <machine/cpu_capabilities.h>
 
 #include <kern/assert.h>
@@ -192,11 +193,11 @@ static boolean_t	vm_map_fork_copy(
 				vm_map_entry_t	*old_entry_p,
 				vm_map_t	new_map);
 
-static void		vm_map_region_top_walk(
+void		vm_map_region_top_walk(
         			vm_map_entry_t		   entry,
 				vm_region_top_info_t       top);
 
-static void		vm_map_region_walk(
+void		vm_map_region_walk(
 				vm_map_t		   map,
 				vm_map_offset_t		   va,
         			vm_map_entry_t		   entry,
@@ -382,6 +383,91 @@ static vm_map_size_t vm_map_aggressive_enter_max;		/* set by bootstrap */
 /* Skip acquiring locks if we're in the midst of a kernel core dump */
 extern unsigned int not_in_kdp;
 
+#ifdef	__i386__
+kern_return_t
+vm_map_apple_protected(
+	vm_map_t	map,
+	vm_map_offset_t	start,
+	vm_map_offset_t	end)
+{
+	boolean_t	map_locked;
+	kern_return_t	kr;
+	vm_map_entry_t	map_entry;
+	memory_object_t	protected_mem_obj;
+	vm_object_t	protected_object;
+	vm_map_offset_t	map_addr;
+
+	vm_map_lock_read(map);
+	map_locked = TRUE;
+
+	/* lookup the protected VM object */
+	if (!vm_map_lookup_entry(map,
+				 start,
+				 &map_entry) ||
+	    map_entry->vme_end != end ||
+	    map_entry->is_sub_map) {
+		/* that memory is not properly mapped */
+		kr = KERN_INVALID_ARGUMENT;
+		goto done;
+	}
+	protected_object = map_entry->object.vm_object;
+	if (protected_object == VM_OBJECT_NULL) {
+		/* there should be a VM object here at this point */
+		kr = KERN_INVALID_ARGUMENT;
+		goto done;
+	}
+
+	/*
+	 * Lookup (and create if necessary) the protected memory object
+	 * matching that VM object.
+	 * If successful, this also grabs a reference on the memory object,
+	 * to guarantee that it doesn't go away before we get a chance to map
+	 * it.
+	 */
+	 
+	protected_mem_obj = apple_protect_pager_setup(protected_object);
+	if (protected_mem_obj == NULL) {
+		kr = KERN_FAILURE;
+		goto done;
+	}
+
+	vm_map_unlock_read(map);
+	map_locked = FALSE;
+
+	/* map this memory object in place of the current one */
+	map_addr = start;
+	kr = mach_vm_map(map,
+			 &map_addr,
+			 end - start,
+			 (mach_vm_offset_t) 0,
+			 VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE,
+			 (ipc_port_t) protected_mem_obj,
+			 map_entry->offset + (start - map_entry->vme_start),
+			 TRUE,
+			 map_entry->protection,
+			 map_entry->max_protection,
+			 map_entry->inheritance);
+	assert(map_addr == start);
+	if (kr == KERN_SUCCESS) {
+		/* let the pager know that this mem_obj is mapped */
+		apple_protect_pager_map(protected_mem_obj);
+	}
+	/*
+	 * Release the reference obtained by apple_protect_pager_setup().
+	 * The mapping (if it succeeded) is now holding a reference on the
+	 * memory object.
+	 */
+	memory_object_deallocate(protected_mem_obj);
+
+done:
+	if (map_locked) {
+		vm_map_unlock_read(map);
+	}
+	return kr;
+}
+#endif	/* __i386__ */
+
+
 void
 vm_map_init(
 	void)
@@ -740,12 +826,13 @@ vm_map_destroy(
 			     VM_MAP_NULL);
 	vm_map_unlock(map);
 	
-#ifdef __PPC__
 	if (map->hdr.nentries!=0)
-		vm_map_remove_commpage64(map);
-#endif /* __PPC__ */
+		vm_map_remove_commpage(map);
 	
-	assert(map->hdr.nentries==0);
+//	assert(map->hdr.nentries==0);
+//	if(map->hdr.nentries) {				/* (BRINGUP) */
+//		panic("vm_map_destroy: hdr.nentries is not 0 (%d) in map %08X\n", map->hdr.nentries, map);
+//	}
 
 	if(map->pmap)
 		pmap_destroy(map->pmap);
@@ -938,16 +1025,32 @@ void vm_map_swapout(vm_map_t map)
 
 
 /*
- *	SAVE_HINT:
+ *	SAVE_HINT_MAP_READ:
  *
  *	Saves the specified entry as the hint for
- *	future lookups.  Performs necessary interlocks.
+ *	future lookups.  only a read lock is held on map,
+ * 	so make sure the store is atomic... OSCompareAndSwap
+ *	guarantees this... also, we don't care if we collide
+ *	and someone else wins and stores their 'hint'
  */
-#define	SAVE_HINT(map,value) \
+#define	SAVE_HINT_MAP_READ(map,value) \
+MACRO_BEGIN \
+                OSCompareAndSwap((UInt32)((map)->hint), (UInt32)value, (UInt32 *)(&(map)->hint)); \
+MACRO_END
+
+
+/*
+ *	SAVE_HINT_MAP_WRITE:
+ *
+ *	Saves the specified entry as the hint for
+ *	future lookups.  write lock held on map,
+ * 	so no one else can be writing or looking
+ * 	until the lock is dropped, so it's safe
+ * 	to just do an assignment
+ */
+#define	SAVE_HINT_MAP_WRITE(map,value) \
 MACRO_BEGIN \
-		mutex_lock(&(map)->s_lock); \
 		(map)->hint = (value); \
-		mutex_unlock(&(map)->s_lock); \
 MACRO_END
 
 /*
@@ -973,11 +1076,7 @@ vm_map_lookup_entry(
 	 *	Start looking either from the head of the
 	 *	list, or from the hint.
 	 */
-	if (not_in_kdp)
-	  mutex_lock(&map->s_lock);
 	cur = map->hint;
-	if (not_in_kdp)
-	  mutex_unlock(&map->s_lock);
 
 	if (cur == vm_map_to_entry(map))
 		cur = cur->vme_next;
@@ -1021,8 +1120,8 @@ vm_map_lookup_entry(
 				 */
 
 				*entry = cur;
-				if (not_in_kdp)
-				  SAVE_HINT(map, cur);
+				SAVE_HINT_MAP_READ(map, cur);
+
 				return(TRUE);
 			}
 			break;
@@ -1030,8 +1129,8 @@ vm_map_lookup_entry(
 		cur = cur->vme_next;
 	}
 	*entry = cur->vme_prev;
-	if (not_in_kdp)
-	  SAVE_HINT(map, *entry);
+	SAVE_HINT_MAP_READ(map, *entry);
+
 	return(FALSE);
 }
 
@@ -1054,6 +1153,7 @@ vm_map_find_space(
 	vm_map_offset_t		*address,	/* OUT */
 	vm_map_size_t		size,
 	vm_map_offset_t		mask,
+	int			flags,
 	vm_map_entry_t		*o_entry)	/* OUT */
 {
 	register vm_map_entry_t	entry, new_entry;
@@ -1168,6 +1268,8 @@ vm_map_find_space(
 	new_entry->in_transition = FALSE;
 	new_entry->needs_wakeup = FALSE;
 
+	VM_GET_FLAGS_ALIAS(flags, new_entry->alias);
+
 	/*
 	 *	Insert the new entry into the list
 	 */
@@ -1179,7 +1281,7 @@ vm_map_find_space(
 	/*
 	 *	Update the lookup hint
 	 */
-	SAVE_HINT(map, new_entry);
+	SAVE_HINT_MAP_WRITE(map, new_entry);
 
 	*o_entry = new_entry;
 	return(KERN_SUCCESS);
@@ -1348,7 +1450,7 @@ vm_map_enter(
 	}
 
 	VM_GET_FLAGS_ALIAS(flags, alias);
-
+	
 #define	RETURN(value)	{ result = value; goto BailOut; }
 
 	assert(page_aligned(*address));
@@ -1595,9 +1697,20 @@ vm_map_enter(
 	 *	LP64todo - for now, we can only allocate 4GB internal objects
 	 *	because the default pager can't page bigger ones.  Remove this
 	 *	when it can.
+	 *
+	 * XXX FBDP
+	 * The reserved "page zero" in each process's address space can
+	 * be arbitrarily large.  Splitting it into separate 4GB objects and
+	 * therefore different VM map entries serves no purpose and just
+	 * slows down operations on the VM map, so let's not split the
+	 * allocation into 4GB chunks if the max protection is NONE.  That
+	 * memory should never be accessible, so it will never get to the
+	 * default pager.
 	 */
 	tmp_start = start;
-	if (object == VM_OBJECT_NULL && size > (vm_map_size_t)VM_MAX_ADDRESS) 
+	if (object == VM_OBJECT_NULL &&
+	    size > (vm_map_size_t)VM_MAX_ADDRESS &&
+	    max_protection != VM_PROT_NONE) 
 		tmp_end = tmp_start + (vm_map_size_t)VM_MAX_ADDRESS;
 	else
 		tmp_end = end;
@@ -1608,8 +1721,7 @@ vm_map_enter(
 					VM_BEHAVIOR_DEFAULT, inheritance, 0);
 		new_entry->alias = alias;
 		entry = new_entry;
-	} while (object == VM_OBJECT_NULL &&
-		 tmp_end != end && 
+	} while (tmp_end != end && 
 		 (tmp_start = tmp_end) &&
 		 (tmp_end = (end - tmp_end > (vm_map_size_t)VM_MAX_ADDRESS) ? 
 		  tmp_end + (vm_map_size_t)VM_MAX_ADDRESS : end));
@@ -1635,6 +1747,11 @@ vm_map_enter(
 	    (!needs_copy) && 
 	    (size < (128*1024))) {
 		pmap_empty = FALSE; /* pmap won't be empty */
+
+#ifdef STACK_ONLY_NX
+		if (alias != VM_MEMORY_STACK && cur_protection)
+		        cur_protection |= VM_PROT_EXECUTE;
+#endif
 		vm_map_pmap_enter(map, start, end, 
 				  object, offset, cur_protection);
 	}
@@ -1742,7 +1859,7 @@ vm_map_enter(
 #if	VM_CPM
 
 #ifdef MACH_ASSERT
-extern vm_offset_t	avail_start, avail_end;
+extern pmap_paddr_t	avail_start, avail_end;
 #endif
 
 /*
@@ -1815,6 +1932,7 @@ vm_map_enter_cpm(
 	for (offset = 0; offset < size; offset += PAGE_SIZE) {
 		m = pages;
 		pages = NEXT_PAGE(m);
+		*(NEXT_PAGE_PTR(m)) = VM_PAGE_NULL;
 
 		assert(!m->gobbled);
 		assert(!m->wanted);
@@ -1828,7 +1946,7 @@ vm_map_enter_cpm(
 		 */
 		ASSERT_PAGE_DECRYPTED(m);
 		assert(m->busy);
-		assert(m->phys_page>=avail_start && m->phys_page<=avail_end);
+		assert(m->phys_page>=(avail_start>>PAGE_SHIFT) && m->phys_page<=(avail_end>>PAGE_SHIFT));
 
 		m->busy = FALSE;
 		vm_page_insert(m, cpm_obj, offset);
@@ -1974,7 +2092,7 @@ vm_map_enter_cpm(
  *	the specified address; if necessary,
  *	it splits the entry into two.
  */
-#ifndef i386
+#ifndef NO_NESTED_PMAP
 #define vm_map_clip_start(map, entry, startaddr) 			\
 MACRO_BEGIN 								\
 	vm_map_t VMCS_map;						\
@@ -2001,7 +2119,7 @@ MACRO_BEGIN 								\
 	}								\
 	UPDATE_FIRST_FREE(VMCS_map, VMCS_map->first_free);		\
 MACRO_END
-#else
+#else /* NO_NESTED_PMAP */
 #define vm_map_clip_start(map, entry, startaddr) 			\
 MACRO_BEGIN 								\
 	vm_map_t VMCS_map;						\
@@ -2015,7 +2133,7 @@ MACRO_BEGIN 								\
 	}								\
 	UPDATE_FIRST_FREE(VMCS_map, VMCS_map->first_free);		\
 MACRO_END
-#endif
+#endif	/* NO_NESTED_PMAP */
 
 #define vm_map_copy_clip_start(copy, entry, startaddr) \
 	MACRO_BEGIN \
@@ -2066,7 +2184,7 @@ _vm_map_clip_start(
  *	the specified address; if necessary,
  *	it splits the entry into two.
  */
-#ifndef i386
+#ifndef NO_NESTED_PMAP 
 #define vm_map_clip_end(map, entry, endaddr) 				\
 MACRO_BEGIN 								\
 	vm_map_t VMCE_map;						\
@@ -2093,7 +2211,7 @@ MACRO_BEGIN 								\
 	}								\
 	UPDATE_FIRST_FREE(VMCE_map, VMCE_map->first_free);		\
 MACRO_END
-#else
+#else	/* NO_NESTED_PMAP */
 #define vm_map_clip_end(map, entry, endaddr) 				\
 MACRO_BEGIN 								\
 	vm_map_t VMCE_map;						\
@@ -2107,7 +2225,8 @@ MACRO_BEGIN 								\
 	}								\
 	UPDATE_FIRST_FREE(VMCE_map, VMCE_map->first_free);		\
 MACRO_END
-#endif
+#endif	/* NO_NESTED_PMAP */
+
 
 #define vm_map_copy_clip_end(copy, entry, endaddr) \
 	MACRO_BEGIN \
@@ -2249,9 +2368,9 @@ vm_map_submap(
 	vm_map_offset_t	end,
 	vm_map_t		submap,
 	vm_map_offset_t	offset,
-#ifdef i386
+#ifdef NO_NESTED_PMAP
 	__unused
-#endif
+#endif	/* NO_NESTED_PMAP */
 	boolean_t		use_pmap)
 {
 	vm_map_entry_t		entry;
@@ -2290,11 +2409,11 @@ vm_map_submap(
 			entry->is_sub_map = TRUE;
 			entry->object.sub_map = submap;
 			vm_map_reference(submap);
-#ifndef i386
+#ifndef NO_NESTED_PMAP
 			if ((use_pmap) && (offset == 0)) {
 				/* nest if platform code will allow */
 				if(submap->pmap == NULL) {
-					submap->pmap = pmap_create((vm_map_size_t) 0);
+					submap->pmap = pmap_create((vm_map_size_t) 0, FALSE);
 					if(submap->pmap == PMAP_NULL) {
 						vm_map_unlock(map);
 						return(KERN_NO_SPACE);
@@ -2308,10 +2427,9 @@ vm_map_submap(
 					panic("vm_map_submap: pmap_nest failed, rc = %08X\n", result);
 				entry->use_pmap = TRUE;
 			}
-#endif
-#ifdef i386
+#else	/* NO_NESTED_PMAP */
 			pmap_remove(map->pmap, (addr64_t)start, (addr64_t)end);
-#endif
+#endif	/* NO_NESTED_PMAP */
 			result = KERN_SUCCESS;
 	}
 	vm_map_unlock(map);
@@ -2460,14 +2578,15 @@ vm_map_protect(
 	 	   if(current->is_sub_map && current->use_pmap) {
 			vm_map_offset_t	pmap_base_addr;
 			vm_map_offset_t	pmap_end_addr;
-#ifdef i386
+#ifdef NO_NESTED_PMAP
 			__unused
-#endif
+#endif	/* NO_NESTED_PMAP */
 			vm_map_entry_t	local_entry;
 
+
 			pmap_base_addr = 0xF0000000 & current->vme_start;
 			pmap_end_addr = (pmap_base_addr + 0x10000000) - 1;
-#ifndef i386
+#ifndef NO_NESTED_PMAP
 			if(!vm_map_lookup_entry(map, 
 					pmap_base_addr, &local_entry))
 			   panic("vm_map_protect: nested pmap area is missing");
@@ -2477,21 +2596,29 @@ vm_map_protect(
 				local_entry = local_entry->vme_next;
 			   }
 			   pmap_unnest(map->pmap, (addr64_t)pmap_base_addr);
-#endif
+#endif	/* NO_NESTED_PMAP */
 		   }
 		   if (!(current->protection & VM_PROT_WRITE)) {
 			/* Look one level in we support nested pmaps */
 			/* from mapped submaps which are direct entries */
 			/* in our map */
-			if(current->is_sub_map && current->use_pmap) {
+
+		        vm_prot_t prot;
+
+			prot = current->protection;
+#ifdef STACK_ONLY_NX
+			if (current->alias != VM_MEMORY_STACK && prot)
+			        prot |= VM_PROT_EXECUTE;
+#endif
+			if (current->is_sub_map && current->use_pmap) {
 				pmap_protect(current->object.sub_map->pmap, 
 					current->vme_start,
 					current->vme_end,
-					current->protection);
+					prot);
 			} else {
 				pmap_protect(map->pmap, current->vme_start,
 					current->vme_end,
-					current->protection);
+					prot);
 			}
 		   }
 		}
@@ -3451,6 +3578,7 @@ vm_map_entry_delete(
 
 }
 
+
 void
 vm_map_submap_pmap_clean(
 	vm_map_t	map,
@@ -3595,7 +3723,7 @@ vm_map_delete(
 		 *	Fix the lookup hint now, rather than each
 		 *	time through the loop.
 		 */
-		SAVE_HINT(map, entry->vme_prev);
+		SAVE_HINT_MAP_WRITE(map, entry->vme_prev);
 	} else {
 		entry = first_entry->vme_next;
 	}
@@ -3651,7 +3779,7 @@ vm_map_delete(
 				entry = first_entry->vme_next;
 			} else {
 				entry = first_entry;
-				SAVE_HINT(map, entry->vme_prev);
+				SAVE_HINT_MAP_WRITE(map, entry->vme_prev);
 			}
 			last_timestamp = map->timestamp;
 			continue;
@@ -3714,7 +3842,7 @@ vm_map_delete(
 						entry = first_entry->vme_next;
 					} else {
 						entry = first_entry;
-						SAVE_HINT(map, entry->vme_prev);
+						SAVE_HINT_MAP_WRITE(map, entry->vme_prev);
 					}
 					last_timestamp = map->timestamp;
 					continue;
@@ -3752,10 +3880,10 @@ vm_map_delete(
 				       	       (!entry->is_sub_map));
 					first_entry = first_entry->vme_next;
 				} else {
-					SAVE_HINT(map, entry->vme_prev);
+					SAVE_HINT_MAP_WRITE(map, entry->vme_prev);
 				}
 			} else {
-				SAVE_HINT(map, entry->vme_prev);
+				SAVE_HINT_MAP_WRITE(map, entry->vme_prev);
 				first_entry = entry;
 			}
 
@@ -3789,9 +3917,10 @@ vm_map_delete(
 		    entry->is_sub_map) {
 			if(entry->is_sub_map) {
 			   if(entry->use_pmap) {
-#ifndef i386
-				pmap_unnest(map->pmap, (addr64_t)entry->vme_start);
-#endif
+#ifndef NO_NESTED_PMAP
+				pmap_unnest(map->pmap,
+					    (addr64_t)entry->vme_start);
+#endif	/* NO_NESTED_PMAP */
 				if((map->mapped) && (map->ref_count)) {
 					/* clean up parent map/maps */
 					vm_map_submap_pmap_clean(
@@ -3817,8 +3946,8 @@ vm_map_delete(
 					VM_PROT_NONE);
 			   } else {
 				   pmap_remove(map->pmap,
-					       entry->vme_start,
-					       entry->vme_end);
+					       (addr64_t)entry->vme_start,
+					       (addr64_t)entry->vme_end);
 			   }
 			}
 		}
@@ -3871,7 +4000,7 @@ vm_map_delete(
         		if (!vm_map_lookup_entry(map, s, &entry)){
 	               		entry = entry->vme_next;
         		} else {
-				 SAVE_HINT(map, entry->vme_prev);
+				 SAVE_HINT_MAP_WRITE(map, entry->vme_prev);
        		 	}
 			/* 
 			 * others can not only allocate behind us, we can 
@@ -3896,6 +4025,7 @@ vm_map_delete(
 	return KERN_SUCCESS;
 }
 
+
 /*
  *	vm_map_remove:
  *
@@ -5146,10 +5276,10 @@ vm_map_copy_overwrite_aligned(
 			if (old_object != VM_OBJECT_NULL) {
 				if(entry->is_sub_map) {
 					if(entry->use_pmap) {
-#ifndef i386
+#ifndef NO_NESTED_PMAP
 						pmap_unnest(dst_map->pmap, 
-							    entry->vme_start);
-#endif
+							    (addr64_t)entry->vme_start);
+#endif	/* NO_NESTED_PMAP */
 						if(dst_map->mapped) {
 							/* clean up parent */
 							/* map/maps */
@@ -5249,6 +5379,10 @@ vm_map_copy_overwrite_aligned(
 					 */
 					if (entry->needs_copy)
 						prot &= ~VM_PROT_WRITE;
+#ifdef STACK_ONLY_NX
+					if (entry->alias != VM_MEMORY_STACK && prot)
+					        prot |= VM_PROT_EXECUTE;
+#endif
 					/* It is our policy to require */
 					/* explicit sync from anyone   */
 					/* writing code and then       */
@@ -5266,7 +5400,7 @@ vm_map_copy_overwrite_aligned(
 					if (!m->active && !m->inactive)
 						vm_page_activate(m);
 					vm_page_unlock_queues();
-					 PAGE_WAKEUP_DONE(m);
+					PAGE_WAKEUP_DONE(m);
 				}
 				vm_object_paging_end(object);
 				vm_object_unlock(object);
@@ -5732,6 +5866,7 @@ vm_map_copyout(
 		    register vm_map_offset_t va;
 		    vm_object_offset_t	 offset;
 		    register vm_object_t object;
+		    vm_prot_t prot;
 
 		    object = entry->object.vm_object;
 		    offset = entry->offset;
@@ -5782,8 +5917,12 @@ vm_map_copyout(
 
 			m->busy = TRUE;
 			vm_object_unlock(object);
-
-			PMAP_ENTER(dst_map->pmap, va, m, entry->protection, 
+			prot = entry->protection;
+#ifdef STACK_ONLY_NX
+			if (entry->alias != VM_MEMORY_STACK && prot)
+			        prot |= VM_PROT_EXECUTE;
+#endif
+			PMAP_ENTER(dst_map->pmap, va, m, prot, 
 						((unsigned int)
 							(m->object->wimg_bits)) 
 							& VM_WIMG_MASK,
@@ -5844,7 +5983,10 @@ vm_map_copyout(
 						prot = entry->protection;
 						if (entry->needs_copy)
 							prot &= ~VM_PROT_WRITE;
-
+#ifdef STACK_ONLY_NX
+						if (entry->alias != VM_MEMORY_STACK && prot)
+						        prot |= VM_PROT_EXECUTE;
+#endif
 						PMAP_ENTER(dst_map->pmap, va, 
 							m, prot, 
 							((unsigned int)
@@ -5879,7 +6021,7 @@ vm_map_copyout(
 	 *	Update the hints and the map size
 	 */
 
-	SAVE_HINT(dst_map, vm_map_copy_last_entry(copy));
+	SAVE_HINT_MAP_WRITE(dst_map, vm_map_copy_last_entry(copy));
 
 	dst_map->size += size;
 
@@ -6208,6 +6350,13 @@ RestartCopy:
 			 */
 
 			if (src_needs_copy && !tmp_entry->needs_copy) {
+			        vm_prot_t prot;
+
+				prot = src_entry->protection & ~VM_PROT_WRITE;
+#ifdef STACK_ONLY_NX
+				if (src_entry->alias != VM_MEMORY_STACK && prot)
+				        prot |= VM_PROT_EXECUTE;
+#endif
 				vm_object_pmap_protect(
 					src_object,
 					src_offset,
@@ -6216,8 +6365,8 @@ RestartCopy:
 						PMAP_NULL
 						: src_map->pmap),
 					src_entry->vme_start,
-					src_entry->protection &
-						~VM_PROT_WRITE);
+					prot);
+
 				tmp_entry->needs_copy = TRUE;
 			}
 
@@ -6508,7 +6657,7 @@ vm_map_fork_share(
 	object = old_entry->object.vm_object;
 	if (old_entry->is_sub_map) {
 		assert(old_entry->wired_count == 0);
-#ifndef i386
+#ifndef NO_NESTED_PMAP
 		if(old_entry->use_pmap) {
 			kern_return_t	result;
 
@@ -6520,7 +6669,7 @@ vm_map_fork_share(
 			if(result)
 				panic("vm_map_fork_share: pmap_nest failed!");
 		}
-#endif
+#endif	/* NO_NESTED_PMAP */
 	} else if (object == VM_OBJECT_NULL) {
 		object = vm_object_allocate((vm_map_size_t)(old_entry->vme_end -
 							old_entry->vme_start));
@@ -6637,7 +6786,14 @@ vm_map_fork_share(
 
 		if (!old_entry->needs_copy &&
 		    (old_entry->protection & VM_PROT_WRITE)) {
-			if(old_map->mapped) {
+		        vm_prot_t prot;
+
+			prot = old_entry->protection & ~VM_PROT_WRITE;
+#ifdef STACK_ONLY_NX
+			if (old_entry->alias != VM_MEMORY_STACK && prot)
+			        prot |= VM_PROT_EXECUTE;
+#endif
+			if (old_map->mapped) {
 				vm_object_pmap_protect(
 					old_entry->object.vm_object,
 					old_entry->offset,
@@ -6645,12 +6801,12 @@ vm_map_fork_share(
 							old_entry->vme_start),
 					PMAP_NULL,
 					old_entry->vme_start,
-					old_entry->protection & ~VM_PROT_WRITE);
+					prot);
 			} else {
 				pmap_protect(old_map->pmap,
 				     old_entry->vme_start,
 				     old_entry->vme_end,
-				     old_entry->protection & ~VM_PROT_WRITE);
+				     prot);
 			}
 		}
 		
@@ -6792,7 +6948,9 @@ vm_map_t
 vm_map_fork(
 	vm_map_t	old_map)
 {
-	pmap_t		new_pmap = pmap_create((vm_map_size_t) 0);
+	pmap_t		new_pmap = pmap_create(
+					(vm_map_size_t) 0,
+					task_has_64BitAddr(current_task()));
 	vm_map_t	new_map;
 	vm_map_entry_t	old_entry;
 	vm_map_size_t	new_size = 0, entry_size;
@@ -6861,6 +7019,13 @@ vm_map_fork(
 			 */
 			
 			if (src_needs_copy && !old_entry->needs_copy) {
+			        vm_prot_t prot;
+
+				prot = old_entry->protection & ~VM_PROT_WRITE;
+#ifdef STACK_ONLY_NX
+				if (old_entry->alias != VM_MEMORY_STACK && prot)
+				        prot |= VM_PROT_EXECUTE;
+#endif
 				vm_object_pmap_protect(
 					old_entry->object.vm_object,
 					old_entry->offset,
@@ -6871,7 +7036,7 @@ vm_map_fork(
 							? PMAP_NULL :
 							old_map->pmap),
 					old_entry->vme_start,
-					old_entry->protection & ~VM_PROT_WRITE);
+					prot);
 
 				old_entry->needs_copy = TRUE;
 			}
@@ -6959,10 +7124,7 @@ vm_map_lookup_locked(
 	 *	If the map has an interesting hint, try it before calling
 	 *	full blown lookup routine.
 	 */
-
-	mutex_lock(&map->s_lock);
 	entry = map->hint;
-	mutex_unlock(&map->s_lock);
 
 	if ((entry == vm_map_to_entry(map)) ||
 	    (vaddr < entry->vme_start) || (vaddr >= entry->vme_end)) {
@@ -7139,12 +7301,17 @@ RetrySubMap:
 						&copy_object);
 					copied_slowly = TRUE;
 			} else {
-				
 				/* set up shadow object */
 				copy_object = submap_entry->object.vm_object;
 				vm_object_reference(copy_object);
 				submap_entry->object.vm_object->shadowed = TRUE;
 				submap_entry->needs_copy = TRUE;
+
+				prot = submap_entry->protection & ~VM_PROT_WRITE;
+#ifdef STACK_ONLY_NX
+				if (submap_entry->alias != VM_MEMORY_STACK && prot)
+				        prot |= VM_PROT_EXECUTE;
+#endif
 				vm_object_pmap_protect(
 					submap_entry->object.vm_object,
 					submap_entry->offset,
@@ -7154,8 +7321,7 @@ RetrySubMap:
 						|| map->mapped) ?
 						PMAP_NULL : map->pmap,
 					submap_entry->vme_start,
-					submap_entry->protection &
-						~VM_PROT_WRITE);
+					prot);
 			}
 			
 
@@ -7229,14 +7395,25 @@ RetrySubMap:
 	 *	Check whether this task is allowed to have
 	 *	this page.
 	 */
-
 	prot = entry->protection;
+
+#ifdef STACK_ONLY_NX
+	if (entry->alias != VM_MEMORY_STACK && prot)
+	        /*
+		 * HACK -- if not a stack, than allow execution
+		 */
+	        prot |= VM_PROT_EXECUTE;
+#endif
 	if ((fault_type & (prot)) != fault_type) {
-	  if (*real_map != map) {
-		vm_map_unlock(*real_map);
-	  }
-	  *real_map = map;
-	  return KERN_PROTECTION_FAILURE;
+	        if (*real_map != map) {
+		        vm_map_unlock(*real_map);
+		}
+		*real_map = map;
+
+		if ((fault_type & VM_PROT_EXECUTE) && prot)
+		        log_nx_failure((addr64_t)vaddr, prot);
+
+		return KERN_PROTECTION_FAILURE;
 	}
 
 	/*
@@ -7246,7 +7423,7 @@ RetrySubMap:
 
 	*wired = (entry->wired_count != 0);
 	if (*wired)
-		prot = fault_type = entry->protection;
+	        fault_type = prot;
 
 	/*
 	 *	If the entry was copy-on-write, we either ...
@@ -7872,7 +8049,7 @@ vm_map_region(
 	}
 }
 
-static void
+void
 vm_map_region_top_walk(
         vm_map_entry_t		   entry,
 	vm_region_top_info_t       top)
@@ -7935,7 +8112,7 @@ vm_map_region_top_walk(
 	}
 }
 
-static void
+void
 vm_map_region_walk(
 	vm_map_t		   	map,
 	vm_map_offset_t		va,
@@ -8214,7 +8391,7 @@ vm_map_simplify_entry(
 		this_entry->offset = prev_entry->offset;
 	 	vm_object_deallocate(prev_entry->object.vm_object);
 		vm_map_entry_dispose(map, prev_entry);
-		SAVE_HINT(map, this_entry);
+		SAVE_HINT_MAP_WRITE(map, this_entry);
 		counter(c_vm_map_simplified++);
 	}
 }
@@ -8811,6 +8988,7 @@ vm_map_entry_insert(
 	new_entry->wired_count = wired_count;
 	new_entry->user_wired_count = 0;
 	new_entry->use_pmap = FALSE;
+	new_entry->alias = 0;
 
 	/*
 	 *	Insert the new entry into the list.
@@ -8823,7 +9001,7 @@ vm_map_entry_insert(
 	 *	Update the free space hint and the lookup hint.
 	 */
 
-	SAVE_HINT(map, new_entry);
+	SAVE_HINT_MAP_WRITE(map, new_entry);
 	return new_entry;
 }
 
@@ -8949,6 +9127,13 @@ vm_map_remap_extract(
 
 				if (!src_entry->needs_copy &&
 				    (src_entry->protection & VM_PROT_WRITE)) {
+				        vm_prot_t prot;
+
+				        prot = src_entry->protection & ~VM_PROT_WRITE;
+#ifdef STACK_ONLY_NX
+					if (src_entry->alias != VM_MEMORY_STACK && prot)
+					        prot |= VM_PROT_EXECUTE;
+#endif
 					if(map->mapped) {
 					   vm_object_pmap_protect(
 						src_entry->object.vm_object,
@@ -8956,14 +9141,12 @@ vm_map_remap_extract(
 						entry_size,
 						PMAP_NULL,
 						src_entry->vme_start,
-						src_entry->protection &
-						       ~VM_PROT_WRITE);
+						prot);
 					} else {
 					   pmap_protect(vm_map_pmap(map),
-					  	 src_entry->vme_start,
-					     	src_entry->vme_end,
-					     	src_entry->protection &
-						     ~VM_PROT_WRITE);
+							src_entry->vme_start,
+							src_entry->vme_end,
+							prot);
 					}
 				}
 
@@ -9023,6 +9206,13 @@ vm_map_remap_extract(
 			 * Handle copy_on_write semantics.
 			 */
 			if (src_needs_copy && !src_entry->needs_copy) {
+			        vm_prot_t prot;
+
+				prot = src_entry->protection & ~VM_PROT_WRITE;
+#ifdef STACK_ONLY_NX
+				if (src_entry->alias != VM_MEMORY_STACK && prot)
+				        prot |= VM_PROT_EXECUTE;
+#endif
 				vm_object_pmap_protect(object,
 						       offset,
 						       entry_size,
@@ -9030,8 +9220,7 @@ vm_map_remap_extract(
 							  || map->mapped) ?
 							PMAP_NULL : map->pmap),
 						       src_entry->vme_start,
-						       src_entry->protection &
-						       ~VM_PROT_WRITE);
+						       prot);
 
 				src_entry->needs_copy = TRUE;
 			}
@@ -9173,7 +9362,7 @@ vm_map_remap(
 {
 	kern_return_t		result;
 	vm_map_entry_t		entry;
-	vm_map_entry_t		insp_entry;
+	vm_map_entry_t		insp_entry = VM_MAP_ENTRY_NULL;
 	vm_map_entry_t		new_entry;
 	struct vm_map_header	map_header;
 
@@ -9236,7 +9425,7 @@ vm_map_remap(
 
 	if (result == KERN_SUCCESS) {
 		target_map->size += size;
-		SAVE_HINT(target_map, insp_entry);
+		SAVE_HINT_MAP_WRITE(target_map, insp_entry);
 	}
 	vm_map_unlock(target_map);
 
@@ -9796,13 +9985,20 @@ kern_return_t vm_map_region_replace(
 	}
 	if ((entry->use_pmap) && 
 			(new_submap->pmap == NULL)) {
-		new_submap->pmap = pmap_create((vm_map_size_t) 0);
+		new_submap->pmap = pmap_create((vm_map_size_t) 0, FALSE);
 		if(new_submap->pmap == PMAP_NULL) {
 			vm_map_unlock(old_submap);
 			vm_map_unlock(target_map);
 			return(KERN_NO_SPACE);
 		}
 	}
+
+	/*
+	 * Mark the new submap as "mapped", so that we get proper
+	 * cleanup of the sub-pmap when we unmap it.
+	 */
+	new_submap->mapped = TRUE;
+
 	addr = entry->vme_start;
 	vm_map_reference(old_submap);
 	while((entry != vm_map_to_entry(target_map)) && 
@@ -9827,7 +10023,7 @@ kern_return_t vm_map_region_replace(
 		addr = entry->vme_start;
 	}
 	if(nested_pmap) {
-#ifndef i386
+#ifndef NO_NESTED_PMAP
 		pmap_unnest(target_map->pmap, (addr64_t)start);
 		if(target_map->mapped) {
 			vm_map_submap_pmap_clean(target_map,
@@ -9836,7 +10032,7 @@ kern_return_t vm_map_region_replace(
 		pmap_nest(target_map->pmap, new_submap->pmap, 
 				(addr64_t)start, (addr64_t)start, 
 				(uint64_t)(end - start));
-#endif /* i386 */
+#endif	/* NO_NESTED_PMAP */
 	} else {
 		vm_map_submap_pmap_clean(target_map,
 				start, end, old_submap, 0);
@@ -10391,91 +10587,212 @@ vm_map_deallocate(
 	vm_map_destroy(map);
 }
 
-#ifdef __PPC__
 
 /* LP64todo - this whole mechanism is temporary.  It should be redone when
  * the pmap layer can handle 64-bit address spaces.  Until then, we trump
  * up a map entry for the 64-bit commpage above the map's max_offset.
  */
 extern vm_map_t		com_region_map64;	/* the submap for 64-bit commpage */
-SInt32			commpage64s_in_use = 0;
+extern vm_map_t		com_region_map32;	/* the submap for 32-bit commpage */
+
  
-void
-vm_map_commpage64(
-        vm_map_t        map )
+static void
+vm_map_commpage(
+        vm_map_t        user_map,
+	vm_map_t	com_region_map,		/* com_region_map32 or com_region_map64 */
+	vm_map_offset_t	base_address,
+	vm_map_size_t	size)
 {
         vm_map_entry_t  entry;
 	vm_object_t	object;
         
-        vm_map_lock(map);
+        vm_map_lock(user_map);
         
         /* The commpage is necessarily the last entry in the map.
 	 * See if one is already there (not sure if this can happen???)
 	 */
-	entry = vm_map_last_entry(map);
-	if (entry != vm_map_to_entry(map)) {
-		if (entry->vme_end >= (vm_map_offset_t)_COMM_PAGE_BASE_ADDRESS) {
-			vm_map_unlock(map);
+	entry = vm_map_last_entry(user_map);
+	if (entry != vm_map_to_entry(user_map)) {
+		if (entry->vme_end >= base_address) {
+			vm_map_unlock(user_map);
 			return;
 		}
 	}
 	
-	entry = vm_map_first_entry(com_region_map64);	/* the 64-bit commpage */
+	entry = vm_map_first_entry(com_region_map);
 	object = entry->object.vm_object;
 	vm_object_reference(object);
-        
+
 	/* We bypass vm_map_enter() because we are adding the entry past the
 	 * map's max_offset.
 	 */
         entry = vm_map_entry_insert(
-                        map,
-                        vm_map_last_entry(map),		/* insert after last entry */
-                        _COMM_PAGE_BASE_ADDRESS,
-                        _COMM_PAGE_BASE_ADDRESS+_COMM_PAGE_AREA_USED,
+                        user_map,
+                        vm_map_last_entry(user_map),	/* insert after last entry */
+                        base_address,
+                        base_address + size,
                         object,
                         0,				/* offset */
                         FALSE,				/* needs_copy */
 			FALSE,				/* is_shared */
                         FALSE,				/* in_transition */
-			VM_PROT_READ,
-			VM_PROT_READ,
+			VM_PROT_READ|VM_PROT_EXECUTE,
+			VM_PROT_READ|VM_PROT_EXECUTE,
 			VM_BEHAVIOR_DEFAULT,
 			VM_INHERIT_NONE,
 			1 );				/* wired_count */
 	
-	vm_map_unlock(map);
-	
-	OSIncrementAtomic(&commpage64s_in_use);
+	vm_map_unlock(user_map);
+}
+
+#ifdef __i386__
+void
+vm_map_commpage32(
+	vm_map_t	map)
+{
+	vm_map_commpage(map,
+			com_region_map32,
+			(vm_map_offset_t) (unsigned) _COMM_PAGE32_BASE_ADDRESS,
+			(vm_map_size_t) (unsigned) _COMM_PAGE32_AREA_USED);
 }
+#endif	/* __i386__ */
+
 
 
-/* LP64todo - remove this!  */
+void
+vm_map_commpage64(
+	vm_map_t	map)
+{
+
+	vm_map_commpage(map,
+			com_region_map64,
+			(vm_map_offset_t) _COMM_PAGE64_BASE_ADDRESS,
+			(vm_map_size_t) _COMM_PAGE64_AREA_USED);
+}
 
 void
-vm_map_remove_commpage64(
+vm_map_remove_commpage(
         vm_map_t        map )
 {
         vm_map_entry_t  entry;
-	int		deleted = 0;
 	
 	while( 1 ) {
 		vm_map_lock(map);
 	
 		entry = vm_map_last_entry(map);
+
 		if ((entry == vm_map_to_entry(map)) || 
-			(entry->vme_start < (vm_map_offset_t)_COMM_PAGE_BASE_ADDRESS))
+			(entry->vme_start < map->max_offset))
 				break;
 				
 		/* clearing the wired count isn't strictly correct */
 		entry->wired_count = 0;
 		vm_map_entry_delete(map,entry);
-		deleted++;
 	}
 	
 	vm_map_unlock(map);
-	
-	if (deleted != 0)
-		OSDecrementAtomic(&commpage64s_in_use);
 }
 
-#endif /* __PPC__ */
+void
+vm_map_disable_NX(vm_map_t map)
+{
+        if (map == NULL)
+	        return;
+        if (map->pmap == NULL)
+	        return;
+
+        pmap_disable_NX(map->pmap);
+}
+
+/* XXX Consider making these constants (VM_MAX_ADDRESS and MACH_VM_MAX_ADDRESS)
+ * more descriptive.
+ */
+void
+vm_map_set_32bit(vm_map_t map)
+{
+	map->max_offset = (vm_map_offset_t)VM_MAX_ADDRESS;
+}
+
+
+void
+vm_map_set_64bit(vm_map_t map)
+{
+	map->max_offset = (vm_map_offset_t)MACH_VM_MAX_ADDRESS;
+}
+
+vm_map_offset_t
+vm_compute_max_offset(unsigned is64)
+{
+	return (is64 ? (vm_map_offset_t)MACH_VM_MAX_ADDRESS : (vm_map_offset_t)VM_MAX_ADDRESS);
+}
+
+boolean_t
+vm_map_has_4GB_pagezero(vm_map_t map)
+{
+	/*
+	 * XXX FBDP
+	 * We should lock the VM map (for read) here but we can get away
+	 * with it for now because there can't really be any race condition:
+	 * the VM map's min_offset is changed only when the VM map is created
+	 * and when the zero page is established (when the binary gets loaded),
+	 * and this routine gets called only when the task terminates and the
+	 * VM map is being torn down, and when a new map is created via
+	 * load_machfile()/execve().
+	 */
+	return (map->min_offset >= 0x100000000ULL);
+}
+
+void
+vm_map_set_4GB_pagezero(vm_map_t map)
+{
+	pmap_set_4GB_pagezero(map->pmap);
+}
+
+void
+vm_map_clear_4GB_pagezero(vm_map_t map)
+{
+	pmap_clear_4GB_pagezero(map->pmap);
+}
+
+/*
+ * Raise a VM map's minimum offset.
+ * To strictly enforce "page zero" reservation.
+ */
+kern_return_t
+vm_map_raise_min_offset(
+	vm_map_t	map,
+	vm_map_offset_t	new_min_offset)
+{
+	vm_map_entry_t	first_entry;
+
+	new_min_offset = vm_map_round_page(new_min_offset);
+
+	vm_map_lock(map);
+
+	if (new_min_offset < map->min_offset) {
+		/*
+		 * Can't move min_offset backwards, as that would expose
+		 * a part of the address space that was previously, and for
+		 * possibly good reasons, inaccessible.
+		 */
+		vm_map_unlock(map);
+		return KERN_INVALID_ADDRESS;
+	}
+
+	first_entry = vm_map_first_entry(map);
+	if (first_entry != vm_map_to_entry(map) &&
+	    first_entry->vme_start < new_min_offset) {
+		/*
+		 * Some memory was already allocated below the new
+		 * minimun offset.  It's too late to change it now...
+		 */
+		vm_map_unlock(map);
+		return KERN_NO_SPACE;
+	}
+
+	map->min_offset = new_min_offset;
+
+	vm_map_unlock(map);
+
+	return KERN_SUCCESS;
+}
diff --git a/osfmk/vm/vm_map.h b/osfmk/vm/vm_map.h
index c87e61a71..1b6c8fd67 100644
--- a/osfmk/vm/vm_map.h
+++ b/osfmk/vm/vm_map.h
@@ -410,6 +410,7 @@ extern kern_return_t vm_map_find_space(
 				vm_map_address_t	*address,	/* OUT */
 				vm_map_size_t		size,
 				vm_map_offset_t		mask,
+				int			flags,
 				vm_map_entry_t		*o_entry);	/* OUT */
 
 /* Lookup map entry containing or the specified address in the given map */
@@ -846,6 +847,31 @@ extern kern_return_t	vm_map_copyin_common(
 				vm_map_copy_t		*copy_result,	/* OUT */
 				boolean_t		use_maxprot);
 
+extern void		vm_map_disable_NX(
+			        vm_map_t		map);
+
+extern void		vm_map_set_64bit(
+			        vm_map_t		map);
+
+extern void		vm_map_set_32bit(
+			        vm_map_t		map);
+
+extern boolean_t	vm_map_has_4GB_pagezero(
+		       		vm_map_t		map);
+
+extern void		vm_map_set_4GB_pagezero(
+			        vm_map_t		map);
+
+extern void		vm_map_clear_4GB_pagezero(
+			        vm_map_t		map);
+
+extern kern_return_t	vm_map_raise_min_offset(
+	vm_map_t	map,
+	vm_map_offset_t	new_min_offset);
+
+extern vm_map_offset_t	vm_compute_max_offset(
+				unsigned		is64);
+
 /*
  *	Macros to invoke vm_map_copyin_common.  vm_map_copyin is the
  *	usual form; it handles a copyin based on the current protection
diff --git a/osfmk/vm/vm_object.c b/osfmk/vm/vm_object.c
index d32573f82..83b3a0eeb 100644
--- a/osfmk/vm/vm_object.c
+++ b/osfmk/vm/vm_object.c
@@ -203,7 +203,7 @@ static zone_t		vm_object_zone;		/* vm backing store zone */
  *	memory object (kernel_object) to avoid wasting data structures.
  */
 static struct vm_object			kernel_object_store;
-__private_extern__ vm_object_t		kernel_object = &kernel_object_store;
+vm_object_t						kernel_object;
 
 /*
  *	The submap object is used as a placeholder for vm_map_submap
@@ -552,7 +552,7 @@ vm_object_reaper_init(void)
 		BASEPRI_PREEMPT - 1,
 		&thread);
 	if (kr != KERN_SUCCESS) {
-		panic("failed to launch vm_object_reaper_thread kr=0x%x", kr);
+		panic("failed to launch vm_object_reaper_thread kr=0x%x\n", kr);
 	}
 	thread_deallocate(thread);
 }
@@ -680,15 +680,22 @@ vm_object_deallocate(
 			if (object->ref_count == 1 &&
 			    object->shadow != VM_OBJECT_NULL) {
 				/*
-				 * We don't use this VM object anymore.  We
-				 * would like to collapse it into its parent(s),
-				 * but we don't have any pointers back to these
-				 * parent object(s).
+				 * There's only one reference left on this
+				 * VM object.  We can't tell if it's a valid
+				 * one (from a mapping for example) or if this
+				 * object is just part of a possibly stale and
+				 * useless shadow chain.
+				 * We would like to try and collapse it into
+				 * its parent, but we don't have any pointers
+				 * back to this parent object.
 				 * But we can try and collapse this object with
 				 * its own shadows, in case these are useless
 				 * too...
+				 * We can't bypass this object though, since we
+				 * don't know if this last reference on it is
+				 * meaningful or not.
 				 */
-				vm_object_collapse(object, 0);
+				vm_object_collapse(object, 0, FALSE);
 			}
 
 			vm_object_unlock(object); 
@@ -1089,6 +1096,7 @@ vm_object_terminate(
 		vm_object_unlock(object);
 		return KERN_SUCCESS;
 	}
+
 	/* complete the VM object termination */
 	vm_object_reap(object);
 	object = VM_OBJECT_NULL;
@@ -3423,7 +3431,8 @@ static unsigned long vm_object_collapse_do_bypass = 0;
 __private_extern__ void
 vm_object_collapse(
 	register vm_object_t			object,
-	register vm_object_offset_t		hint_offset)
+	register vm_object_offset_t		hint_offset,
+	boolean_t				can_bypass)
 {
 	register vm_object_t			backing_object;
 	register unsigned int			rcount;
@@ -3435,7 +3444,8 @@ vm_object_collapse(
 
 	vm_object_collapse_calls++;
 
-	if (! vm_object_collapse_allowed && ! vm_object_bypass_allowed) {
+	if (! vm_object_collapse_allowed &&
+	    ! (can_bypass && vm_object_bypass_allowed)) {
 		return;
 	}
 
@@ -3618,7 +3628,7 @@ vm_object_collapse(
 		 *	or permitted, so let's try bypassing it.
 		 */
 
-		if (! vm_object_bypass_allowed) {
+		if (! (can_bypass && vm_object_bypass_allowed)) {
 			/* try and collapse the rest of the shadow chain */
 			if (object != original_object) {
 				vm_object_unlock(object);
@@ -3946,7 +3956,7 @@ vm_object_coalesce(
 	/*
 	 *	Try to collapse the object first
 	 */
-	vm_object_collapse(prev_object, prev_offset);
+	vm_object_collapse(prev_object, prev_offset, TRUE);
 
 	/*
 	 *	Can't coalesce if pages not mapped to
@@ -4486,7 +4496,7 @@ vm_object_populate_with_private(
  *	memory_object_free_from_cache:
  *
  *	Walk the vm_object cache list, removing and freeing vm_objects 
- *	which are backed by the pager identified by the caller, (pager_id).  
+ *	which are backed by the pager identified by the caller, (pager_ops).  
  *	Remove up to "count" objects, if there are that may available
  *	in the cache.
  *
@@ -4497,7 +4507,7 @@ vm_object_populate_with_private(
 __private_extern__ kern_return_t
 memory_object_free_from_cache(
 	__unused host_t		host,
-	int		*pager_id,
+	memory_object_pager_ops_t pager_ops,
 	int		*count)
 {
 
@@ -4516,7 +4526,8 @@ memory_object_free_from_cache(
 
 	queue_iterate(&vm_object_cached_list, object, 
 					vm_object_t, cached_list) {
-		if (object->pager && (pager_id == object->pager->pager)) {
+		if (object->pager &&
+		    (pager_ops == object->pager->mo_pager_ops)) {
 			vm_object_lock(object);
 			queue_remove(&vm_object_cached_list, object, 
 					vm_object_t, cached_list);
@@ -5587,3 +5598,238 @@ done:
 
 	return retval;
 }
+
+
+/* Allow manipulation of individual page state.  This is actually part of */
+/* the UPL regimen but takes place on the VM object rather than on a UPL */
+
+kern_return_t
+vm_object_page_op(
+	vm_object_t		object,
+	vm_object_offset_t	offset,
+	int			ops,
+	ppnum_t			*phys_entry,
+	int			*flags)
+{
+	vm_page_t		dst_page;
+
+	vm_object_lock(object);
+
+	if(ops & UPL_POP_PHYSICAL) {
+		if(object->phys_contiguous) {
+			if (phys_entry) {
+				*phys_entry = (ppnum_t)
+					(object->shadow_offset >> 12);
+			}
+			vm_object_unlock(object);
+			return KERN_SUCCESS;
+		} else {
+			vm_object_unlock(object);
+			return KERN_INVALID_OBJECT;
+		}
+	}
+	if(object->phys_contiguous) {
+		vm_object_unlock(object);
+		return KERN_INVALID_OBJECT;
+	}
+
+	while(TRUE) {
+		if((dst_page = vm_page_lookup(object,offset)) == VM_PAGE_NULL) {
+			vm_object_unlock(object);
+			return KERN_FAILURE;
+		}
+
+		/* Sync up on getting the busy bit */
+		if((dst_page->busy || dst_page->cleaning) && 
+			   (((ops & UPL_POP_SET) && 
+			   (ops & UPL_POP_BUSY)) || (ops & UPL_POP_DUMP))) {
+			/* someone else is playing with the page, we will */
+			/* have to wait */
+			PAGE_SLEEP(object, dst_page, THREAD_UNINT);
+			continue;
+		}
+
+		if (ops & UPL_POP_DUMP) {
+		        vm_page_lock_queues();
+
+			if (dst_page->no_isync == FALSE)
+			        pmap_disconnect(dst_page->phys_page);
+			vm_page_free(dst_page);
+
+			vm_page_unlock_queues();
+			break;
+		}
+
+		if (flags) {
+		        *flags = 0;
+
+			/* Get the condition of flags before requested ops */
+			/* are undertaken */
+
+			if(dst_page->dirty) *flags |= UPL_POP_DIRTY;
+			if(dst_page->pageout) *flags |= UPL_POP_PAGEOUT;
+			if(dst_page->precious) *flags |= UPL_POP_PRECIOUS;
+			if(dst_page->absent) *flags |= UPL_POP_ABSENT;
+			if(dst_page->busy) *flags |= UPL_POP_BUSY;
+		}
+
+		/* The caller should have made a call either contingent with */
+		/* or prior to this call to set UPL_POP_BUSY */
+		if(ops & UPL_POP_SET) {
+			/* The protection granted with this assert will */
+			/* not be complete.  If the caller violates the */
+			/* convention and attempts to change page state */
+			/* without first setting busy we may not see it */
+			/* because the page may already be busy.  However */
+			/* if such violations occur we will assert sooner */
+			/* or later. */
+			assert(dst_page->busy || (ops & UPL_POP_BUSY));
+			if (ops & UPL_POP_DIRTY) dst_page->dirty = TRUE;
+			if (ops & UPL_POP_PAGEOUT) dst_page->pageout = TRUE;
+			if (ops & UPL_POP_PRECIOUS) dst_page->precious = TRUE;
+			if (ops & UPL_POP_ABSENT) dst_page->absent = TRUE;
+			if (ops & UPL_POP_BUSY) dst_page->busy = TRUE;
+		}
+
+		if(ops & UPL_POP_CLR) {
+			assert(dst_page->busy);
+			if (ops & UPL_POP_DIRTY) dst_page->dirty = FALSE;
+			if (ops & UPL_POP_PAGEOUT) dst_page->pageout = FALSE;
+			if (ops & UPL_POP_PRECIOUS) dst_page->precious = FALSE;
+			if (ops & UPL_POP_ABSENT) dst_page->absent = FALSE;
+			if (ops & UPL_POP_BUSY) {
+			        dst_page->busy = FALSE;
+				PAGE_WAKEUP(dst_page);
+			}
+		}
+
+		if (dst_page->encrypted) {
+			/*
+			 * ENCRYPTED SWAP:
+			 * We need to decrypt this encrypted page before the
+			 * caller can access its contents.
+			 * But if the caller really wants to access the page's
+			 * contents, they have to keep the page "busy".
+			 * Otherwise, the page could get recycled or re-encrypted
+			 * at any time.
+			 */
+			if ((ops & UPL_POP_SET) && (ops & UPL_POP_BUSY) &&
+			    dst_page->busy) {
+				/*
+				 * The page is stable enough to be accessed by
+				 * the caller, so make sure its contents are
+				 * not encrypted.
+				 */
+				vm_page_decrypt(dst_page, 0);
+			} else {
+				/*
+				 * The page is not busy, so don't bother
+				 * decrypting it, since anything could
+				 * happen to it between now and when the
+				 * caller wants to access it.
+				 * We should not give the caller access
+				 * to this page.
+				 */
+				assert(!phys_entry);
+			}
+		}
+
+		if (phys_entry) {
+			/*
+			 * The physical page number will remain valid
+			 * only if the page is kept busy.
+			 * ENCRYPTED SWAP: make sure we don't let the
+			 * caller access an encrypted page.
+			 */
+			assert(dst_page->busy);
+			assert(!dst_page->encrypted);
+			*phys_entry = dst_page->phys_page;
+		}
+
+		break;
+	}
+
+	vm_object_unlock(object);
+	return KERN_SUCCESS;
+				
+}
+
+/*
+ * vm_object_range_op offers performance enhancement over 
+ * vm_object_page_op for page_op functions which do not require page 
+ * level state to be returned from the call.  Page_op was created to provide 
+ * a low-cost alternative to page manipulation via UPLs when only a single 
+ * page was involved.  The range_op call establishes the ability in the _op 
+ * family of functions to work on multiple pages where the lack of page level
+ * state handling allows the caller to avoid the overhead of the upl structures.
+ */
+
+kern_return_t
+vm_object_range_op(
+	vm_object_t		object,
+	vm_object_offset_t	offset_beg,
+	vm_object_offset_t	offset_end,
+	int                     ops,
+	int                     *range)
+{
+        vm_object_offset_t	offset;
+	vm_page_t		dst_page;
+
+	if (object->resident_page_count == 0) {
+	        if (range) {
+		        if (ops & UPL_ROP_PRESENT)
+			        *range = 0;
+			else
+			        *range = offset_end - offset_beg;
+		}
+		return KERN_SUCCESS;
+	}
+	vm_object_lock(object);
+
+	if (object->phys_contiguous) {
+		vm_object_unlock(object);
+	        return KERN_INVALID_OBJECT;
+	}
+	
+	offset = offset_beg;
+
+	while (offset < offset_end) {
+		dst_page = vm_page_lookup(object, offset);
+		if (dst_page != VM_PAGE_NULL) {
+			if (ops & UPL_ROP_DUMP) {
+				if (dst_page->busy || dst_page->cleaning) {
+				        /*
+					 * someone else is playing with the 
+					 * page, we will have to wait
+					 */
+				        PAGE_SLEEP(object,
+						dst_page, THREAD_UNINT);
+					/*
+					 * need to relook the page up since it's
+					 * state may have changed while we slept
+					 * it might even belong to a different object
+					 * at this point
+					 */
+					continue;
+				}
+				vm_page_lock_queues();
+
+				if (dst_page->no_isync == FALSE)
+				        pmap_disconnect(dst_page->phys_page);
+				vm_page_free(dst_page);
+
+				vm_page_unlock_queues();
+			} else if (ops & UPL_ROP_ABSENT)
+			        break;
+		} else if (ops & UPL_ROP_PRESENT)
+		        break;
+
+		offset += PAGE_SIZE;
+	}
+	vm_object_unlock(object);
+
+	if (range)
+	        *range = offset - offset_beg;
+
+	return KERN_SUCCESS;
+}
diff --git a/osfmk/vm/vm_object.h b/osfmk/vm/vm_object.h
index a598722ab..d7a3433df 100644
--- a/osfmk/vm/vm_object.h
+++ b/osfmk/vm/vm_object.h
@@ -136,7 +136,7 @@ struct vm_object {
 						 * asserted.
 						 */
 
-	unsigned int		paging_in_progress;
+	int			paging_in_progress;
 						/* The memory object ports are
 						 * being used (e.g., for pagein
 						 * or pageout) -- don't change
@@ -277,6 +277,17 @@ struct vm_object {
 			not_in_use:24;
 #ifdef	UPL_DEBUG
 	queue_head_t		uplq;		/* List of outstanding upls */
+#ifdef	VM_PIP_DEBUG
+/*
+ * Keep track of the stack traces for the first holders
+ * of a "paging_in_progress" reference for this VM object.
+ */
+#define VM_PIP_DEBUG_STACK_FRAMES	25	/* depth of each stack trace */
+#define VM_PIP_DEBUG_MAX_REFS		10	/* track that many references */
+	struct __pip_backtrace {
+		void *pip_retaddr[VM_PIP_DEBUG_STACK_FRAMES];
+	} pip_holders[VM_PIP_DEBUG_MAX_REFS];
+#endif	/* VM_PIP_DEBUG  */
 #endif /* UPL_DEBUG */
 };
 
@@ -457,7 +468,8 @@ __private_extern__ boolean_t	vm_object_shadow(
 
 __private_extern__ void		vm_object_collapse(
 					vm_object_t		object,
-					vm_object_offset_t	offset);
+					vm_object_offset_t	offset,
+					boolean_t		can_bypass);
 
 __private_extern__ boolean_t	vm_object_copy_quickly(
 				vm_object_t		*_object,
@@ -568,10 +580,24 @@ __private_extern__ kern_return_t vm_object_populate_with_private(
 	ppnum_t			phys_page,
 	vm_size_t		size);
 
-__private_extern__ kern_return_t adjust_vm_object_cache(
+extern kern_return_t adjust_vm_object_cache(
 	vm_size_t oval,
 	vm_size_t nval);
 
+extern kern_return_t vm_object_page_op(
+	vm_object_t		object,
+	vm_object_offset_t	offset,
+	int			ops,
+	ppnum_t			*phys_entry,
+	int			*flags);
+
+extern kern_return_t vm_object_range_op(
+	vm_object_t		object,
+	vm_object_offset_t	offset_beg,
+	vm_object_offset_t	offset_end,
+	int                     ops,
+	int                     *range);
+
 /*
  *	Event waiting handling
  */
@@ -620,15 +646,30 @@ __private_extern__ kern_return_t adjust_vm_object_cache(
 /*
  *	Routines implemented as macros
  */
+#ifdef VM_PIP_DEBUG
+extern unsigned OSBacktrace(void **bt, unsigned maxAddrs);
+#define VM_PIP_DEBUG_BEGIN(object)					\
+	MACRO_BEGIN							\
+	if ((object)->paging_in_progress < VM_PIP_DEBUG_MAX_REFS) {	\
+		int pip = (object)->paging_in_progress;			\
+		(void) OSBacktrace(&(object)->pip_holders[pip].retaddr[0], \
+				   VM_PIP_DEBUG_STACK_FRAMES);		\
+	}								\
+	MACRO_END
+#else	/* VM_PIP_DEBUG */
+#define VM_PIP_DEBUG_BEGIN(object)
+#endif	/* VM_PIP_DEBUG */
 
 #define		vm_object_paging_begin(object) 				\
 	MACRO_BEGIN							\
+	assert((object)->paging_in_progress >= 0);			\
+	VM_PIP_DEBUG_BEGIN((object));					\
 	(object)->paging_in_progress++;					\
 	MACRO_END
 
 #define		vm_object_paging_end(object) 				\
 	MACRO_BEGIN							\
-	assert((object)->paging_in_progress != 0);			\
+	assert((object)->paging_in_progress > 0);			\
 	if (--(object)->paging_in_progress == 0) {			\
 		vm_object_wakeup(object,				\
 			VM_OBJECT_EVENT_PAGING_IN_PROGRESS);		\
diff --git a/osfmk/vm/vm_page.h b/osfmk/vm/vm_page.h
index 4e6031fd5..6199ec304 100644
--- a/osfmk/vm/vm_page.h
+++ b/osfmk/vm/vm_page.h
@@ -300,6 +300,14 @@ extern vm_offset_t	vm_page_fictitious_addr;
 
 extern boolean_t	vm_page_deactivate_hint;
 
+// 0 = all pages avail, 1 = disable high mem, 2 = prefer himem
+extern int		vm_himemory_mode;
+
+extern ppnum_t		vm_lopage_poolend;
+extern int		vm_lopage_poolsize;
+extern uint64_t		max_valid_dma_address;
+
+
 /*
  * Prototypes for functions exported by this module.
  */
@@ -331,6 +339,8 @@ extern int		vm_pool_low(void);
 
 extern vm_page_t	vm_page_grab(void);
 
+extern vm_page_t	vm_page_grablo(void);
+
 extern void		vm_page_release(
 					vm_page_t	page);
 
@@ -341,6 +351,10 @@ extern vm_page_t	vm_page_alloc(
 					vm_object_t		object,
 					vm_object_offset_t	offset);
 
+extern vm_page_t	vm_page_alloclo(
+					vm_object_t		object,
+					vm_object_offset_t	offset);
+
 extern void		vm_page_init(
 					vm_page_t	page,
 					ppnum_t		phys_page);
diff --git a/osfmk/vm/vm_pageout.c b/osfmk/vm/vm_pageout.c
index 87c5bd40c..fdc811c81 100644
--- a/osfmk/vm/vm_pageout.c
+++ b/osfmk/vm/vm_pageout.c
@@ -103,9 +103,6 @@
 /*
  * ENCRYPTED SWAP:
  */
-#ifdef __ppc__
-#include <ppc/mappings.h>
-#endif /* __ppc__ */
 #include <../bsd/crypto/aes/aes.h>
 
 extern ipc_port_t	memory_manager_default;
@@ -1069,9 +1066,6 @@ struct flow_control {
         mach_timespec_t	ts;
 };
 
-extern kern_return_t	sysclk_gettime(mach_timespec_t *);
-
-
 void
 vm_pageout_scan(void)
 {
@@ -1357,7 +1351,9 @@ done_with_activepage:
 reset_deadlock_timer:
 			        ts.tv_sec = vm_pageout_deadlock_wait / 1000;
 				ts.tv_nsec = (vm_pageout_deadlock_wait % 1000) * 1000 * NSEC_PER_USEC;
-				sysclk_gettime(&flow_control.ts);
+				clock_get_system_nanotime(
+					&flow_control.ts.tv_sec,
+					(uint32_t *) &flow_control.ts.tv_nsec);
 				ADD_MACH_TIMESPEC(&flow_control.ts, &ts);
 				
 				flow_control.state = FCS_DELAYED;
@@ -1366,7 +1362,9 @@ reset_deadlock_timer:
 				break;
 					
 			case FCS_DELAYED:
-			        sysclk_gettime(&ts);
+			        clock_get_system_nanotime(
+					&ts.tv_sec,
+					(uint32_t *) &ts.tv_nsec);
 
 				if (CMP_MACH_TIMESPEC(&ts, &flow_control.ts) >= 0) {
 				        /*
@@ -1939,7 +1937,9 @@ vm_pageout_iothread_continue(struct vm_pageout_queue *q)
 			    */
 
 			   if (!object->pager_initialized)
-			           vm_object_collapse(object, (vm_object_offset_t)0);
+			           vm_object_collapse(object,
+						      (vm_object_offset_t) 0,
+						      TRUE);
 			   if (!object->pager_initialized)
 			           vm_object_pager_create(object);
 			   if (!object->pager_initialized) {
@@ -2189,7 +2189,6 @@ vm_pageout(void)
 
 	vm_object_reaper_init();
 
-
 	vm_pageout_continue();
 	/*NOTREACHED*/
 }
@@ -2221,6 +2220,7 @@ upl_create(
 	upl->size = 0;
 	upl->map_object = NULL;
 	upl->ref_count = 1;
+	upl->highest_page = 0;
 	upl_lock_init(upl);
 #ifdef UPL_DEBUG
 	upl->ubc_alias1 = 0;
@@ -2399,7 +2399,7 @@ vm_object_upl_request(
 			*page_list_count = MAX_UPL_TRANSFER;
 
 	if((!object->internal) && (object->paging_offset != 0))
-		panic("vm_object_upl_request: vnode object with non-zero paging offset\n");
+		panic("vm_object_upl_request: external object with non-zero paging offset\n");
 
 	if((cntrl_flags & UPL_COPYOUT_FROM) && (upl_ptr == NULL)) {
 		return KERN_SUCCESS;
@@ -2507,6 +2507,7 @@ vm_object_upl_request(
 				   (offset + object->shadow_offset)>>PAGE_SHIFT;
 				user_page_list[0].device = TRUE;
 			}
+			upl->highest_page = (offset + object->shadow_offset + size - 1)>>PAGE_SHIFT;
 
 			if(page_list_count != NULL) {
 				if (upl->flags & UPL_INTERNAL) {
@@ -2819,6 +2820,9 @@ check_busy:
 					}
 				}
 
+				if (dst_page->phys_page > upl->highest_page)
+					upl->highest_page = dst_page->phys_page;
+
 				if(user_page_list) {
 					user_page_list[entry].phys_addr
 						= dst_page->phys_page;
@@ -3131,6 +3135,10 @@ check_busy:
 				dst_page->precious = 
 					(cntrl_flags & UPL_PRECIOUS) 
 							? TRUE : FALSE;
+
+				if (dst_page->phys_page > upl->highest_page)
+					upl->highest_page = dst_page->phys_page;
+
 				if(user_page_list) {
 					user_page_list[entry].phys_addr
 						= dst_page->phys_page;
@@ -3239,7 +3247,7 @@ vm_fault_list_request(
 	int			page_list_count,
 	int			cntrl_flags)
 {
-	int			local_list_count;
+	unsigned int		local_list_count;
 	upl_page_info_t		*user_page_list;
 	kern_return_t		kr;
 
@@ -4698,6 +4706,21 @@ vm_object_iopl_request(
 		 */
 		return KERN_INVALID_VALUE;
 	}
+	if (vm_lopage_poolsize == 0)
+	        cntrl_flags &= ~UPL_NEED_32BIT_ADDR;
+
+	if (cntrl_flags & UPL_NEED_32BIT_ADDR) {
+	        if ( (cntrl_flags & (UPL_SET_IO_WIRE | UPL_SET_LITE)) != (UPL_SET_IO_WIRE | UPL_SET_LITE))
+		        return KERN_INVALID_VALUE;
+
+		if (object->phys_contiguous) {
+		        if ((offset + object->shadow_offset) >= (vm_object_offset_t)max_valid_dma_address)
+			        return KERN_INVALID_ADDRESS;
+			  
+		        if (((offset + object->shadow_offset) + size) >= (vm_object_offset_t)max_valid_dma_address)
+			        return KERN_INVALID_ADDRESS;
+		}
+	}
 
 	if (cntrl_flags & UPL_ENCRYPT) {
 		/*
@@ -4730,7 +4753,7 @@ vm_object_iopl_request(
 		return KERN_INVALID_ARGUMENT;
 
 	if((!object->internal) && (object->paging_offset != 0))
-		panic("vm_object_upl_request: vnode object with non-zero paging offset\n");
+		panic("vm_object_upl_request: external object with non-zero paging offset\n");
 
 	if(object->phys_contiguous) {
 		/* No paging operations are possible against this memory */
@@ -4798,6 +4821,7 @@ vm_object_iopl_request(
 				  (offset + object->shadow_offset)>>PAGE_SHIFT;
 				user_page_list[0].device = TRUE;
 			}
+			upl->highest_page = (offset + object->shadow_offset + size - 1)>>PAGE_SHIFT;
 
 			if(page_list_count != NULL) {
 				if (upl->flags & UPL_INTERNAL) {
@@ -4967,24 +4991,75 @@ vm_object_iopl_request(
 				ret = (error_code ? error_code:
 					KERN_MEMORY_ERROR);
 				vm_object_lock(object);
-				for(; offset < dst_offset;
-						offset += PAGE_SIZE) {
-				   dst_page = vm_page_lookup(
-						object, offset);
-				   if(dst_page == VM_PAGE_NULL)
-					panic("vm_object_iopl_request: Wired pages missing. \n");
-				   vm_page_lock_queues();
-				   vm_page_unwire(dst_page);
-				   vm_page_unlock_queues();
-				   VM_STAT(reactivations++);
-				}
-				vm_object_unlock(object);
-				upl_destroy(upl);
-			   	return ret;
+
+				goto return_err;
 			}
 		   } while ((result != VM_FAULT_SUCCESS) 
 				|| (result == VM_FAULT_INTERRUPTED));
 		}
+
+		if ( (cntrl_flags & UPL_NEED_32BIT_ADDR) &&
+		     dst_page->phys_page >= (max_valid_dma_address >> PAGE_SHIFT) ) {
+		        vm_page_t	low_page;
+			int 		refmod;
+
+			/*
+			 * support devices that can't DMA above 32 bits
+			 * by substituting pages from a pool of low address
+			 * memory for any pages we find above the 4G mark
+			 * can't substitute if the page is already wired because
+			 * we don't know whether that physical address has been
+			 * handed out to some other 64 bit capable DMA device to use
+			 */
+		        if (dst_page->wire_count) {
+			        ret = KERN_PROTECTION_FAILURE;
+				goto return_err;
+			}
+		        if (delayed_unlock) {
+			        delayed_unlock = 0;
+			        vm_page_unlock_queues();
+			}
+			low_page = vm_page_grablo();
+
+			if (low_page == VM_PAGE_NULL) {
+			        ret = KERN_RESOURCE_SHORTAGE;
+				goto return_err;
+			}
+			/*
+			 * from here until the vm_page_replace completes
+			 * we musn't drop the object lock... we don't
+			 * want anyone refaulting this page in and using
+			 * it after we disconnect it... we want the fault
+			 * to find the new page being substituted.
+			 */
+			refmod = pmap_disconnect(dst_page->phys_page);
+
+			vm_page_copy(dst_page, low_page);
+			
+			low_page->reference = dst_page->reference;
+			low_page->dirty     = dst_page->dirty;
+
+			if (refmod & VM_MEM_REFERENCED)
+			        low_page->reference = TRUE;
+			if (refmod & VM_MEM_MODIFIED)
+			        low_page->dirty = TRUE;
+
+		        vm_page_lock_queues();
+			vm_page_replace(low_page, object, dst_offset);
+			/*
+			 * keep the queue lock since we're going to 
+			 * need it immediately
+			 */
+			delayed_unlock = 1;
+
+			dst_page = low_page;
+			/*
+			 * vm_page_grablo returned the page marked
+			 * BUSY... we don't need a PAGE_WAKEUP_DONE
+			 * here, because we've never dropped the object lock
+			 */
+			dst_page->busy = FALSE;
+		}
 		if (delayed_unlock == 0)
 		        vm_page_lock_queues();
 		vm_page_wire(dst_page);
@@ -5030,6 +5105,9 @@ vm_object_iopl_request(
 				dst_page->dirty = TRUE;
 			alias_page = NULL;
 
+			if (dst_page->phys_page > upl->highest_page)
+				upl->highest_page = dst_page->phys_page;
+
 			if (user_page_list) {
 				user_page_list[entry].phys_addr
 					= dst_page->phys_page;
@@ -5082,8 +5160,30 @@ vm_object_iopl_request(
 	}
 
 	return KERN_SUCCESS;
+
+
+return_err:
+	if (delayed_unlock)
+	        vm_page_unlock_queues();
+
+	for (; offset < dst_offset; offset += PAGE_SIZE) {
+	        dst_page = vm_page_lookup(object, offset);
+
+		if (dst_page == VM_PAGE_NULL)
+		        panic("vm_object_iopl_request: Wired pages missing. \n");
+		vm_page_lock_queues();
+		vm_page_unwire(dst_page);
+		vm_page_unlock_queues();
+		VM_STAT(reactivations++);
+	}
+	vm_object_paging_end(object);
+	vm_object_unlock(object);
+	upl_destroy(upl);
+
+	return ret;
 }
 
+
 kern_return_t
 upl_transpose(
 	upl_t		upl1,
@@ -5231,16 +5331,12 @@ vm_paging_map_object(
 	vm_map_offset_t		page_map_offset;
 	vm_map_size_t		map_size;
 	vm_object_offset_t	object_offset;
-#ifdef __ppc__
 	int			i;
 	vm_map_entry_t		map_entry;
-#endif /* __ppc__ */
 
 
-#ifdef __ppc__
 	if (page != VM_PAGE_NULL && *size == PAGE_SIZE) {
 		/*
-		 * Optimization for the PowerPC.
 		 * Use one of the pre-allocated kernel virtual addresses
 		 * and just enter the VM page in the kernel address space
 		 * at that virtual address.
@@ -5259,6 +5355,7 @@ vm_paging_map_object(
 					       &page_map_offset,
 					       VM_PAGING_NUM_PAGES * PAGE_SIZE,
 					       0,
+					       0,
 					       &map_entry);
 			if (kr != KERN_SUCCESS) {
 				panic("vm_paging_map_object: "
@@ -5310,14 +5407,17 @@ vm_paging_map_object(
 			}
 			vm_paging_page_inuse[i] = TRUE;
 			simple_unlock(&vm_paging_lock);
-			pmap_map_block(kernel_pmap,
-				       page_map_offset,
-				       page->phys_page,
-				       1,						/* Size is number of 4k pages */
-				       VM_PROT_DEFAULT,
-				       ((int) page->object->wimg_bits &
-					VM_WIMG_MASK),
-				       0);
+			if (page->no_isync == TRUE) {
+				pmap_sync_page_data_phys(page->phys_page);
+			}
+			assert(pmap_verify_free(page->phys_page));
+			PMAP_ENTER(kernel_pmap,
+				   page_map_offset,
+				   page,
+				   VM_PROT_DEFAULT,
+				   ((int) page->object->wimg_bits &
+				    VM_WIMG_MASK),
+				   TRUE);
 			vm_paging_objects_mapped++;
 			vm_paging_pages_mapped++; 
 			*address = page_map_offset;
@@ -5336,7 +5436,6 @@ vm_paging_map_object(
 		simple_unlock(&vm_paging_lock);
 		vm_object_lock(object);
 	}
-#endif /* __ppc__ */
 
 	object_offset = vm_object_trunc_page(offset);
 	map_size = vm_map_round_page(*size);
@@ -5394,12 +5493,13 @@ vm_paging_map_object(
 		}
 		cache_attr = ((unsigned int) object->wimg_bits) & VM_WIMG_MASK;
 
+		assert(pmap_verify_free(page->phys_page));
 		PMAP_ENTER(kernel_pmap,
 			   *address + page_map_offset,
 			   page,
 			   VM_PROT_DEFAULT,
 			   cache_attr,
-			   FALSE);
+			   TRUE);
 	}
 			   
 	vm_paging_objects_mapped_slow++;
@@ -5424,11 +5524,9 @@ vm_paging_unmap_object(
 	vm_map_offset_t	end)
 {
 	kern_return_t	kr;
-#ifdef __ppc__
 	int		i;
-#endif /* __ppc__ */
 
-	if ((vm_paging_base_address != 0) ||
+	if ((vm_paging_base_address == 0) ||
 	    (start < vm_paging_base_address) ||
 	    (end > (vm_paging_base_address
 		    + (VM_PAGING_NUM_PAGES * PAGE_SIZE)))) {
@@ -5451,17 +5549,15 @@ vm_paging_unmap_object(
 		 * pre-allocated pool.  Put it back in the pool
 		 * for next time.
 		 */
-#ifdef __ppc__
 		assert(end - start == PAGE_SIZE);
 		i = (start - vm_paging_base_address) >> PAGE_SHIFT;
 
 		/* undo the pmap mapping */
-		mapping_remove(kernel_pmap, start);
+		pmap_remove(kernel_pmap, start, end);
 
 		simple_lock(&vm_paging_lock);
 		vm_paging_page_inuse[i] = FALSE;
 		simple_unlock(&vm_paging_lock);
-#endif /* __ppc__ */
 	}
 }
 
@@ -5853,7 +5949,7 @@ vm_page_decrypt(
 	 * be part of a DMA transfer from a driver that expects the memory to
 	 * be coherent at this point, we have to flush the data cache.
 	 */
-	pmap_sync_page_data_phys(page->phys_page);
+	pmap_sync_page_attributes_phys(page->phys_page);
 	/*
 	 * Since the page is not mapped yet, some code might assume that it
 	 * doesn't need to invalidate the instruction cache when writing to
@@ -5976,18 +6072,16 @@ upl_get_internal_pagelist_offset(void)
 	return sizeof(struct upl);
 }
 
-void
-upl_set_dirty(
-	upl_t	upl)
-{
-	upl->flags |= UPL_CLEAR_DIRTY;
-}
-
 void
 upl_clear_dirty(
-	upl_t	upl)
+	upl_t		upl,
+	boolean_t 	value)
 {
-	upl->flags &= ~UPL_CLEAR_DIRTY;
+	if (value) {
+		upl->flags |= UPL_CLEAR_DIRTY;
+	} else {
+		upl->flags &= ~UPL_CLEAR_DIRTY;
+	}
 }
 
 
@@ -6082,6 +6176,12 @@ vm_countdirtypages(void)
 }
 #endif /* MACH_BSD */
 
+ppnum_t upl_get_highest_page(
+	upl_t			upl)
+{
+	return upl->highest_page;
+}
+
 #ifdef UPL_DEBUG
 kern_return_t  upl_ubc_alias_set(upl_t upl, unsigned int alias1, unsigned int alias2)
 {
diff --git a/osfmk/vm/vm_pageout.h b/osfmk/vm/vm_pageout.h
index dddf37fc2..098dc398e 100644
--- a/osfmk/vm/vm_pageout.h
+++ b/osfmk/vm/vm_pageout.h
@@ -85,6 +85,9 @@ extern kern_return_t vm_map_create_upl(
 	unsigned int		*count,
 	int			*flags);
 
+extern ppnum_t upl_get_highest_page(
+	upl_t			upl);
+
 #ifdef	MACH_KERNEL_PRIVATE
 
 #include <vm/vm_page.h>
@@ -146,6 +149,7 @@ struct upl {
 	upl_size_t	size;	    /* size in bytes of the address space */
 	vm_offset_t	kaddr;      /* secondary mapping in kernel */
 	vm_object_t	map_object;
+	ppnum_t		highest_page;
 #ifdef	UPL_DEBUG
 	unsigned int	ubc_alias1;
 	unsigned int	ubc_alias2;
diff --git a/osfmk/vm/vm_protos.h b/osfmk/vm/vm_protos.h
index 8e1d08c0d..f324e5993 100644
--- a/osfmk/vm/vm_protos.h
+++ b/osfmk/vm/vm_protos.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -63,6 +63,7 @@ extern kern_return_t device_close(
  */
 extern int start_def_pager(
 	char *bs_device);
+extern int default_pager_init_flag;
 
 /*
  * osfmk
@@ -71,6 +72,8 @@ extern int start_def_pager(
 /* these should be exported cleanly from OSFMK since BSD needs them */
 extern ipc_port_t convert_task_to_port(
 	task_t		task);
+extern ipc_port_t convert_task_name_to_port(
+	task_name_t	task_name);
 #endif /* _KERN_IPC_TT_H_ */
 #ifndef _IPC_IPC_PORT_H_
 extern mach_port_name_t ipc_port_copyout_send(
@@ -139,13 +142,22 @@ extern mach_vm_offset_t mach_get_vm_end(vm_map_t);
 extern vm_offset_t get_vm_start(vm_map_t);
 extern vm_offset_t get_vm_end(vm_map_t);
 
-#ifdef __PPC__
 /*
  * LP64todo - map in the commpage cleanly and remove these.
  */
 extern void vm_map_commpage64( vm_map_t );
-extern void vm_map_remove_commpage64( vm_map_t );
-#endif /* __PPC__ */
+extern void vm_map_remove_commpage( vm_map_t );
+#ifdef __i386__
+extern void vm_map_commpage32(vm_map_t);
+extern kern_return_t vm_map_apple_protected(
+	vm_map_t	map,
+	vm_map_offset_t	start,
+	vm_map_offset_t	end);
+extern void apple_protect_pager_bootstrap(void);
+extern memory_object_t apple_protect_pager_setup(vm_object_t backing_object);
+extern void apple_protect_pager_map(memory_object_t mem_obj);
+#endif	/* __i386__ */
+
 
 /*
  * bsd
@@ -186,6 +198,14 @@ extern memory_object_t vnode_pager_setup(
 	struct vnode *, memory_object_t);
 extern vm_object_offset_t vnode_pager_get_filesize(
 	struct vnode *);
+extern kern_return_t vnode_pager_get_pathname(
+	struct vnode	*vp,
+	char		*pathname,
+	vm_size_t	*length_p);
+extern kern_return_t vnode_pager_get_filename(
+	struct vnode	*vp,
+	char		**filename);
+	
 #endif /* _VNODE_PAGER_ */
 extern void vnode_pager_bootstrap(void);
 extern kern_return_t
@@ -201,6 +221,13 @@ extern kern_return_t vnode_pager_init(
 extern kern_return_t vnode_pager_get_object_size(
 	memory_object_t,
 	memory_object_offset_t *);
+extern kern_return_t vnode_pager_get_object_pathname(
+	memory_object_t	mem_obj,
+	char		*pathname,
+	vm_size_t	*length_p);
+extern kern_return_t vnode_pager_get_object_filename(
+	memory_object_t	mem_obj,
+	char		**filename);
 extern kern_return_t vnode_pager_data_request( 
 	memory_object_t, 
 	memory_object_offset_t, 
@@ -239,9 +266,6 @@ extern void vnode_pager_release_from_cache(
 extern void ubc_unmap(
 	struct vnode *vp);
 
-extern int	vnode_pager_workaround;
-extern int	device_pager_workaround;
-
 extern void   dp_memory_object_reference(memory_object_t);
 extern void   dp_memory_object_deallocate(memory_object_t);
 #ifndef _memory_object_server_
@@ -292,6 +316,8 @@ extern	kern_return_t   device_pager_data_request(memory_object_t,
 extern kern_return_t device_pager_data_return(memory_object_t,
 					      memory_object_offset_t,
 					      vm_size_t,
+					      memory_object_offset_t *,
+					      int *,
 					      boolean_t,
 					      boolean_t,
 					      int);
@@ -331,6 +357,8 @@ extern int macx_swapinfo(
 	vm_size_t		*pagesize_p,
 	boolean_t		*encrypted_p);
 
+extern void log_nx_failure(addr64_t vaddr, vm_prot_t prot);
+
 #endif	/* _VM_VM_PROTOS_H_ */
 
 #endif	/* XNU_KERNEL_PRIVATE */
diff --git a/osfmk/vm/vm_resident.c b/osfmk/vm/vm_resident.c
index ffcca3dbb..91b359f80 100644
--- a/osfmk/vm/vm_resident.c
+++ b/osfmk/vm/vm_resident.c
@@ -250,6 +250,12 @@ unsigned int	vm_page_gobble_count_warning = 0;
 unsigned int	vm_page_purgeable_count = 0; /* # of pages purgeable now */
 uint64_t	vm_page_purged_count = 0;    /* total count of purged pages */
 
+ppnum_t		vm_lopage_poolstart = 0;
+ppnum_t		vm_lopage_poolend = 0;
+int		vm_lopage_poolsize = 0;
+uint64_t	max_valid_dma_address = 0xffffffffffffffffULL;
+
+
 /*
  *	Several page replacement parameters are also
  *	shared with this module, so that page allocation
@@ -555,6 +561,8 @@ pmap_startup(
 	vm_page_t	pages;
 	ppnum_t		phys_page;
 	addr64_t	tmpaddr;
+	unsigned int	num_of_lopages = 0;
+	unsigned int	last_index;
 
 	/*
 	 *	We calculate how many page frames we will have
@@ -570,7 +578,6 @@ pmap_startup(
 	/*
 	 *	Initialize the page frames.
 	 */
-
 	for (i = 0, pages_initialized = 0; i < npages; i++) {
 		if (!pmap_next_page(&phys_page))
 			break;
@@ -580,21 +587,65 @@ pmap_startup(
 		pages_initialized++;
 	}
 
+	/*
+	 * Check if we want to initialize pages to a known value
+	 */
+	fill = 0;								/* Assume no fill */
+	if (PE_parse_boot_arg("fill", &fillval)) fill = 1;			/* Set fill */
+
+	/*
+	 * if vm_lopage_poolsize is non-zero, than we need to reserve
+	 * a pool of pages whose addresess are less than 4G... this pool
+	 * is used by drivers whose hardware can't DMA beyond 32 bits...
+	 *
+	 * note that I'm assuming that the page list is ascending and
+	 * ordered w/r to the physical address
+	 */
+	for (i = 0, num_of_lopages = vm_lopage_poolsize; num_of_lopages && i < pages_initialized; num_of_lopages--, i++) {
+	        vm_page_t m;
+
+		m = &pages[i];
+
+		if (m->phys_page >= (1 << (32 - PAGE_SHIFT)))
+		        panic("couldn't reserve the lopage pool: not enough lo pages\n");
+
+		if (m->phys_page < vm_lopage_poolend)
+		        panic("couldn't reserve the lopage pool: page list out of order\n");
+
+		vm_lopage_poolend = m->phys_page;
+
+		if (vm_lopage_poolstart == 0)
+		        vm_lopage_poolstart = m->phys_page;
+		else {
+		        if (m->phys_page < vm_lopage_poolstart)
+			        panic("couldn't reserve the lopage pool: page list out of order\n");
+		}
+
+		if (fill)
+		        fillPage(m->phys_page, fillval);		/* Fill the page with a know value if requested at boot */			
+
+		vm_page_release(m);
+	} 
+	last_index = i;
+
+	// -debug code remove
+	if (2 == vm_himemory_mode) {
+		// free low -> high so high is preferred
+		for (i = last_index + 1; i <= pages_initialized; i++) {
+			if(fill) fillPage(pages[i - 1].phys_page, fillval);		/* Fill the page with a know value if requested at boot */			
+			vm_page_release(&pages[i - 1]);
+		}
+	}
+	else
+	// debug code remove-
+
 	/*
 	 * Release pages in reverse order so that physical pages
 	 * initially get allocated in ascending addresses. This keeps
 	 * the devices (which must address physical memory) happy if
 	 * they require several consecutive pages.
 	 */
-
-/*
- *		Check if we want to initialize pages to a known value
- */
-	
-	fill = 0;													/* Assume no fill */
-	if (PE_parse_boot_arg("fill", &fillval)) fill = 1;			/* Set fill */
-	
-	for (i = pages_initialized; i > 0; i--) {
+	for (i = pages_initialized; i > last_index; i--) {
 		if(fill) fillPage(pages[i - 1].phys_page, fillval);		/* Fill the page with a know value if requested at boot */			
 		vm_page_release(&pages[i - 1]);
 	}
@@ -812,7 +863,8 @@ vm_page_replace(
 	register vm_object_t		object,
 	register vm_object_offset_t	offset)
 {
-	register vm_page_bucket_t *bucket;
+	vm_page_bucket_t *bucket;
+	vm_page_t	 found_m = VM_PAGE_NULL;
 
 	VM_PAGE_CHECK(mem);
 #if DEBUG
@@ -838,46 +890,60 @@ vm_page_replace(
 
 	bucket = &vm_page_buckets[vm_page_hash(object, offset)];
 	simple_lock(&vm_page_bucket_lock);
+
 	if (bucket->pages) {
 		vm_page_t *mp = &bucket->pages;
 		register vm_page_t m = *mp;
+
 		do {
 			if (m->object == object && m->offset == offset) {
 				/*
-				 * Remove page from bucket and from object,
-				 * and return it to the free list.
+				 * Remove old page from hash list
 				 */
 				*mp = m->next;
-				VM_PAGE_REMOVE(m);
-				m->tabled = FALSE;
-				m->object = VM_OBJECT_NULL;
-				m->offset = (vm_object_offset_t) -1;
-				object->resident_page_count--;
-
-				if (object->purgable == VM_OBJECT_PURGABLE_VOLATILE ||
-				    object->purgable == VM_OBJECT_PURGABLE_EMPTY) {
-					assert(vm_page_purgeable_count > 0);
-					vm_page_purgeable_count--;
-				}
-					
-				/*
-				 * Return page to the free list.
-				 * Note the page is not tabled now, so this
-				 * won't self-deadlock on the bucket lock.
-				 */
 
-				vm_page_free(m);
+				found_m = m;
 				break;
 			}
 			mp = &m->next;
 		} while ((m = *mp));
+
 		mem->next = bucket->pages;
 	} else {
 		mem->next = VM_PAGE_NULL;
 	}
+	/*
+	 * insert new page at head of hash list
+	 */
 	bucket->pages = mem;
+
 	simple_unlock(&vm_page_bucket_lock);
 
+	if (found_m) {
+	        /*
+		 * there was already a page at the specified
+		 * offset for this object... remove it from
+		 * the object and free it back to the free list
+		 */
+		VM_PAGE_REMOVE(found_m);
+		found_m->tabled = FALSE;
+
+		found_m->object = VM_OBJECT_NULL;
+		found_m->offset = (vm_object_offset_t) -1;
+		object->resident_page_count--;
+
+		if (object->purgable == VM_OBJECT_PURGABLE_VOLATILE ||
+		    object->purgable == VM_OBJECT_PURGABLE_EMPTY) {
+		        assert(vm_page_purgeable_count > 0);
+			vm_page_purgeable_count--;
+		}
+					
+		/*
+		 * Return page to the free list.
+		 * Note the page is not tabled now
+		 */
+		vm_page_free(found_m);
+	}
 	/*
 	 *	Now link into the object's list of backed pages.
 	 */
@@ -1040,7 +1106,19 @@ vm_page_lookup(
 
 	bucket = &vm_page_buckets[vm_page_hash(object, offset)];
 
+        /*
+         * since we hold the object lock, we are guaranteed that no
+         * new pages can be inserted into this object... this in turn
+         * guarantess that the page we're looking for can't exist
+         * if the bucket it hashes to is currently NULL even when looked
+         * at outside the scope of the hash bucket lock... this is a
+         * really cheap optimiztion to avoid taking the lock
+         */
+        if (bucket->pages == VM_PAGE_NULL) {
+                return (VM_PAGE_NULL);
+        }
 	simple_lock(&vm_page_bucket_lock);
+
 	for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem->next) {
 		VM_PAGE_CHECK(mem);
 		if ((mem->object == object) && (mem->offset == offset))
@@ -1342,6 +1420,55 @@ vm_pool_low(void)
 	return( vm_page_free_count < vm_page_free_reserved );
 }
 
+
+
+/*
+ * this is an interface to support bring-up of drivers
+ * on platforms with physical memory > 4G...
+ */
+int		vm_himemory_mode = 0;
+
+
+/*
+ * this interface exists to support hardware controllers
+ * incapable of generating DMAs with more than 32 bits
+ * of address on platforms with physical memory > 4G...
+ */
+unsigned int	vm_lopage_free_count = 0;
+unsigned int	vm_lopage_max_count = 0;
+vm_page_t	vm_lopage_queue_free = VM_PAGE_NULL;
+
+vm_page_t
+vm_page_grablo(void)
+{
+	register vm_page_t	mem;
+	unsigned int vm_lopage_alloc_count;
+
+	if (vm_lopage_poolsize == 0)
+	        return (vm_page_grab());
+
+	mutex_lock(&vm_page_queue_free_lock);
+
+	if ((mem = vm_lopage_queue_free) != VM_PAGE_NULL) {
+
+	        vm_lopage_queue_free = (vm_page_t) mem->pageq.next;
+		mem->pageq.next = NULL;
+		mem->pageq.prev = NULL;
+		mem->free = FALSE;
+		mem->no_isync = TRUE;
+
+		vm_lopage_free_count--;
+		vm_lopage_alloc_count = (vm_lopage_poolend - vm_lopage_poolstart) - vm_lopage_free_count;
+		if (vm_lopage_alloc_count > vm_lopage_max_count)
+			vm_lopage_max_count = vm_lopage_alloc_count;
+	}
+	mutex_unlock(&vm_page_queue_free_lock);
+
+	return (mem);
+}
+
+
+
 /*
  *	vm_page_grab:
  *
@@ -1467,36 +1594,46 @@ vm_page_release(
 	assert(mem->object == VM_OBJECT_NULL);
 	assert(mem->pageq.next == NULL &&
 	       mem->pageq.prev == NULL);
-	mem->pageq.next = (queue_entry_t) vm_page_queue_free;
-	vm_page_queue_free = mem;
-	vm_page_free_count++;
 
-	/*
-	 *	Check if we should wake up someone waiting for page.
-	 *	But don't bother waking them unless they can allocate.
-	 *
-	 *	We wakeup only one thread, to prevent starvation.
-	 *	Because the scheduling system handles wait queues FIFO,
-	 *	if we wakeup all waiting threads, one greedy thread
-	 *	can starve multiple niceguy threads.  When the threads
-	 *	all wakeup, the greedy threads runs first, grabs the page,
-	 *	and waits for another page.  It will be the first to run
-	 *	when the next page is freed.
-	 *
-	 *	However, there is a slight danger here.
-	 *	The thread we wake might not use the free page.
-	 *	Then the other threads could wait indefinitely
-	 *	while the page goes unused.  To forestall this,
-	 *	the pageout daemon will keep making free pages
-	 *	as long as vm_page_free_wanted is non-zero.
-	 */
+	if (mem->phys_page <= vm_lopage_poolend && mem->phys_page >= vm_lopage_poolstart) {
+	        /*
+		 * this exists to support hardware controllers
+		 * incapable of generating DMAs with more than 32 bits
+		 * of address on platforms with physical memory > 4G...
+		 */
+	        mem->pageq.next = (queue_entry_t) vm_lopage_queue_free;
+		vm_lopage_queue_free = mem;
+		vm_lopage_free_count++;
+	} else {	  
+	        mem->pageq.next = (queue_entry_t) vm_page_queue_free;
+		vm_page_queue_free = mem;
+		vm_page_free_count++;
+		/*
+		 *	Check if we should wake up someone waiting for page.
+		 *	But don't bother waking them unless they can allocate.
+		 *
+		 *	We wakeup only one thread, to prevent starvation.
+		 *	Because the scheduling system handles wait queues FIFO,
+		 *	if we wakeup all waiting threads, one greedy thread
+		 *	can starve multiple niceguy threads.  When the threads
+		 *	all wakeup, the greedy threads runs first, grabs the page,
+		 *	and waits for another page.  It will be the first to run
+		 *	when the next page is freed.
+		 *
+		 *	However, there is a slight danger here.
+		 *	The thread we wake might not use the free page.
+		 *	Then the other threads could wait indefinitely
+		 *	while the page goes unused.  To forestall this,
+		 *	the pageout daemon will keep making free pages
+		 *	as long as vm_page_free_wanted is non-zero.
+		 */
 
-	if ((vm_page_free_wanted > 0) &&
-	    (vm_page_free_count >= vm_page_free_reserved)) {
-		vm_page_free_wanted--;
-		thread_wakeup_one((event_t) &vm_page_free_count);
+		if ((vm_page_free_wanted > 0) &&
+		    (vm_page_free_count >= vm_page_free_reserved)) {
+		        vm_page_free_wanted--;
+			thread_wakeup_one((event_t) &vm_page_free_count);
+		}
 	}
-
 	mutex_unlock(&vm_page_queue_free_lock);
 }
 
@@ -1574,6 +1711,27 @@ vm_page_alloc(
 	return(mem);
 }
 
+
+vm_page_t
+vm_page_alloclo(
+	vm_object_t		object,
+	vm_object_offset_t	offset)
+{
+	register vm_page_t	mem;
+
+#if DEBUG
+	_mutex_assert(&object->Lock, MA_OWNED);
+#endif
+	mem = vm_page_grablo();
+	if (mem == VM_PAGE_NULL)
+		return VM_PAGE_NULL;
+
+	vm_page_insert(mem, object, offset);
+
+	return(mem);
+}
+
+
 counter(unsigned int c_laundry_pages_freed = 0;)
 
 int vm_pagein_cluster_unused = 0;
diff --git a/osfmk/vm/vm_shared_memory_server.c b/osfmk/vm/vm_shared_memory_server.c
index dd5323fc7..81906665c 100644
--- a/osfmk/vm/vm_shared_memory_server.c
+++ b/osfmk/vm/vm_shared_memory_server.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -53,11 +53,14 @@
 #include <vm/vm_kern.h>
 #include <vm/vm_map.h>
 #include <vm/vm_page.h>
+#include <vm/vm_protos.h>
 
 #include <mach/mach_vm.h>
 #include <mach/shared_memory_server.h>
 #include <vm/vm_shared_memory_server.h>
 
+int shared_region_trace_level = SHARED_REGION_TRACE_ERROR;
+
 #if DEBUG
 int lsf_debug = 0;
 int lsf_alloc_debug = 0;
@@ -115,6 +118,7 @@ lsf_hash_lookup(
 
 static load_struct_t *
 lsf_hash_delete(
+	load_struct_t			*target_entry, /* optional */
         void            		*file_object,
 	vm_offset_t			base_offset,
 	shared_region_task_mappings_t	sm_info);
@@ -149,6 +153,7 @@ lsf_unload(
 
 static void
 lsf_deallocate(
+	load_struct_t			*target_entry,	/* optional */
         void     			*file_object,
 	vm_offset_t			base_offset,
 	shared_region_task_mappings_t	sm_info,
@@ -159,11 +164,6 @@ lsf_deallocate(
 		((((natural_t)file_object) & 0xffffff) % size)
 
 /* Implementation */
-vm_offset_t		shared_file_text_region;
-vm_offset_t		shared_file_data_region;
-
-ipc_port_t		shared_text_region_handle;
-ipc_port_t		shared_data_region_handle;
 vm_offset_t		shared_file_mapping_array = 0;
 
 shared_region_mapping_t default_environment_shared_regions = NULL;
@@ -187,7 +187,8 @@ ipc_port_t		com_region_handle32 = NULL;
 ipc_port_t		com_region_handle64 = NULL;
 vm_map_t		com_region_map32 = NULL;
 vm_map_t		com_region_map64 = NULL;
-vm_size_t		com_region_size = _COMM_PAGE_AREA_LENGTH;
+vm_size_t		com_region_size32 = _COMM_PAGE32_AREA_LENGTH;
+vm_size_t		com_region_size64 = _COMM_PAGE64_AREA_LENGTH;
 shared_region_mapping_t	com_mapping_resource = NULL;
 
 
@@ -215,12 +216,33 @@ vm_set_shared_region(
 	task_t	task,
 	shared_region_mapping_t	shared_region)
 {
+	shared_region_mapping_t old_region;
+
 	SHARED_REGION_DEBUG(("vm_set_shared_region(task=%p, "
-			     "shared_region=%p)\n",
-			     task, shared_region));
+			     "shared_region=%p[%x,%x,%x])\n",
+			     task, shared_region,
+			     shared_region ? shared_region->fs_base : 0,
+			     shared_region ? shared_region->system : 0,
+			     shared_region ? shared_region->flags : 0));
 	if (shared_region) {
 		assert(shared_region->ref_count > 0);
 	}
+
+	old_region = task->system_shared_region;
+	SHARED_REGION_TRACE(
+		SHARED_REGION_TRACE_INFO,
+		("shared_region: %p set_region(task=%p)"
+		 "old=%p[%x,%x,%x], new=%p[%x,%x,%x]\n",
+		 current_thread(), task,
+		 old_region,
+		 old_region ? old_region->fs_base : 0,
+		 old_region ? old_region->system : 0,
+		 old_region ? old_region->flags : 0,
+		 shared_region,
+		 shared_region ? shared_region->fs_base : 0,
+		 shared_region ? shared_region->system : 0,
+		 shared_region ? shared_region->flags : 0));
+
 	task->system_shared_region = shared_region;
 	return KERN_SUCCESS;
 }
@@ -293,7 +315,9 @@ shared_region_mapping_create(
 	vm_offset_t		client_base,
 	shared_region_mapping_t	*shared_region,
 	vm_offset_t		alt_base,
-	vm_offset_t		alt_next)
+	vm_offset_t		alt_next,
+	int			fs_base,
+	int			system)
 {
 	SHARED_REGION_DEBUG(("shared_region_mapping_create()\n"));
 	*shared_region = (shared_region_mapping_t) 
@@ -306,8 +330,8 @@ shared_region_mapping_create(
 	shared_region_mapping_lock_init((*shared_region));
 	(*shared_region)->text_region = text_region;
 	(*shared_region)->text_size = text_size;
-	(*shared_region)->fs_base = ENV_DEFAULT_ROOT;
-	(*shared_region)->system = cpu_type();
+	(*shared_region)->fs_base = fs_base;
+	(*shared_region)->system = system;
 	(*shared_region)->data_region = data_region;
 	(*shared_region)->data_size = data_size;
 	(*shared_region)->region_mappings = region_mappings;
@@ -362,6 +386,8 @@ shared_region_mapping_info(
 	*next = shared_region->next;
 
 	shared_region_mapping_unlock(shared_region);
+	
+	return KERN_SUCCESS;
 }
 
 kern_return_t
@@ -387,7 +413,7 @@ shared_region_mapping_dealloc_lock(
 {
 	struct shared_region_task_mappings sm_info;
 	shared_region_mapping_t next = NULL;
-	int ref_count;
+	unsigned int ref_count;
 
 	SHARED_REGION_DEBUG(("shared_region_mapping_dealloc_lock"
 			     "(shared_region=%p,%d,%d) ref_count=%d\n",
@@ -523,7 +549,7 @@ shared_region_object_create(
 
 	/* Create a named object based on a submap of specified size */
 
-	new_map = vm_map_create(pmap_create(0), 0, size, TRUE);
+	new_map = vm_map_create(pmap_create(0, FALSE), 0, size, TRUE);
 	user_entry->backing.map = new_map;
 	user_entry->internal = TRUE;
 	user_entry->is_sub_map = TRUE;
@@ -544,7 +570,9 @@ shared_region_object_create(
 /* relevant as the system default flag is not set */
 kern_return_t
 shared_file_create_system_region(
-		shared_region_mapping_t	*shared_region)
+	shared_region_mapping_t	*shared_region,
+	int			fs_base,
+	int			system)
 {
 	ipc_port_t		text_handle;
 	ipc_port_t		data_handle;
@@ -566,10 +594,15 @@ shared_file_create_system_region(
 				     kret));
 		return kret;
 	}
-	kret = shared_region_mapping_create(text_handle,
-			text_size, data_handle, data_size, mapping_array,
-			GLOBAL_SHARED_TEXT_SEGMENT, shared_region, 
-			SHARED_ALTERNATE_LOAD_BASE, SHARED_ALTERNATE_LOAD_BASE);
+	kret = shared_region_mapping_create(text_handle, text_size,
+					    data_handle, data_size,
+					    mapping_array,
+					    GLOBAL_SHARED_TEXT_SEGMENT,
+					    shared_region, 
+					    SHARED_ALTERNATE_LOAD_BASE,
+					    SHARED_ALTERNATE_LOAD_BASE,
+					    fs_base,
+					    system);
 	if(kret) {
 		SHARED_REGION_DEBUG(("shared_file_create_system_region: "
 				     "shared_region_mapping_create failed "
@@ -875,14 +908,14 @@ shared_com_boot_time_init(void)
 
 	/* create com page regions, 1 each for 32 and 64-bit code  */
 	if((kret = shared_region_object_create(
-			com_region_size, 
+			com_region_size32, 
 			&com_region_handle32))) {
 		panic("shared_com_boot_time_init: "
 				"unable to create 32-bit comm page\n");
 		return;
 	}
 	if((kret = shared_region_object_create(
-			com_region_size, 
+			com_region_size64, 
 			&com_region_handle64))) {
 		panic("shared_com_boot_time_init: "
 				"unable to create 64-bit comm page\n");
@@ -898,9 +931,12 @@ shared_com_boot_time_init(void)
 	/* wrap the com region in its own shared file mapping structure */
 	/* 64-bit todo: call "shared_region_mapping_create" on com_region_handle64 */
 	kret = shared_region_mapping_create(com_region_handle32,
-		com_region_size, NULL, 0, 0,
-		_COMM_PAGE_BASE_ADDRESS, &com_mapping_resource,
-		0, 0);
+					    com_region_size32,
+					    NULL, 0, 0,
+					    _COMM_PAGE_BASE_ADDRESS,
+					    &com_mapping_resource,
+					    0, 0,
+					    ENV_DEFAULT_ROOT, cpu_type());
 	if (kret) {
 	  panic("shared_region_mapping_create failed for commpage");
 	}
@@ -911,6 +947,8 @@ shared_file_boot_time_init(
 		unsigned int fs_base, 
 		unsigned int system)
 {
+	mach_port_t		text_region_handle;
+	mach_port_t		data_region_handle;
 	long			text_region_size;
 	long			data_region_size;
 	shared_region_mapping_t	new_system_region;
@@ -921,24 +959,23 @@ shared_file_boot_time_init(
 			     fs_base, system));
 	text_region_size = 0x10000000;
 	data_region_size = 0x10000000;
-	shared_file_init(&shared_text_region_handle,
+	shared_file_init(&text_region_handle,
 			 text_region_size,
-			 &shared_data_region_handle,
+			 &data_region_handle,
 			 data_region_size,
 			 &shared_file_mapping_array);
 	
-	shared_region_mapping_create(shared_text_region_handle,
+	shared_region_mapping_create(text_region_handle,
 				     text_region_size,
-				     shared_data_region_handle,
+				     data_region_handle,
 				     data_region_size,
 				     shared_file_mapping_array,
 				     GLOBAL_SHARED_TEXT_SEGMENT,
 				     &new_system_region,
 				     SHARED_ALTERNATE_LOAD_BASE,
-				     SHARED_ALTERNATE_LOAD_BASE);
+				     SHARED_ALTERNATE_LOAD_BASE,
+				     fs_base, system);
 
-	new_system_region->fs_base = fs_base;
-	new_system_region->system = system;
 	new_system_region->flags = SHARED_REGION_SYSTEM;
 
 	/* grab an extra reference for the caller */
@@ -976,8 +1013,7 @@ shared_file_init(
 	vm_offset_t	*file_mapping_array)
 {
 	shared_file_info_t	*sf_head;
-	vm_offset_t		table_mapping_address;
-	int			data_table_size;
+	vm_size_t		data_table_size;
 	int			hash_size;
 	kern_return_t		kret;
 
@@ -1005,14 +1041,13 @@ shared_file_init(
 
 	data_table_size = data_region_size >> 9;
 	hash_size = data_region_size >> 14;
-	table_mapping_address = data_region_size - data_table_size;
 
 	if(shared_file_mapping_array == 0) {
 		vm_map_address_t map_addr;
 		buf_object = vm_object_allocate(data_table_size);
 
 		if(vm_map_find_space(kernel_map, &map_addr,
-				     data_table_size, 0, &entry)
+				     data_table_size, 0, 0, &entry)
 		   != KERN_SUCCESS) {
 			panic("shared_file_init: no space");
 		}
@@ -1081,16 +1116,8 @@ shared_file_init(
 		*file_mapping_array = shared_file_mapping_array;
 	}
 
-	kret = vm_map(((vm_named_entry_t)
-		       (*data_region_handle)->ip_kobject)->backing.map,
-		      &table_mapping_address,
-		      data_table_size, 0,
-		      SHARED_LIB_ALIAS | VM_FLAGS_FIXED, 
-		      sfma_handle, 0, FALSE, 
-		      VM_PROT_READ, VM_PROT_READ, VM_INHERIT_NONE);
-
 	SHARED_REGION_DEBUG(("shared_file_init() done\n"));
-	return kret;
+	return KERN_SUCCESS;
 }
 
 static kern_return_t
@@ -1131,8 +1158,10 @@ shared_file_header_init(
 		if (vm_map_wire(kernel_map, hash_cram_address,
 				hash_cram_address + cram_size, 
 				VM_PROT_DEFAULT, FALSE) != KERN_SUCCESS) {
-			printf("shared_file_header_init: "
-			       "No memory for data table\n");
+			SHARED_REGION_TRACE(
+				SHARED_REGION_TRACE_ERROR,
+				("shared_region: shared_file_header_init: "
+				 "No memory for data table\n"));
 			return KERN_NO_SPACE;
 		}
 		allocable_hash_pages -= cram_pages;
@@ -1146,6 +1175,175 @@ shared_file_header_init(
 }
 
 
+extern void shared_region_dump_file_entry(
+	int		trace_level,
+	load_struct_t	*entry);	/* forward */
+
+void shared_region_dump_file_entry(
+	int		trace_level,
+	load_struct_t	*entry)
+{
+	int			i;
+	loaded_mapping_t	*mapping;
+
+	if (trace_level > shared_region_trace_level) {
+		return;
+	}
+	printf("shared region: %p: "
+	       "file_entry %p  base_address=0x%x  file_offset=0x%x  "
+	       "%d mappings\n",
+	       current_thread(), entry,
+	       entry->base_address, entry->file_offset, entry->mapping_cnt);
+	mapping = entry->mappings;
+	for (i = 0; i < entry->mapping_cnt; i++) {
+		printf("shared region: %p:\t#%d: "
+		       "offset=0x%x size=0x%x file_offset=0x%x prot=%d\n",
+		       current_thread(),
+		       i,
+		       mapping->mapping_offset,
+		       mapping->size,
+		       mapping->file_offset,
+		       mapping->protection);
+		mapping = mapping->next;
+	}
+}
+
+extern void shared_region_dump_mappings(
+	int				trace_level,
+	struct shared_file_mapping_np	*mappings,
+	int				map_cnt,
+	mach_vm_offset_t		base_offset);	/* forward */
+
+void shared_region_dump_mappings(
+	int				trace_level,
+	struct shared_file_mapping_np	*mappings,
+	int				map_cnt,
+	mach_vm_offset_t		base_offset)
+{
+	int	i;
+
+	if (trace_level > shared_region_trace_level) {
+		return;
+	}
+
+	printf("shared region: %p: %d mappings  base_offset=0x%llx\n",
+	       current_thread(), map_cnt, (uint64_t) base_offset);
+	for (i = 0; i < map_cnt; i++) {
+		printf("shared region: %p:\t#%d: "
+		       "addr=0x%llx, size=0x%llx, file_offset=0x%llx, "
+		       "prot=(%d,%d)\n",
+		       current_thread(),
+		       i,
+		       (uint64_t) mappings[i].sfm_address,
+		       (uint64_t) mappings[i].sfm_size,
+		       (uint64_t) mappings[i].sfm_file_offset,
+		       mappings[i].sfm_max_prot,
+		       mappings[i].sfm_init_prot);
+	}
+}
+
+extern void shared_region_dump_conflict_info(
+	int		trace_level,
+	vm_map_t	map,
+	vm_map_offset_t	offset,
+	vm_map_size_t	size);	/* forward */
+
+void
+shared_region_dump_conflict_info(
+	int		trace_level,
+	vm_map_t	map,
+	vm_map_offset_t	offset,
+	vm_map_size_t	size)
+{
+	vm_map_entry_t	entry;
+	vm_object_t	object;
+	memory_object_t	mem_object;
+	kern_return_t	kr;
+	char		*filename;
+
+	if (trace_level > shared_region_trace_level) {
+		return;
+	}
+
+	object = VM_OBJECT_NULL;
+
+	vm_map_lock_read(map);
+	if (!vm_map_lookup_entry(map, offset, &entry)) {
+		entry = entry->vme_next;
+	}
+	
+	if (entry != vm_map_to_entry(map)) {
+		if (entry->is_sub_map) {
+			printf("shared region: %p: conflict with submap "
+			       "at 0x%llx size 0x%llx !?\n",
+			       current_thread(),
+			       (uint64_t) offset,
+			       (uint64_t) size);
+			goto done;
+		}
+
+		object = entry->object.vm_object;
+		if (object == VM_OBJECT_NULL) {
+			printf("shared region: %p: conflict with NULL object "
+			       "at 0x%llx size 0x%llx !?\n",
+			       current_thread(),
+			       (uint64_t) offset,
+			       (uint64_t) size);
+			object = VM_OBJECT_NULL;
+			goto done;
+		}
+
+		vm_object_lock(object);
+		while (object->shadow != VM_OBJECT_NULL) {
+			vm_object_t	shadow;
+
+			shadow = object->shadow;
+			vm_object_lock(shadow);
+			vm_object_unlock(object);
+			object = shadow;
+		}
+
+		if (object->internal) {
+			printf("shared region: %p: conflict with anonymous "
+			       "at 0x%llx size 0x%llx\n",
+			       current_thread(),
+			       (uint64_t) offset,
+			       (uint64_t) size);
+			goto done;
+		}
+		if (! object->pager_ready) {
+			printf("shared region: %p: conflict with uninitialized "
+			       "at 0x%llx size 0x%llx\n",
+			       current_thread(),
+			       (uint64_t) offset,
+			       (uint64_t) size);
+			goto done;
+		}
+
+		mem_object = object->pager;
+
+		/*
+		 * XXX FBDP: "!internal" doesn't mean it's a vnode pager...
+		 */
+		kr = vnode_pager_get_object_filename(mem_object,
+						     &filename);
+		if (kr != KERN_SUCCESS) {
+			filename = NULL;
+		}
+		printf("shared region: %p: conflict with '%s' "
+		       "at 0x%llx size 0x%llx\n",
+		       current_thread(),
+		       filename ? filename : "<unknown>",
+		       (uint64_t) offset,
+		       (uint64_t) size);
+	}
+done:
+	if (object != VM_OBJECT_NULL) {
+		vm_object_unlock(object);
+	}
+	vm_map_unlock_read(map);
+}
+
 /*
  * map_shared_file:
  *
@@ -1183,6 +1381,11 @@ map_shared_file(
 	if(shared_file_header->hash_init == FALSE) {
 		ret = shared_file_header_init(shared_file_header);
 		if (ret != KERN_SUCCESS) {
+			SHARED_REGION_TRACE(
+				SHARED_REGION_TRACE_ERROR,
+				("shared_region: %p: map_shared_file: "
+				 "shared_file_header_init() failed kr=0x%x\n",
+				 current_thread(), ret));
 			mutex_unlock(&shared_file_header->lock);
 			return KERN_NO_SPACE;
 		}
@@ -1208,6 +1411,19 @@ map_shared_file(
 		file_mapping = file_entry->mappings;
 		while(file_mapping != NULL) {
 			if(i>=map_cnt) {
+				SHARED_REGION_TRACE(
+					SHARED_REGION_TRACE_CONFLICT,
+					("shared_region: %p: map_shared_file: "
+					 "already mapped with "
+					 "more than %d mappings\n",
+					 current_thread(), map_cnt));
+				shared_region_dump_file_entry(
+					SHARED_REGION_TRACE_INFO,
+					file_entry);
+				shared_region_dump_mappings(
+					SHARED_REGION_TRACE_INFO,
+					mappings, map_cnt, base_offset);
+
 				mutex_unlock(&shared_file_header->lock);
 				return KERN_INVALID_ARGUMENT;
 			}
@@ -1217,12 +1433,37 @@ map_shared_file(
 			   mappings[i].sfm_size != file_mapping->size ||	
 			   mappings[i].sfm_file_offset != file_mapping->file_offset ||	
 			   mappings[i].sfm_init_prot != file_mapping->protection) {
+				SHARED_REGION_TRACE(
+					SHARED_REGION_TRACE_CONFLICT,
+					("shared_region: %p: "
+					 "mapping #%d differs\n",
+					 current_thread(), i));
+				shared_region_dump_file_entry(
+					SHARED_REGION_TRACE_INFO,
+					file_entry);
+				shared_region_dump_mappings(
+					SHARED_REGION_TRACE_INFO,
+					mappings, map_cnt, base_offset);
+
 				break;
 			}
 			file_mapping = file_mapping->next;
 			i++;
 		}
 		if(i!=map_cnt) {
+			SHARED_REGION_TRACE(
+				SHARED_REGION_TRACE_CONFLICT,
+				("shared_region: %p: map_shared_file: "
+				 "already mapped with "
+				 "%d mappings instead of %d\n",
+				 current_thread(), i, map_cnt));
+			shared_region_dump_file_entry(
+				SHARED_REGION_TRACE_INFO,
+				file_entry);
+			shared_region_dump_mappings(
+				SHARED_REGION_TRACE_INFO,
+				mappings, map_cnt, base_offset);
+
 			mutex_unlock(&shared_file_header->lock);
 			return KERN_INVALID_ARGUMENT;
 		}
@@ -1249,6 +1490,13 @@ map_shared_file(
 				 * requested address too ?
 				 */
 				ret = KERN_FAILURE;
+				SHARED_REGION_TRACE(
+					SHARED_REGION_TRACE_CONFLICT,
+					("shared_region: %p: "
+					 "map_shared_file: already mapped, "
+					 "would need to slide 0x%llx\n",
+					 current_thread(),
+					 slide));
 			} else {
 				/*
 				 * The file is already mapped at the correct
@@ -1473,7 +1721,7 @@ static load_struct_t  *
 lsf_hash_lookup(
 	queue_head_t			*hash_table,
 	void				*file_object,
-  vm_offset_t                           recognizableOffset,
+	vm_offset_t			recognizableOffset,
 	int				size,
 	boolean_t			regular,
 	boolean_t			alternate,
@@ -1617,6 +1865,7 @@ lsf_remove_regions_mappings(
 
 static load_struct_t *
 lsf_hash_delete(
+	load_struct_t	*target_entry,	/* optional:  NULL if not relevant */
 	void		*file_object,
 	vm_offset_t	base_offset,
 	shared_region_task_mappings_t	sm_info)
@@ -1625,8 +1874,8 @@ lsf_hash_delete(
 	shared_file_info_t	*shared_file_header;
 	load_struct_t		*entry;
 
-	LSF_DEBUG(("lsf_hash_delete(file=%p,base=0x%x,sm_info=%p)\n",
-		   file_object, base_offset, sm_info));
+	LSF_DEBUG(("lsf_hash_delete(target=%p,file=%p,base=0x%x,sm_info=%p)\n",
+		   target_entry, file_object, base_offset, sm_info));
 
 	shared_file_header = (shared_file_info_t *)sm_info->region_mappings;
 
@@ -1638,8 +1887,10 @@ lsf_hash_delete(
 		entry = (load_struct_t *)queue_next(&entry->links)) {
 		if((!(sm_info->self)) || ((shared_region_mapping_t)
 				sm_info->self == entry->regions_instance)) {
-			if ((entry->file_object == (int) file_object)  &&
-				(entry->base_address == base_offset)) {
+			if ((target_entry == NULL ||
+			     entry == target_entry) &&
+			    (entry->file_object == (int) file_object)  &&
+			    (entry->base_address == base_offset)) {
 				queue_remove(bucket, entry, 
 						load_struct_ptr_t, links);
 				LSF_DEBUG(("lsf_hash_delete: found it\n"));
@@ -1800,6 +2051,10 @@ start_over:
 	wiggle_room = base_offset;
 
 	for (i = (signed) map_cnt - 1; i >= 0; i--) {
+		if (mappings[i].sfm_size == 0) {
+			/* nothing to map here... */
+			continue;
+		}
 		if (mappings[i].sfm_init_prot & VM_PROT_COW) {
 			/* copy-on-write mappings are in the data submap */
 			map = data_map;
@@ -1969,6 +2224,7 @@ lsf_map(
 	kern_return_t		kr;
 	int			i;
 	mach_vm_offset_t	original_base_offset;
+	mach_vm_size_t		total_size;
 
 	/* get the VM object from the file's memory object handle */
 	file_object = memory_object_control_to_vm_object(file_control);
@@ -1991,7 +2247,11 @@ restart_after_slide:
 		   map_cnt, file_object,
 		   sm_info, entry));
 	if (entry == NULL) {
-		printf("lsf_map: unable to allocate memory\n");
+		SHARED_REGION_TRACE(
+			SHARED_REGION_TRACE_ERROR,
+			("shared_region: %p: "
+			 "lsf_map: unable to allocate entry\n",
+			 current_thread()));
 		return KERN_NO_SPACE;
 	}
 	shared_file_available_hash_ele--;
@@ -2011,7 +2271,7 @@ restart_after_slide:
 	tptr = &(entry->mappings);
 
 	entry->base_address = base_offset;
-
+	total_size = 0;
 
 	/* establish each requested mapping */
 	for (i = 0; i < map_cnt; i++) {
@@ -2026,8 +2286,20 @@ restart_after_slide:
 			    (((mappings[i].sfm_address + base_offset +
 			       mappings[i].sfm_size - 1)
 			      & GLOBAL_SHARED_SEGMENT_MASK) != 0x10000000)) {
-				lsf_unload(file_object, 
-					entry->base_address, sm_info);
+				SHARED_REGION_TRACE(
+					SHARED_REGION_TRACE_ERROR,
+					("shared_region: %p: lsf_map: "
+					 "RW mapping #%d not in segment",
+					 current_thread(), i));
+				shared_region_dump_mappings(
+					SHARED_REGION_TRACE_ERROR,
+					mappings, map_cnt, base_offset);
+
+				lsf_deallocate(entry,
+					       file_object, 
+					       entry->base_address,
+					       sm_info,
+					       TRUE);
 				return KERN_INVALID_ARGUMENT;
 			}
 		} else {
@@ -2038,15 +2310,41 @@ restart_after_slide:
 			    ((mappings[i].sfm_address + base_offset +
 			      mappings[i].sfm_size - 1)
 			     & GLOBAL_SHARED_SEGMENT_MASK)) {
-				lsf_unload(file_object, 
-					entry->base_address, sm_info);
+				SHARED_REGION_TRACE(
+					SHARED_REGION_TRACE_ERROR,
+					("shared_region: %p: lsf_map: "
+					 "RO mapping #%d not in segment",
+					 current_thread(), i));
+				shared_region_dump_mappings(
+					SHARED_REGION_TRACE_ERROR,
+					mappings, map_cnt, base_offset);
+
+				lsf_deallocate(entry,
+					       file_object, 
+					       entry->base_address,
+					       sm_info,
+					       TRUE);
 				return KERN_INVALID_ARGUMENT;
 			}
 		}
 		if (!(mappings[i].sfm_init_prot & VM_PROT_ZF) &&
 		    ((mappings[i].sfm_file_offset + mappings[i].sfm_size) >
 		     (file_size))) {
-			lsf_unload(file_object, entry->base_address, sm_info);
+			SHARED_REGION_TRACE(
+				SHARED_REGION_TRACE_ERROR,
+				("shared_region: %p: lsf_map: "
+				 "ZF mapping #%d beyond EOF",
+				 current_thread(), i));
+			shared_region_dump_mappings(SHARED_REGION_TRACE_ERROR,
+						    mappings, map_cnt,
+						    base_offset);
+
+
+			lsf_deallocate(entry,
+				       file_object,
+				       entry->base_address,
+				       sm_info,
+				       TRUE);
 			return KERN_INVALID_ARGUMENT;
 		}
 		target_address = entry->base_address +
@@ -2058,7 +2356,13 @@ restart_after_slide:
 		}
 		region_entry = (vm_named_entry_t) region_handle->ip_kobject;
 
-		if (mach_vm_map(region_entry->backing.map,
+		total_size += mappings[i].sfm_size;
+		if (mappings[i].sfm_size == 0) {
+			/* nothing to map... */
+			kr = KERN_SUCCESS;
+		} else {
+			kr = mach_vm_map(
+				region_entry->backing.map,
 				&target_address,
 				vm_map_round_page(mappings[i].sfm_size),
 				0,
@@ -2070,8 +2374,18 @@ restart_after_slide:
 				 (VM_PROT_READ|VM_PROT_EXECUTE)),
 				(mappings[i].sfm_max_prot &
 				 (VM_PROT_READ|VM_PROT_EXECUTE)),
-				VM_INHERIT_DEFAULT) != KERN_SUCCESS) {
-			lsf_unload(file_object, entry->base_address, sm_info);
+				VM_INHERIT_DEFAULT);
+		}
+		if (kr != KERN_SUCCESS) {
+			vm_offset_t old_base_address;
+
+			old_base_address = entry->base_address;
+			lsf_deallocate(entry,
+				       file_object,
+				       entry->base_address,
+				       sm_info,
+				       TRUE);
+			entry = NULL;
 
 			if (slide_p != NULL) {
 				/*
@@ -2080,25 +2394,65 @@ restart_after_slide:
 				 * shared region, so let's try and slide it...
 				 */
 
+				SHARED_REGION_TRACE(
+					SHARED_REGION_TRACE_CONFLICT,
+					("shared_region: %p: lsf_map: "
+					 "mapping #%d failed to map, "
+					 "kr=0x%x, sliding...\n",
+					 current_thread(), i, kr));
+				shared_region_dump_mappings(
+					SHARED_REGION_TRACE_INFO,
+					mappings, map_cnt, base_offset);
+				shared_region_dump_conflict_info(
+					SHARED_REGION_TRACE_CONFLICT,
+					region_entry->backing.map,
+					(old_base_address +
+					 ((mappings[i].sfm_address)
+					  & region_mask)),
+					vm_map_round_page(mappings[i].sfm_size));
+
 				/* lookup an appropriate spot */
 				kr = lsf_slide(map_cnt, mappings,
 					       sm_info, &base_offset);
 				if (kr == KERN_SUCCESS) {
 					/* try and map it there ... */
-					entry->base_address = base_offset;
 					goto restart_after_slide;
 				}
 				/* couldn't slide ... */
 			}
-
+			       
+			SHARED_REGION_TRACE(
+				SHARED_REGION_TRACE_CONFLICT,
+				("shared_region: %p: lsf_map: "
+				 "mapping #%d failed to map, "
+				 "kr=0x%x, no sliding\n",
+				 current_thread(), i, kr));
+			shared_region_dump_mappings(
+				SHARED_REGION_TRACE_INFO,
+				mappings, map_cnt, base_offset);
+			shared_region_dump_conflict_info(
+				SHARED_REGION_TRACE_CONFLICT,
+				region_entry->backing.map,
+				(old_base_address +
+				 ((mappings[i].sfm_address)
+				  & region_mask)),
+				vm_map_round_page(mappings[i].sfm_size));
 			return KERN_FAILURE;
 		}
 
 		/* record this mapping */
 		file_mapping = (loaded_mapping_t *)zalloc(lsf_zone);
 		if (file_mapping == NULL) {
-			lsf_unload(file_object, entry->base_address, sm_info);
-			printf("lsf_map: unable to allocate memory\n");
+			lsf_deallocate(entry,
+				       file_object,
+				       entry->base_address,
+				       sm_info,
+				       TRUE);
+			SHARED_REGION_TRACE(
+				SHARED_REGION_TRACE_ERROR,
+				("shared_region: %p: "
+				 "lsf_map: unable to allocate mapping\n",
+				 current_thread()));
 			return KERN_NO_SPACE;
 		}
 		shared_file_available_hash_ele--;
@@ -2124,14 +2478,20 @@ restart_after_slide:
 		*slide_p = base_offset - original_base_offset;
 	}
 
-	if (sm_info->flags & SHARED_REGION_STANDALONE) {
+	if ((sm_info->flags & SHARED_REGION_STANDALONE) ||
+	    (total_size == 0)) {
 		/*
-		 * We have a standalone and private shared region, so we
+		 * Two cases:
+		 * 1. we have a standalone and private shared region, so we
 		 * don't really need to keep the information about each file
 		 * and each mapping.  Just deallocate it all.
+		 * 2. the total size of the mappings is 0, so nothing at all
+		 * was mapped.  Let's not waste kernel resources to describe
+		 * nothing.
+		 *
 		 * XXX we still have the hash table, though...
 		 */
-		lsf_deallocate(file_object, entry->base_address, sm_info,
+		lsf_deallocate(entry, file_object, entry->base_address, sm_info,
 			       FALSE);
 	}
 
@@ -2150,7 +2510,7 @@ lsf_unload(
 	vm_offset_t	        base_offset,
 	shared_region_task_mappings_t	sm_info)
 {
-	lsf_deallocate(file_object, base_offset, sm_info, TRUE);
+	lsf_deallocate(NULL, file_object, base_offset, sm_info, TRUE);
 }
 
 /*
@@ -2162,6 +2522,7 @@ lsf_unload(
  */
 static void
 lsf_deallocate(
+	load_struct_t		*target_entry,
 	void			*file_object,
 	vm_offset_t		base_offset,
 	shared_region_task_mappings_t	sm_info,
@@ -2170,11 +2531,15 @@ lsf_deallocate(
 	load_struct_t		*entry;
 	loaded_mapping_t	*map_ele;
 	loaded_mapping_t	*back_ptr;
+	kern_return_t		kr;
 
-	LSF_DEBUG(("lsf_deallocate(file=%p,base=0x%x,sm_info=%p,unload=%d)\n",
-		   file_object, base_offset, sm_info, unload));
-	entry = lsf_hash_delete(file_object, base_offset, sm_info);
-	if(entry) {
+	LSF_DEBUG(("lsf_deallocate(target=%p,file=%p,base=0x%x,sm_info=%p,unload=%d)\n",
+		   target_entry, file_object, base_offset, sm_info, unload));
+	entry = lsf_hash_delete(target_entry,
+				file_object,
+				base_offset,
+				sm_info);
+	if (entry) {
 		map_ele = entry->mappings;
 		while(map_ele != NULL) {
 			if (unload) {
@@ -2191,10 +2556,11 @@ lsf_deallocate(
 				region_entry = (vm_named_entry_t)
 					region_handle->ip_kobject;
 				
-				vm_deallocate(region_entry->backing.map,
-					      (entry->base_address + 
-					       map_ele->mapping_offset),
-					      map_ele->size);
+				kr = vm_deallocate(region_entry->backing.map,
+						   (entry->base_address + 
+						    map_ele->mapping_offset),
+						   map_ele->size);
+				assert(kr == KERN_SUCCESS);
 			}
 			back_ptr = map_ele;
 			map_ele = map_ele->next;
@@ -2203,14 +2569,14 @@ lsf_deallocate(
 				   back_ptr, back_ptr->mapping_offset,
 				   back_ptr->size));
 			zfree(lsf_zone, back_ptr);
-		        shared_file_available_hash_ele++;
+			shared_file_available_hash_ele++;
 		}
 		LSF_DEBUG(("lsf_deallocate: freeing entry %p\n", entry));
 		LSF_ALLOC_DEBUG(("lsf_deallocate: entry=%p", entry));
 		zfree(lsf_zone, entry);
 	        shared_file_available_hash_ele++;
 	}
-	LSF_DEBUG(("lsf_unload: done\n"));
+	LSF_DEBUG(("lsf_deallocate: done\n"));
 }
 
 /* integer is from 1 to 100 and represents percent full */
diff --git a/osfmk/vm/vm_shared_memory_server.h b/osfmk/vm/vm_shared_memory_server.h
index 4c7aca6e2..ca5e06f11 100644
--- a/osfmk/vm/vm_shared_memory_server.h
+++ b/osfmk/vm/vm_shared_memory_server.h
@@ -43,6 +43,7 @@
 #include <mach/shared_memory_server.h>
 
 #include <kern/kern_types.h>
+#include <kern/macro_help.h>
 
 #if DEBUG
 extern int shared_region_debug;
@@ -56,8 +57,17 @@ extern int shared_region_debug;
 #define SHARED_REGION_DEBUG(args)
 #endif /* DEBUG */
 
-extern mach_port_t      shared_text_region_handle;
-extern mach_port_t      shared_data_region_handle;
+extern int shared_region_trace_level;
+#define SHARED_REGION_TRACE_NONE	0	/* no trace */
+#define SHARED_REGION_TRACE_ERROR	1	/* trace abnormal events */
+#define SHARED_REGION_TRACE_CONFLICT	2	/* trace library conflicts */
+#define SHARED_REGION_TRACE_INFO	3	/* trace all events */
+#define SHARED_REGION_TRACE(level, args)		\
+	MACRO_BEGIN					\
+	if (level <= shared_region_trace_level) {	\
+		printf args;				\
+	}						\
+	MACRO_END
 
 struct shared_region_task_mappings {
 	mach_port_t		text_region;
@@ -146,7 +156,7 @@ typedef struct shared_region_object_chain *shared_region_object_chain_t;
 /* address space shared region descriptor */
 struct shared_region_mapping {
         decl_mutex_data(,       Lock)   /* Synchronization */
-	int			ref_count;
+	unsigned int		ref_count;
 	unsigned int		fs_base;
 	unsigned int		system;
 	mach_port_t		text_region;
@@ -217,7 +227,9 @@ extern kern_return_t shared_region_mapping_create(
 				vm_offset_t		client_base,
 				shared_region_mapping_t	*shared_region,
 				vm_offset_t		alt_base,
-				vm_offset_t		alt_next);
+				vm_offset_t		alt_next,
+				int			fs_base,
+				int			system);
 
 extern kern_return_t shared_region_mapping_ref(
 				shared_region_mapping_t	shared_region);
@@ -263,7 +275,9 @@ __private_extern__ struct load_struct *lsf_remove_regions_mappings_lock(
 extern unsigned int lsf_mapping_pool_gauge(void);
 
 extern kern_return_t shared_file_create_system_region(
-	shared_region_mapping_t	*shared_region);
+	shared_region_mapping_t	*shared_region,
+	int			fs_base,
+	int			system);
 
 extern void remove_all_shared_regions(void);
 
@@ -283,6 +297,20 @@ extern kern_return_t mach_memory_entry_purgable_control(
 	vm_purgable_t	control,
 	int		*state);
 
+extern kern_return_t mach_memory_entry_page_op(
+	ipc_port_t		entry_port,
+	vm_object_offset_t	offset,
+	int			ops,
+	ppnum_t			*phys_entry,
+	int			*flags);
+
+extern kern_return_t mach_memory_entry_range_op(
+	ipc_port_t		entry_port,
+	vm_object_offset_t	offset_beg,
+	vm_object_offset_t	offset_end,
+	int                     ops,
+	int                     *range);
+
 #endif /* KERNEL_PRIVATE */
 
 #endif	/* _VM_SHARED_MEMORY_SERVER_H_ */
diff --git a/osfmk/vm/vm_user.c b/osfmk/vm/vm_user.c
index 5db1859b8..24c1aa3c9 100644
--- a/osfmk/vm/vm_user.c
+++ b/osfmk/vm/vm_user.c
@@ -2350,6 +2350,11 @@ redo_lookup:
 				shadow_object = map_entry->object.vm_object;
 				vm_object_unlock(object);
 
+				prot = map_entry->protection & ~VM_PROT_WRITE;
+#ifdef STACK_ONLY_NX
+				if (map_entry->alias != VM_MEMORY_STACK && prot)
+				        prot |= VM_PROT_EXECUTE;
+#endif
 				vm_object_pmap_protect(
 					object, map_entry->offset,
 					total_size,
@@ -2358,7 +2363,7 @@ redo_lookup:
 							? PMAP_NULL :
 							target_map->pmap),
 					map_entry->vme_start,
-					map_entry->protection & ~VM_PROT_WRITE);
+					prot);
 				total_size -= (map_entry->vme_end 
 						- map_entry->vme_start);
 				next_entry = map_entry->vme_next;
@@ -2875,6 +2880,106 @@ mach_destroy_memory_entry(
 		mutex_unlock(&(named_entry)->Lock);
 }
 
+/* Allow manipulation of individual page state.  This is actually part of */
+/* the UPL regimen but takes place on the memory entry rather than on a UPL */
+
+kern_return_t
+mach_memory_entry_page_op(
+	ipc_port_t		entry_port,
+	vm_object_offset_t	offset,
+	int			ops,
+	ppnum_t			*phys_entry,
+	int			*flags)
+{
+	vm_named_entry_t	mem_entry;
+	vm_object_t		object;
+	kern_return_t		kr;
+
+	if (entry_port == IP_NULL ||
+	    ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
+
+	named_entry_lock(mem_entry);
+
+	if (mem_entry->is_sub_map || mem_entry->is_pager) {
+		named_entry_unlock(mem_entry);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	object = mem_entry->backing.object;
+	if (object == VM_OBJECT_NULL) {
+		named_entry_unlock(mem_entry);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	vm_object_reference(object);
+	named_entry_unlock(mem_entry);
+
+	kr = vm_object_page_op(object, offset, ops, phys_entry, flags);
+
+	vm_object_deallocate(object);	
+
+	return kr;
+}
+
+/*
+ * mach_memory_entry_range_op offers performance enhancement over 
+ * mach_memory_entry_page_op for page_op functions which do not require page 
+ * level state to be returned from the call.  Page_op was created to provide 
+ * a low-cost alternative to page manipulation via UPLs when only a single 
+ * page was involved.  The range_op call establishes the ability in the _op 
+ * family of functions to work on multiple pages where the lack of page level
+ * state handling allows the caller to avoid the overhead of the upl structures.
+ */
+
+kern_return_t
+mach_memory_entry_range_op(
+	ipc_port_t		entry_port,
+	vm_object_offset_t	offset_beg,
+	vm_object_offset_t	offset_end,
+	int                     ops,
+	int                     *range)
+{
+	vm_named_entry_t	mem_entry;
+	vm_object_t		object;
+	kern_return_t		kr;
+
+	if (entry_port == IP_NULL ||
+	    ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
+
+	named_entry_lock(mem_entry);
+
+	if (mem_entry->is_sub_map || mem_entry->is_pager) {
+		named_entry_unlock(mem_entry);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	object = mem_entry->backing.object;
+	if (object == VM_OBJECT_NULL) {
+		named_entry_unlock(mem_entry);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	vm_object_reference(object);
+	named_entry_unlock(mem_entry);
+
+	kr = vm_object_range_op(object,
+				offset_beg,
+				offset_end,
+				ops,
+				range);
+
+	vm_object_deallocate(object);
+
+	return kr;
+}
 
 
 kern_return_t
diff --git a/pexpert/conf/Makefile.i386 b/pexpert/conf/Makefile.i386
index f807e9ecb..895c8b5fe 100644
--- a/pexpert/conf/Makefile.i386
+++ b/pexpert/conf/Makefile.i386
@@ -10,6 +10,7 @@ CWARNFLAGS= $(filter-out -Wbad-function-cast, $(CWARNFLAGS_STD))
 OBJS_NO_WERROR=		\
 	ioconf.o	\
 	bootargs.o	\
+	pe_init.o	\
 	device_tree.o
 
 OBJS_WERROR=$(filter-out $(OBJS_NO_WERROR),$(OBJS))
diff --git a/pexpert/gen/bootargs.c b/pexpert/gen/bootargs.c
index 73b65b997..2d7f80b3f 100644
--- a/pexpert/gen/bootargs.c
+++ b/pexpert/gen/bootargs.c
@@ -171,8 +171,8 @@ getval(
 	char *s, 
 	int *val)
 {
-	register unsigned radix, intval;
-	register unsigned char c;
+	unsigned int radix, intval;
+        char c;
 	int sign = 1;
 
 	if (*s == '=') {
@@ -181,7 +181,7 @@ getval(
 			sign = -1, s++;
 		intval = *s++-'0';
 		radix = 10;
-		if (intval == 0)
+		if (intval == 0) {
 			switch(*s) {
 
 			case 'x':
@@ -205,28 +205,44 @@ getval(
 				if (!isargsep(*s))
 					return (STR);
 			}
+                } else if (intval >= radix) {
+                    return (STR);
+                }
 		for(;;) {
-			if (((c = *s++) >= '0') && (c <= '9'))
+                        c = *s++;
+                        if (isargsep(c))
+                            break;
+                        if ((radix <= 10) &&
+                            ((c >= '0') && (c <= ('9' - (10 - radix))))) {
+                                c -= '0';
+                        } else if ((radix == 16) &&
+                                   ((c >= '0') && (c <= '9'))) {
 				c -= '0';
-			else if ((c >= 'a') && (c <= 'f'))
+                        } else if ((radix == 16) &&
+                                   ((c >= 'a') && (c <= 'f'))) {
 				c -= 'a' - 10;
-			else if ((c >= 'A') && (c <= 'F'))
+                        } else if ((radix == 16) &&
+                                   ((c >= 'A') && (c <= 'F'))) {
 				c -= 'A' - 10;
-			else if (c == 'k' || c == 'K')
-				{ sign *= 1024; break; }
-			else if (c == 'm' || c == 'M')
-				{ sign *= 1024 * 1024; break; }
-			else if (c == 'g' || c == 'G')
-				{ sign *= 1024 * 1024 * 1024; break; }
-			else if (isargsep(c))
+                        } else if (c == 'k' || c == 'K') {
+				sign *= 1024;
 				break;
-			else
+			} else if (c == 'm' || c == 'M') {
+				sign *= 1024 * 1024;
+                                break;
+			} else if (c == 'g' || c == 'G') {
+				sign *= 1024 * 1024 * 1024;
+                                break;
+			} else {
 				return (STR);
+                        }
 			if (c >= radix)
 				return (STR);
 			intval *= radix;
 			intval += c;
 		}
+                if (!isargsep(c) && !isargsep(*s))
+                    return STR;
 		*val = intval * sign;
 		return (NUM);
 	}
diff --git a/pexpert/i386/kd.c b/pexpert/i386/kd.c
index c87371b69..7424ed2e4 100644
--- a/pexpert/i386/kd.c
+++ b/pexpert/i386/kd.c
@@ -87,8 +87,6 @@
 
 extern void cpu_shutdown(void);
 
-int	cngetc(void);
-int	cnmaygetc(void);
 void	kdreboot(void);
 
 /*
@@ -122,35 +120,6 @@ void	kdreboot(void);
 #define K_CMD_ECHO      0xee    /* used for diagnostic testing */
 #define K_CMD_RESET     0xfe    /* issue a system reset */
 
-/* 
- * cngetc / cnmaygetc
- * 
- * Get one character using polling, rather than interrupts.
- * Used by the kernel debugger.
- */
-
-int
-cngetc(void)
-{
-    char c;
-
-    if ( 0 == (*PE_poll_input)(0, &c) )
-        return ( c );
-    else
-        return ( 0 );
-}
-
-int
-cnmaygetc(void)
-{
-    char c;
-
-    if ( 0 == (*PE_poll_input)(0, &c) )
-        return ( c );
-    else
-        return ( 0 );
-}
-
 /*
  * kd_sendcmd
  *
diff --git a/pexpert/i386/pe_bootargs.c b/pexpert/i386/pe_bootargs.c
index fd4fb6112..7c2f531a6 100644
--- a/pexpert/i386/pe_bootargs.c
+++ b/pexpert/i386/pe_bootargs.c
@@ -32,5 +32,5 @@ char *
 PE_boot_args(
 	void)
 {
-	return((char *)((KernelBootArgs_t *)PE_state.bootArgs)->bootString);
+    return ((boot_args *)PE_state.bootArgs)->CommandLine;
 }
diff --git a/pexpert/i386/pe_identify_machine.c b/pexpert/i386/pe_identify_machine.c
index 9a1a2c705..2d59f1b66 100644
--- a/pexpert/i386/pe_identify_machine.c
+++ b/pexpert/i386/pe_identify_machine.c
@@ -29,9 +29,6 @@
 #include <pexpert/protos.h>
 #include <pexpert/device_tree.h>
 
-/* External declarations */
-unsigned int LockTimeOut = 1250000000; /* XXX - Need real value for i386 */
-
 /* Local declarations */
 void pe_identify_machine(boot_args *args);
 
diff --git a/pexpert/i386/pe_init.c b/pexpert/i386/pe_init.c
index 67dfb28f0..f85167df9 100644
--- a/pexpert/i386/pe_init.c
+++ b/pexpert/i386/pe_init.c
@@ -37,6 +37,7 @@
 #include <pexpert/device_tree.h>
 #include <pexpert/pe_images.h>
 #include <kern/sched_prim.h>
+#include <kern/debug.h>
 
 #include "fakePPCStructs.h"
 #include "fakePPCDeviceTree.h"
@@ -46,9 +47,6 @@
 extern void pe_identify_machine(void * args);
 extern void initialize_screen(void *, unsigned int);
 
-/* Local references */
-static int         PE_fb_mode  = TEXT_MODE;
-
 /* private globals */
 PE_state_t  PE_state;
 dt_data     gMemoryMapNode;
@@ -59,6 +57,9 @@ dt_data     gCompatibleProp;
 /* Clock Frequency Info */
 clock_frequency_info_t gPEClockFrequencyInfo;
 
+void *gPEEFISystemTable     = 0;
+void *gPEEFIRuntimeServices = 0;
+
 int PE_initialize_console( PE_Video * info, int op )
 {
     static int   last_console = -1;
@@ -78,7 +79,7 @@ int PE_initialize_console( PE_Video * info, int op )
         bootInfo.v_depth     = info->v_depth;
         bInfo = &bootInfo;
         if (info == &PE_state.video) {
-            bootInfo.v_display  = PE_fb_mode;
+            bootInfo.v_display   = info->v_display;
         } else {
             bootInfo.v_display  = GRAPHICS_MODE;
         }
@@ -90,19 +91,16 @@ int PE_initialize_console( PE_Video * info, int op )
 
         case kPEDisableScreen:
             initialize_screen((void *) bInfo, op);
-#ifdef FIXME
-            last_console = switch_to_serial_console();
-#endif
             kprintf("kPEDisableScreen %d\n", last_console);
+	    if (!console_is_serial())
+		last_console = switch_to_serial_console();
             break;
 
         case kPEEnableScreen:
             initialize_screen((void *) bInfo, op);
             kprintf("kPEEnableScreen %d\n", last_console);
-#ifdef FIXME
             if( last_console != -1)
                 switch_to_old_console( last_console);
-#endif
             break;
 	
         default:
@@ -115,11 +113,7 @@ int PE_initialize_console( PE_Video * info, int op )
 
 void PE_init_iokit(void)
 {
-    long * dt;
-    int    i;
-    KernelBootArgs_t *kap = (KernelBootArgs_t *)PE_state.bootArgs;
     enum { kMaxBootVar = 128 };
-    char *rdValue, *platformValue;
         
     typedef struct {
         char            name[32];
@@ -127,98 +121,50 @@ void PE_init_iokit(void)
         unsigned long   value[2];
     } DriversPackageProp;
 
+    boolean_t bootClutInitialized = FALSE;
+    boolean_t norootInitialized = FALSE;
+    DTEntry             entry;
+    int			size;
+    void **		map;
+
     PE_init_kprintf(TRUE);
     PE_init_printf(TRUE);
 
+    kprintf("Kernel boot args: '%s'\n", PE_boot_args());
+
     /*
-     * Update the fake device tree with the driver information provided by
-     * the booter.
+     * Fetch the CLUT and the noroot image.
      */
+	boot_progress_element * bootPict;
 
-    gDriversProp.length   = kap->numBootDrivers * sizeof(DriversPackageProp);
-    gMemoryMapNode.length = 2 * sizeof(long);
-
-    rdValue = kalloc(kMaxBootVar);
-    if ( PE_parse_boot_arg("rd", rdValue) ) {
-        if (*rdValue == '*') {
-            gRootpathProp.address = (rdValue + 1);
-        } else {
-            gRootpathProp.address = rdValue;
-        }
-        strcat(rdValue, ",");
-    } else {
-        gRootpathProp.address = rdValue;
-        rdValue[0] = '\0';
-    }
-    strcat(rdValue, kap->bootFile);
-    gRootpathProp.length = strlen(rdValue) + 1;
-
-    platformValue = kalloc(kMaxBootVar);
-    if ( ! PE_parse_boot_arg("platform", platformValue) ) {
-        strcpy(platformValue, kDefaultPlatformName);
-    }
-    gCompatibleProp.address = platformValue;
-    gCompatibleProp.length = strlen(platformValue) + 1;
-
-    dt = (long *) createdt( fakePPCDeviceTree,
-            	  &((boot_args*)PE_state.fakePPCBootArgs)->deviceTreeLength );
-
-    kfree(rdValue, kMaxBootVar);
-    kfree(platformValue, kMaxBootVar);
-
-
-    if ( dt )
-    {
-        DriversPackageProp * prop = (DriversPackageProp *) gDriversProp.address;
-
-        /* Copy driver info in kernBootStruct to fake device tree */
-
-        for ( i = 0; i < kap->numBootDrivers; i++, prop++ )
-        {
-            switch ( kap->driverConfig[i].type )
-            {
-                case kBootDriverTypeKEXT:
-                    sprintf(prop->name, "Driver-%lx", kap->driverConfig[i].address);
-                    break;
-                
-                 case kBootDriverTypeMKEXT:
-                    sprintf(prop->name, "DriversPackage-%lx", kap->driverConfig[i].address);
-                    break;
-
-                default:
-                    sprintf(prop->name, "DriverBogus-%lx", kap->driverConfig[i].address);
-                    break;
-            }
-            prop->length   = sizeof(prop->value);
-            prop->value[0] = kap->driverConfig[i].address;
-            prop->value[1] = kap->driverConfig[i].size;
+    if( kSuccess == DTLookupEntry(0, "/chosen/memory-map", &entry)) {
+	if( kSuccess == DTGetProperty(entry, "BootCLUT", (void **) &map, &size)) {
+	    bcopy( map[0], appleClut8, sizeof(appleClut8) );
+            bootClutInitialized = TRUE;
         }
 
-        *((long *)gMemoryMapNode.address) = kap->numBootDrivers + 1;
+	if( kSuccess == DTGetProperty(entry, "Pict-FailedBoot", (void **) &map, &size)) {
+	    bootPict = (boot_progress_element *) map[0];
+	    default_noroot.width  = bootPict->width;
+	    default_noroot.height = bootPict->height;
+	    default_noroot.dx     = 0;
+	    default_noroot.dy     = bootPict->yOffset;
+	    default_noroot_data   = &bootPict->data[0];
+            norootInitialized = TRUE;
+	}
     }
 
-    /* Setup powermac_info and powermac_machine_info structures */
-
-    ((boot_args*)PE_state.fakePPCBootArgs)->deviceTreeP	= (unsigned long *) dt;
-    ((boot_args*)PE_state.fakePPCBootArgs)->topOfKernelData = (unsigned long) kalloc(0x2000);
-
-    /* 
-     * Setup the OpenFirmware Device Tree routines
-     * so the console can be found and the right I/O space 
-     * can be used..
-     */
-    DTInit(dt);
-
-    /*
-     * Fetch the CLUT and the noroot image.
-     */
+    if (!bootClutInitialized) {
     bcopy( (void *) (uintptr_t) bootClut, (void *) appleClut8, sizeof(appleClut8) );
+    }
 
+    if (!norootInitialized) {
     default_noroot.width  = kFailedBootWidth;
     default_noroot.height = kFailedBootHeight;
     default_noroot.dx     = 0;
     default_noroot.dy     = kFailedBootOffset;
     default_noroot_data   = failedBootPict;
+    }
     
     /*
      * Initialize the panic UI
@@ -231,53 +177,50 @@ void PE_init_iokit(void)
     vc_progress_initialize( &default_progress, default_progress_data,
                             (unsigned char *) appleClut8 );
 
-    (void) StartIOKit( (void*)dt, PE_state.bootArgs, 0, 0);
+    (void) StartIOKit( PE_state.deviceTreeHead, PE_state.bootArgs, gPEEFIRuntimeServices, 0);
 }
 
-void PE_init_platform(boolean_t vm_initialized, void * args)
+void PE_init_platform(boolean_t vm_initialized, void * _args)
 {
-	if (PE_state.initialized == FALSE)
-	{
-	    KernelBootArgs_t *kap = (KernelBootArgs_t *) args;
+    boot_args *args = (boot_args *)_args;
 
+    if (PE_state.initialized == FALSE) {
 	    PE_state.initialized        = TRUE;
-	    PE_state.bootArgs           = args;
-	    PE_state.video.v_baseAddr   = kap->video.v_baseAddr;
-	    PE_state.video.v_rowBytes   = kap->video.v_rowBytes;
-	    PE_state.video.v_height     = kap->video.v_height;
-	    PE_state.video.v_width      = kap->video.v_width;
-	    PE_state.video.v_depth      = kap->video.v_depth;
-	    PE_state.video.v_display    = kap->video.v_display;
-	    PE_fb_mode                  = kap->graphicsMode;
-	    PE_state.fakePPCBootArgs    = (boot_args *)&fakePPCBootArgs;
-	    ((boot_args *)PE_state.fakePPCBootArgs)->machineType	= 386;
-
-        if (PE_fb_mode == TEXT_MODE)
-        {
-            /* Force a text display if the booter did not setup a
-             * VESA frame buffer.
-             */
-            PE_state.video.v_display = 0;
-        }
+
+        // New EFI-style
+        PE_state.bootArgs           = _args;
+        PE_state.deviceTreeHead	    = args->deviceTreeP;
+        PE_state.video.v_baseAddr   = args->Video.v_baseAddr;
+        PE_state.video.v_rowBytes   = args->Video.v_rowBytes;
+        PE_state.video.v_width	    = args->Video.v_width;
+        PE_state.video.v_height	    = args->Video.v_height;
+        PE_state.video.v_depth	    = args->Video.v_depth;
+        PE_state.video.v_display    = args->Video.v_display;
+        strcpy( PE_state.video.v_pixelFormat, "PPPPPPPP");
     }
 
-    if (!vm_initialized)
-    {
+    if (!vm_initialized) {
 		/* Hack! FIXME.. */ 
         outb(0x21, 0xff);   /* Maskout all interrupts Pic1 */
         outb(0xa1, 0xff);   /* Maskout all interrupts Pic2 */
  
-        pe_identify_machine(args);
+        if (PE_state.deviceTreeHead) {
+            DTInit(PE_state.deviceTreeHead);
     }
-    else
-    {
+
+        pe_identify_machine(args);
+    } else {
+        DTEntry entry;
+        void *ptr;
+        uint32_t size;
+
         pe_init_debug();
     }
 }
 
 void PE_create_console( void )
 {
-    if ( PE_state.video.v_display )
+    if ( PE_state.video.v_display == GRAPHICS_MODE )
         PE_initialize_console( &PE_state.video, kPEGraphicsMode );
     else
         PE_initialize_console( &PE_state.video, kPETextMode );
@@ -287,22 +230,6 @@ int PE_current_console( PE_Video * info )
 {
     *info = PE_state.video;
 
-    if ( PE_fb_mode == TEXT_MODE )
-    {
-        /*
-         * FIXME: Prevent the IOBootFrameBuffer from starting up
-         * when we are in Text mode.
-         */
-        info->v_baseAddr = 0;
-        
-        /*
-         * Scale the size of the text screen from characters
-         * to pixels.
-         */
-        info->v_width  *= 8;   // CHARWIDTH
-        info->v_height *= 16;  // CHARHEIGHT
-    }
-
     return (0);
 }
 
diff --git a/pexpert/i386/pe_interrupt.c b/pexpert/i386/pe_interrupt.c
index 559a0d813..38ee8e3ac 100644
--- a/pexpert/i386/pe_interrupt.c
+++ b/pexpert/i386/pe_interrupt.c
@@ -32,7 +32,7 @@
 #include <sys/kdebug.h>
 
 
-void PE_incoming_interrupt(int, void *);
+void PE_incoming_interrupt(x86_saved_state_t *);
 
 
 struct i386_interrupt_handler {
@@ -49,21 +49,43 @@ i386_interrupt_handler_t	PE_interrupt_handler;
 
 
 void
-PE_incoming_interrupt(int interrupt, void *state)
+PE_incoming_interrupt(x86_saved_state_t *state)
 {
 	i386_interrupt_handler_t	*vector;
+	uint64_t			rip;
+	int				interrupt;
+	boolean_t			user_mode = FALSE;
+
+        if (is_saved_state64(state) == TRUE) {
+	        x86_saved_state64_t	*state64;
+
+	        state64 = saved_state64(state);
+		rip = state64->isf.rip;
+		interrupt = state64->isf.trapno;
+		user_mode = TRUE;
+	} else {
+	        x86_saved_state32_t	*state32;
+
+	        state32 = saved_state32(state);
+		if (state32->cs & 0x03)
+		        user_mode = TRUE;
+		rip = state32->eip;
+		interrupt = state32->trapno;
+	}
 
-	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
-			      0, ((unsigned int *)state)[7], 0, 0, 0);
+	KERNEL_DEBUG_CONSTANT(
+		MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
+		interrupt, (unsigned int)rip, user_mode, 0, 0);
 
 	vector = &PE_interrupt_handler;
 
 	if (!lapic_interrupt(interrupt, state)) {
-		vector->handler(vector->target, state, vector->nub, interrupt);
+		vector->handler(vector->target, NULL, vector->nub, interrupt);
 	}
 
-	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END,
-	   0, 0, 0, 0, 0);
+	KERNEL_DEBUG_CONSTANT(
+		MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END,
+		0, 0, 0, 0, 0);
 }
 
 void PE_install_interrupt_handler(void *nub,
diff --git a/pexpert/i386/pe_kprintf.c b/pexpert/i386/pe_kprintf.c
index a39ecf10f..628dbdff0 100644
--- a/pexpert/i386/pe_kprintf.c
+++ b/pexpert/i386/pe_kprintf.c
@@ -115,3 +115,10 @@ void kprintf(const char *fmt, ...)
         ml_set_interrupts_enabled(state);
     }
 }
+
+extern void kprintf_break_lock(void);
+void
+kprintf_break_lock(void)
+{
+	simple_lock_init(&kprintf_lock, 0);
+}
diff --git a/pexpert/i386/pe_serial.c b/pexpert/i386/pe_serial.c
index ff104bba5..f72de17ab 100644
--- a/pexpert/i386/pe_serial.c
+++ b/pexpert/i386/pe_serial.c
@@ -73,12 +73,18 @@ enum {
 };
 
 enum {
+    UART_LSR_DR    = 0x01,
+    UART_LSR_OE    = 0x02,
+    UART_LSR_PE    = 0x04,
+    UART_LSR_FE    = 0x08,
     UART_LSR_THRE  = 0x20
 };
 
-#define UART_BAUD_RATE  115200
+static unsigned uart_baud_rate = 115200;
 #define UART_PORT_ADDR  COM1_PORT_ADDR
 
+#define UART_CLOCK  1843200   /* 1.8432 MHz clock */
+
 #define WRITE(r, v)  outb(UART_PORT_ADDR + UART_##r, v)
 #define READ(r)      inb(UART_PORT_ADDR + UART_##r)
 #define DELAY(x)     { volatile int _d_; for (_d_ = 0; _d_ < (10000*x); _d_++) ; }
@@ -102,8 +108,6 @@ uart_probe( void )
 static void
 uart_set_baud_rate( unsigned long baud_rate )
 {
-    #define UART_CLOCK  1843200   /* 1.8432 MHz clock */
-
     const unsigned char lcr = READ( LCR );
     unsigned long       div;
 
@@ -126,8 +130,40 @@ uart_putc( char c )
     WRITE( THR, c );
 }
 
+static int
+uart_getc( void )
+{
+    /*
+     * This function returns:
+     * -1 : no data
+     * -2 : receiver error
+     * >0 : character received
+     */
+
+    unsigned char lsr;
+
+    if (!uart_initted) return -1;
+
+    lsr = READ( LSR );
+
+    if ( lsr & (UART_LSR_FE | UART_LSR_PE | UART_LSR_OE) )
+    {
+        READ( RBR ); /* discard */
+        return -2;
+    }
+
+    if ( lsr & UART_LSR_DR )
+    {
+        return READ( RBR );
+    }
+
+    return -1;
+}
+
 int serial_init( void )
 {
+    unsigned serial_baud_rate = 0;
+	
     if ( /*uart_initted ||*/ uart_probe() == 0 ) return 0;
 
     /* Disable hardware interrupts */
@@ -143,9 +179,16 @@ int serial_init( void )
 
     WRITE( LCR, UART_LCR_8BITS );
 
-    /* Set baud rate */
+    /* Set baud rate - use the supplied boot-arg if available */
 
-    uart_set_baud_rate( UART_BAUD_RATE );
+    if (PE_parse_boot_arg("serialbaud", &serial_baud_rate))
+    {
+	    /* Valid divisor? */
+	    if (!((UART_CLOCK / 16) % serial_baud_rate)) {
+		    uart_baud_rate = serial_baud_rate;
+	    }
+    }
+    uart_set_baud_rate( uart_baud_rate );
 
     /* Assert DTR# and RTS# lines (OUT2?) */
 
@@ -168,5 +211,5 @@ void serial_putc( char c )
 
 int serial_getc( void )
 {
-    return 0;  /* not supported */
+    return uart_getc();
 }
diff --git a/pexpert/pexpert/i386/Makefile b/pexpert/pexpert/i386/Makefile
index 2b6faf73e..181965c90 100644
--- a/pexpert/pexpert/i386/Makefile
+++ b/pexpert/pexpert/i386/Makefile
@@ -9,6 +9,7 @@ include $(MakeInc_def)
 
 DATAFILES = \
         boot.h \
+        efi.h \
 	fb_entries.h \
 	protos.h
 
diff --git a/pexpert/pexpert/i386/boot.h b/pexpert/pexpert/i386/boot.h
index 5ae33c682..73254f6e4 100644
--- a/pexpert/pexpert/i386/boot.h
+++ b/pexpert/pexpert/i386/boot.h
@@ -28,14 +28,11 @@
 #ifndef _PEXPERT_I386_BOOT_H
 #define _PEXPERT_I386_BOOT_H
 
-/*
- * What the booter leaves behind for the kernel.
- */
+#include <stdint.h>
 
 /*
- * Maximum number of boot drivers that can be loaded.
+ * What the booter leaves behind for the kernel.
  */
-#define NDRIVERS  500
 
 /*
  * Types of boot driver that may be loaded by the booter.
@@ -46,267 +43,112 @@ enum {
     kBootDriverTypeMKEXT   = 2
 };
 
-typedef struct {
-    unsigned long address;  // address where driver was loaded
-    unsigned long size;     // number of bytes
-    unsigned long type;     // driver type
-} driver_config_t;
-
-/*
- * APM BIOS information.
- */
-typedef struct {
-    unsigned short major_vers;    // == 0 if not present
-    unsigned short minor_vers;
-    unsigned long  cs32_base;
-    unsigned long  cs16_base;
-    unsigned long  ds_base;
-    unsigned long  cs_length;
-    unsigned long  ds_length;
-    unsigned long  entry_offset;
-    union {
-        struct {
-            unsigned long mode_16        :1;
-            unsigned long mode_32        :1;
-            unsigned long idle_slows_cpu :1;
-            unsigned long reserved       :29;
-        } f;
-        unsigned long data;
-    } flags;
-    unsigned long connected;
-} APM_config_t;
-
-/*
- * PCI bus information.
- */
-typedef struct _PCI_bus_info_t {
-    union {
-        struct {
-            unsigned char configMethod1 :1;
-            unsigned char configMethod2 :1;
-            unsigned char               :2;
-            unsigned char specialCycle1 :1;
-            unsigned char specialCycle2 :1;
-        } s;
-        unsigned char d;
-    } u_bus;
-    unsigned char maxBusNum;
-    unsigned char majorVersion;
-    unsigned char minorVersion;
-    unsigned char BIOSPresent;
-} PCI_bus_info_t;
-
 /*
  * Video information.
  */
 struct boot_video {
-    unsigned long v_baseAddr;	// Base address of video memory
-    unsigned long v_display;    // Display Code (if Applicable
-    unsigned long v_rowBytes;   // Number of bytes per pixel row
-    unsigned long v_width;      // Width
-    unsigned long v_height;     // Height
-    unsigned long v_depth;      // Pixel Depth
+    uint32_t v_baseAddr;	// Base address of video memory
+    uint32_t v_display;    // Display Code
+    uint32_t v_rowBytes;   // Number of bytes per pixel row
+    uint32_t v_width;      // Width
+    uint32_t v_height;     // Height
+    uint32_t v_depth;      // Pixel Depth
 };
 
 typedef struct boot_video  boot_video;
 
+/* Values for v_display */
+
+#define VGA_TEXT_MODE         0
 #define GRAPHICS_MODE     1
-#define TEXT_MODE         0
+#define FB_TEXT_MODE          2
 
 
-/*
- * INT15, E820h - Query System Address Map.
- *
- * Documented in ACPI Specification Rev 2.0,
- * Chapter 15 (System Address Map Interfaces).
- */
-
-/*
- * ACPI defined memory range types.
- */
 enum {
-    kMemoryRangeUsable   = 1,    // RAM usable by the OS.
-    kMemoryRangeReserved = 2,    // Reserved. (Do not use)
-    kMemoryRangeACPI     = 3,    // ACPI tables. Can be reclaimed.
-    kMemoryRangeNVS      = 4,    // ACPI NVS memory. (Do not use)
-
-    /* Undefined types should be treated as kMemoryRangeReserved */
+    kEfiReservedMemoryType	= 0,
+    kEfiLoaderCode		= 1,
+    kEfiLoaderData		= 2,
+    kEfiBootServicesCode	= 3,
+    kEfiBootServicesData	= 4,
+    kEfiRuntimeServicesCode	= 5,
+    kEfiRuntimeServicesData	= 6,
+    kEfiConventionalMemory	= 7,
+    kEfiUnusableMemory		= 8,
+    kEfiACPIReclaimMemory	= 9,
+    kEfiACPIMemoryNVS		= 10,
+    kEfiMemoryMappedIO		= 11,
+    kEfiMemoryMappedIOPortSpace = 12,
+    kEfiPalCode			= 13,
+    kEfiMaxMemoryType		= 14
 };
 
 /*
  * Memory range descriptor.
  */
-typedef struct MemoryRange {
-    unsigned long long base;     // 64-bit base address
-    unsigned long long length;   // 64-bit length in bytes
-    unsigned long      type;     // type of memory range
-    unsigned long      reserved;
-} MemoryRange;
-
-#define kMemoryMapCountMax 40
-
-/*
- * BIOS drive information.
- */
-struct boot_drive_info {
-    struct drive_params {
-	unsigned short buf_size;
-	unsigned short info_flags;
-	unsigned long  phys_cyls;
-	unsigned long  phys_heads;
-	unsigned long  phys_spt;
-	unsigned long long phys_sectors;
-	unsigned short phys_nbps;
-	unsigned short dpte_offset;
-	unsigned short dpte_segment;
-	unsigned short key;
-	unsigned char  path_len;
-	unsigned char  reserved1;
-	unsigned short reserved2;
-	unsigned char  bus_type[4];
-	unsigned char  interface_type[8];
-	unsigned char  interface_path[8];
-	unsigned char  dev_path[8];
-	unsigned char  reserved3;
-	unsigned char  checksum;
-    } params __attribute__((packed));
-    struct drive_dpte {
-	unsigned short io_port_base;
-	unsigned short control_port_base;
-	unsigned char  head_flags;
-	unsigned char  vendor_info;
-	unsigned char  irq         : 4;
-	unsigned char  irq_unused  : 4;
-	unsigned char  block_count;
-	unsigned char  dma_channel : 4;
-	unsigned char  dma_type    : 4;
-	unsigned char  pio_type    : 4;
-	unsigned char  pio_unused  : 4;
-	unsigned short option_flags;
-	unsigned short reserved;
-	unsigned char  revision;
-	unsigned char  checksum;
-    } dpte __attribute__((packed));
-} __attribute__((packed));
-typedef struct boot_drive_info boot_drive_info_t;
-
-#define MAX_BIOS_DEVICES 8
-
-#define OLD_BOOT_STRING_LEN   160
-#define BOOT_STRING_LEN   1024
-#define CONFIG_SIZE       (12 * 4096)
-
-/* Old structure for compatibility */
-
-typedef struct {
-    short            version;
-    char             bootString[OLD_BOOT_STRING_LEN];  // boot arguments
-    int              magicCookie;                  // KERNBOOTMAGIC
-    int              numIDEs;                      // number of IDE drives
-    int              rootdev;                      // root device
-    int              convmem;                      // conventional memory
-    int              extmem;                       // extended memory
-    char             bootFile[128];                // kernel file name
-    int              firstAddr0;                   // first address for kern convmem
-    int              diskInfo[4];                  // info for bios dev 80-83
-    int              graphicsMode;                 // booted in graphics mode?
-    int              kernDev;                      // device kernel was fetched from
-    int              numBootDrivers;               // number of drivers loaded
-    char *           configEnd;                    // pointer to end of config files
-    int              kaddr;                        // kernel load address
-    int              ksize;                        // size of kernel
-    driver_config_t  driverConfig[NDRIVERS];
-    char             _reserved[2052];
-    boot_video       video;
-    PCI_bus_info_t   pciInfo;
-    APM_config_t     apmConfig;
-    char             config[CONFIG_SIZE];
-} KERNBOOTSTRUCT;
-
-#define KERNSTRUCT_ADDR   ((KERNBOOTSTRUCT *) 0x11000)
-#define KERNBOOTMAGIC     0xa7a7a7a7
-
-#define BOOT_LINE_LENGTH        256
+typedef struct EfiMemoryRange {
+    uint32_t Type;
+    uint32_t pad;
+    uint64_t PhysicalStart;
+    uint64_t VirtualStart;
+    uint64_t NumberOfPages;
+    uint64_t Attribute;
+} EfiMemoryRange;
+
+#define BOOT_LINE_LENGTH        1024
+#define BOOT_STRING_LEN         BOOT_LINE_LENGTH
 
 /*
  * Video information.. 
  */
 
 struct Boot_Video {
-	unsigned long	v_baseAddr;	/* Base address of video memory */
-	unsigned long	v_display;	/* Display Code (if Applicable */
-	unsigned long	v_rowBytes;	/* Number of bytes per pixel row */
-	unsigned long	v_width;	/* Width */
-	unsigned long	v_height;	/* Height */
-	unsigned long	v_depth;	/* Pixel Depth */
-};
+	uint32_t	v_baseAddr;	/* Base address of video memory */
+	uint32_t	v_display;	/* Display Code (if Applicable */
+	uint32_t	v_rowBytes;	/* Number of bytes per pixel row */
+	uint32_t	v_width;	/* Width */
+	uint32_t	v_height;	/* Height */
+	uint32_t	v_depth;	/* Pixel Depth */
+} __attribute__((aligned(4)));
 
 typedef struct Boot_Video	Boot_Video;
 
-/* DRAM Bank definitions - describes physical memory layout.
- */
-#define	kMaxDRAMBanks	26		/* maximum number of DRAM banks */
-
-struct DRAMBank
-{
-	unsigned long	base;		/* physical base of DRAM bank */
-	unsigned long	size;		/* size of bank */
-};
-typedef struct DRAMBank DRAMBank;
-
 
 /* Boot argument structure - passed into Mach kernel at boot time.
  */
+#define kBootArgsRevision		4
 #define kBootArgsVersion		1
-#define kBootArgsRevision		1
+
+#define kBootArgsEfiMode32              32
+#define kBootArgsEfiMode64              64
 
 typedef struct boot_args {
-  unsigned short	Revision;	/* Revision of boot_args structure */
-  unsigned short	Version;	/* Version of boot_args structure */
-  char		CommandLine[BOOT_LINE_LENGTH];	/* Passed in command line */
-  DRAMBank	PhysicalDRAM[kMaxDRAMBanks];	/* base and range pairs for the 26 DRAM banks */
-  Boot_Video	Video;		/* Video Information */
-    unsigned long	machineType;	/* Machine Type (gestalt) */
-    void		*deviceTreeP;	/* Base of flattened device tree */
-    unsigned long	deviceTreeLength;/* Length of flattened tree */
-    unsigned long	topOfKernelData;/* Highest address used in kernel data area */
-} boot_args;
+    uint16_t	Revision;	/* Revision of boot_args structure */
+    uint16_t	Version;	/* Version of boot_args structure */
+
+    char	CommandLine[BOOT_LINE_LENGTH];	/* Passed in command line */
+
+    uint32_t    MemoryMap;
+    uint32_t    MemoryMapSize;
+    uint32_t    MemoryMapDescriptorSize;
+    uint32_t    MemoryMapDescriptorVersion;
 
-extern boot_args passed_args;
+    Boot_Video	Video;		/* Video Information */
 
-/* New structures */
+    uint32_t    deviceTreeP;	/* Base of flattened device tree */
+    uint32_t	deviceTreeLength;/* Length of flattened tree */
 
+    uint32_t    kaddr;
+    uint32_t    ksize;
 
-#define KERNEL_BOOT_MAGIC    0xa5b6d7e8
+    uint32_t    efiRuntimeServicesPageStart;
+    uint32_t    efiRuntimeServicesPageCount;
+    uint32_t    efiSystemTable;
 
-typedef struct KernelBootArgs {
-    unsigned int     magicCookie;                  // KERNEL_BOOT_MAGIC
-    unsigned short   version;
-    unsigned short   revision;
-    unsigned int     size;                         // size of KernelBootArgs structure
-    int              numDrives;                    // number of BIOS drives
-    int              rootdev;                      // root device
-    int              convmem;                      // conventional memory
-    int              extmem;                       // extended memory
-    unsigned int     firstAddr0;                   // first address for kern convmem
-    int              graphicsMode;                 // booted in graphics mode?
-    int              kernDev;                      // device kernel was fetched from
-    int              numBootDrivers;               // number of drivers loaded
-    char *           configEnd;                    // pointer to end of config files
-    unsigned int     kaddr;                        // kernel load address
-    unsigned int     ksize;                        // size of kernel
-    char             bootFile[128];                // kernel file name
-    char             bootString[BOOT_STRING_LEN];  // boot arguments
-    driver_config_t  driverConfig[NDRIVERS];
-    unsigned long    memoryMapCount;
-    MemoryRange      memoryMap[kMemoryMapCountMax];
-    boot_drive_info_t  driveInfo[MAX_BIOS_DEVICES];
-    boot_video       video;
-    PCI_bus_info_t   pciInfo;
-    APM_config_t     apmConfig;
-    char             config[CONFIG_SIZE];
-} KernelBootArgs_t;
+    uint8_t     efiMode;       /* 32 = 32-bit, 64 = 64-bit */
+    uint8_t     __reserved1[3];
+    uint32_t    __reserved2[7];
 
+} __attribute__((aligned(4))) boot_args;
 
 #endif /* _PEXPERT_I386_BOOT_H */
 
diff --git a/pexpert/pexpert/i386/efi.h b/pexpert/pexpert/i386/efi.h
new file mode 100644
index 000000000..6cef5a5e7
--- /dev/null
+++ b/pexpert/pexpert/i386/efi.h
@@ -0,0 +1,549 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _PEXPERT_I386_EFI_H
+#define _PEXPERT_I386_EFI_H
+
+typedef uint8_t   EFI_UINT8;
+typedef uint16_t  EFI_UINT16;
+typedef uint32_t  EFI_UINT32;
+typedef uint64_t  EFI_UINT64;
+
+typedef uint32_t  EFI_UINTN;
+
+typedef int8_t    EFI_INT8;
+typedef int16_t   EFI_INT16;
+typedef int32_t   EFI_INT32;
+typedef int64_t   EFI_INT64;
+
+typedef int8_t    EFI_CHAR8;
+typedef int16_t   EFI_CHAR16;
+typedef int32_t   EFI_CHAR32;
+typedef int64_t   EFI_CHAR64;
+
+typedef uint32_t  EFI_STATUS;
+typedef boolean_t EFI_BOOLEAN;
+typedef void      VOID;
+typedef VOID *    EFI_HANDLE;
+
+typedef uint64_t  EFI_PTR64;
+typedef uint64_t  EFI_HANDLE64;
+/*
+
+Portions Copyright 2004, Intel Corporation
+All rights reserved. This program and the accompanying materials
+are licensed and made available under the terms and conditions of the BSD License
+which accompanies this distribution.  The full text of the license may be found at
+    http://opensource.org/licenses/bsd-license.php
+
+THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
+WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+
+*/
+
+
+//
+// Modifiers for EFI Runtime and Boot Services
+//
+#define EFI_RUNTIMESERVICE
+#define EFIAPI
+#define IN
+#define OUT
+#define OPTIONAL
+
+#define EFI_MAX_BIT       0x80000000
+
+//
+// Set the upper bit to indicate EFI Error.
+//
+#define EFIERR(a)                 (EFI_MAX_BIT | (a))
+
+#define EFIWARN(a)                (a)
+#define EFI_ERROR(a)              (((INTN) (a)) < 0)
+
+#define EFI_SUCCESS               0
+#define EFI_LOAD_ERROR            EFIERR (1)
+#define EFI_INVALID_PARAMETER     EFIERR (2)
+#define EFI_UNSUPPORTED           EFIERR (3)
+#define EFI_BAD_BUFFER_SIZE       EFIERR (4)
+#define EFI_BUFFER_TOO_SMALL      EFIERR (5)
+#define EFI_NOT_READY             EFIERR (6)
+#define EFI_DEVICE_ERROR          EFIERR (7)
+#define EFI_WRITE_PROTECTED       EFIERR (8)
+#define EFI_OUT_OF_RESOURCES      EFIERR (9)
+#define EFI_VOLUME_CORRUPTED      EFIERR (10)
+#define EFI_VOLUME_FULL           EFIERR (11)
+#define EFI_NO_MEDIA              EFIERR (12)
+#define EFI_MEDIA_CHANGED         EFIERR (13)
+#define EFI_NOT_FOUND             EFIERR (14)
+#define EFI_ACCESS_DENIED         EFIERR (15)
+#define EFI_NO_RESPONSE           EFIERR (16)
+#define EFI_NO_MAPPING            EFIERR (17)
+#define EFI_TIMEOUT               EFIERR (18)
+#define EFI_NOT_STARTED           EFIERR (19)
+#define EFI_ALREADY_STARTED       EFIERR (20)
+#define EFI_ABORTED               EFIERR (21)
+#define EFI_ICMP_ERROR            EFIERR (22)
+#define EFI_TFTP_ERROR            EFIERR (23)
+#define EFI_PROTOCOL_ERROR        EFIERR (24)
+#define EFI_INCOMPATIBLE_VERSION  EFIERR (25)
+#define EFI_SECURITY_VIOLATION    EFIERR (26)
+#define EFI_CRC_ERROR             EFIERR (27)
+
+#define EFI_WARN_UNKNOWN_GLYPH    EFIWARN (1)
+#define EFI_WARN_DELETE_FAILURE   EFIWARN (2)
+#define EFI_WARN_WRITE_FAILURE    EFIWARN (3)
+#define EFI_WARN_BUFFER_TOO_SMALL EFIWARN (4)
+
+//
+// EFI Specification Revision information
+//
+#define EFI_SPECIFICATION_MAJOR_REVISION  1
+#define EFI_SPECIFICATION_MINOR_REVISION  10
+
+typedef struct {
+  EFI_UINT32  Data1;
+  EFI_UINT16  Data2;
+  EFI_UINT16  Data3;
+  EFI_UINT8   Data4[8];
+} EFI_GUID;
+
+#define APPLE_VENDOR_GUID \
+    {0xAC39C713, 0x7E50, 0x423D, {0x88, 0x9D, 0x27,0x8F, 0xCC, 0x34, 0x22, 0xB6} }
+
+#define EFI_GLOBAL_VARIABLE_GUID \
+    {0x8BE4DF61, 0x93CA, 0x11d2, {0xAA, 0x0D, 0x00, 0xE0, 0x98, 0x03, 0x2B, 0x8C} }
+
+typedef union {
+  EFI_GUID  Guid;
+  EFI_UINT8     Raw[16];
+} EFI_GUID_UNION;
+
+//
+// EFI Time Abstraction:
+//  Year:       2000 - 20XX
+//  Month:      1 - 12
+//  Day:        1 - 31
+//  Hour:       0 - 23
+//  Minute:     0 - 59
+//  Second:     0 - 59
+//  Nanosecond: 0 - 999,999,999
+//  TimeZone:   -1440 to 1440 or 2047
+//
+typedef struct {
+  EFI_UINT16  Year;
+  EFI_UINT8   Month;
+  EFI_UINT8   Day;
+  EFI_UINT8   Hour;
+  EFI_UINT8   Minute;
+  EFI_UINT8   Second;
+  EFI_UINT8   Pad1;
+  EFI_UINT32  Nanosecond;
+  EFI_INT16   TimeZone;
+  EFI_UINT8   Daylight;
+  EFI_UINT8   Pad2;
+} EFI_TIME;
+
+//
+// Bit definitions for EFI_TIME.Daylight
+//
+#define EFI_TIME_ADJUST_DAYLIGHT  0x01
+#define EFI_TIME_IN_DAYLIGHT      0x02
+
+//
+// Value definition for EFI_TIME.TimeZone
+//
+#define EFI_UNSPECIFIED_TIMEZONE  0x07FF
+
+typedef enum {
+  EfiReservedMemoryType,
+  EfiLoaderCode,
+  EfiLoaderData,
+  EfiBootServicesCode,
+  EfiBootServicesData,
+  EfiRuntimeServicesCode,
+  EfiRuntimeServicesData,
+  EfiConventionalMemory,
+  EfiUnusableMemory,
+  EfiACPIReclaimMemory,
+  EfiACPIMemoryNVS,
+  EfiMemoryMappedIO,
+  EfiMemoryMappedIOPortSpace,
+  EfiPalCode,
+  EfiMaxMemoryType
+} EFI_MEMORY_TYPE;
+
+typedef struct {
+  EFI_UINT64  Signature;
+  EFI_UINT32  Revision;
+  EFI_UINT32  HeaderSize;
+  EFI_UINT32  CRC32;
+  EFI_UINT32  Reserved;
+} __attribute__((aligned(8))) EFI_TABLE_HEADER;
+
+//
+// possible caching types for the memory range
+//
+#define EFI_MEMORY_UC   0x0000000000000001ULL
+#define EFI_MEMORY_WC   0x0000000000000002ULL
+#define EFI_MEMORY_WT   0x0000000000000004ULL
+#define EFI_MEMORY_WB   0x0000000000000008ULL
+#define EFI_MEMORY_UCE  0x0000000000000010ULL
+
+//
+// physical memory protection on range
+//
+#define EFI_MEMORY_WP 0x0000000000001000ULL
+#define EFI_MEMORY_RP 0x0000000000002000ULL
+#define EFI_MEMORY_XP 0x0000000000004000ULL
+
+//
+// range requires a runtime mapping
+//
+#define EFI_MEMORY_RUNTIME  0x8000000000000000ULL
+
+typedef EFI_UINT64  EFI_PHYSICAL_ADDRESS;
+typedef EFI_UINT64  EFI_VIRTUAL_ADDRESS;
+
+#define EFI_MEMORY_DESCRIPTOR_VERSION 1
+typedef struct {
+  EFI_UINT32            Type;
+  EFI_UINT32            Pad;
+  EFI_PHYSICAL_ADDRESS  PhysicalStart;
+  EFI_VIRTUAL_ADDRESS   VirtualStart;
+  EFI_UINT64            NumberOfPages;
+  EFI_UINT64            Attribute;
+} __attribute__((aligned(8))) EFI_MEMORY_DESCRIPTOR;
+
+
+typedef
+EFI_RUNTIMESERVICE
+EFI_STATUS
+(EFIAPI *EFI_SET_VIRTUAL_ADDRESS_MAP) (
+  IN EFI_UINTN                    MemoryMapSize,
+  IN EFI_UINTN                    DescriptorSize,
+  IN EFI_UINT32                   DescriptorVersion,
+  IN EFI_MEMORY_DESCRIPTOR        * VirtualMap
+  ) __attribute__((regparm(0)));
+
+typedef
+EFI_RUNTIMESERVICE
+EFI_STATUS
+(EFIAPI *EFI_CONVERT_POINTER) (
+  IN EFI_UINTN                DebugDisposition,
+  IN OUT VOID                 **Address
+  ) __attribute__((regparm(0)));
+
+//
+// Variable attributes
+//
+#define EFI_VARIABLE_NON_VOLATILE       0x00000001
+#define EFI_VARIABLE_BOOTSERVICE_ACCESS 0x00000002
+#define EFI_VARIABLE_RUNTIME_ACCESS     0x00000004
+#define EFI_VARIABLE_READ_ONLY          0x00000008
+
+typedef
+EFI_RUNTIMESERVICE
+EFI_STATUS
+(EFIAPI *EFI_GET_VARIABLE) (
+  IN EFI_CHAR16               * VariableName,
+  IN EFI_GUID                 * VendorGuid,
+  OUT EFI_UINT32              * Attributes OPTIONAL,
+  IN OUT EFI_UINTN            * DataSize,
+  OUT VOID                    * Data
+  ) __attribute__((regparm(0)));
+
+typedef
+EFI_RUNTIMESERVICE
+EFI_STATUS
+(EFIAPI *EFI_GET_NEXT_VARIABLE_NAME) (
+  IN OUT EFI_UINTN            * VariableNameSize,
+  IN OUT EFI_CHAR16           * VariableName,
+  IN OUT EFI_GUID             * VendorGuid
+  ) __attribute__((regparm(0)));
+
+typedef
+EFI_RUNTIMESERVICE
+EFI_STATUS
+(EFIAPI *EFI_SET_VARIABLE) (
+  IN EFI_CHAR16               * VariableName,
+  IN EFI_GUID                 * VendorGuid,
+  IN EFI_UINT32               Attributes,
+  IN EFI_UINTN                DataSize,
+  IN VOID                     * Data
+  ) __attribute__((regparm(0)));
+
+//
+// EFI Time
+//
+typedef struct {
+  EFI_UINT32  Resolution;
+  EFI_UINT32  Accuracy;
+  EFI_BOOLEAN SetsToZero;
+} __attribute__((aligned(4))) EFI_TIME_CAPABILITIES;
+
+typedef
+EFI_RUNTIMESERVICE
+EFI_STATUS
+(EFIAPI *EFI_GET_TIME) (
+  OUT EFI_TIME                * Time,
+  OUT EFI_TIME_CAPABILITIES   * Capabilities OPTIONAL
+  ) __attribute__((regparm(0)));
+
+typedef
+EFI_RUNTIMESERVICE
+EFI_STATUS
+(EFIAPI *EFI_SET_TIME) (
+  IN EFI_TIME                 * Time
+  ) __attribute__((regparm(0)));
+
+typedef
+EFI_RUNTIMESERVICE
+EFI_STATUS
+(EFIAPI *EFI_GET_WAKEUP_TIME) (
+  OUT EFI_BOOLEAN             * Enabled,
+  OUT EFI_BOOLEAN             * Pending,
+  OUT EFI_TIME                * Time
+  ) __attribute__((regparm(0)));
+
+typedef
+EFI_RUNTIMESERVICE
+EFI_STATUS
+(EFIAPI *EFI_SET_WAKEUP_TIME) (
+  IN EFI_BOOLEAN              Enable,
+  IN EFI_TIME                 * Time OPTIONAL
+  ) __attribute((regparm(0)));
+
+typedef enum {
+  EfiResetCold,
+  EfiResetWarm,
+  EfiResetShutdown,
+
+#ifdef TIANO_EXTENSION_FLAG
+  EfiResetUpdate
+#endif
+
+} EFI_RESET_TYPE;
+
+typedef
+EFI_RUNTIMESERVICE
+VOID
+(EFIAPI *EFI_RESET_SYSTEM) (
+  IN EFI_RESET_TYPE               ResetType,
+  IN EFI_STATUS                   ResetStatus,
+  IN EFI_UINTN                    DataSize,
+  IN EFI_CHAR16                   * ResetData OPTIONAL
+  ) __attribute__((regparm(0)));
+
+typedef
+EFI_RUNTIMESERVICE
+EFI_STATUS
+(EFIAPI *EFI_GET_NEXT_HIGH_MONO_COUNT) (
+  OUT EFI_UINT32                  * HighCount
+  ) __attribute__((regparm(0)));
+
+//
+// Definition of Status Code extended data header
+//
+//  HeaderSize    The size of the architecture. This is specified to enable
+//                the future expansion
+//
+//  Size          The size of the data in bytes. This does not include the size
+//                of the header structure.
+//
+//  Type          A GUID defining the type of the data
+//
+//
+#ifdef TIANO_EXTENSION_FLAG
+
+typedef
+EFI_RUNTIMESERVICE
+EFI_STATUS
+(EFIAPI *EFI_REPORT_STATUS_CODE) (
+  IN EFI_STATUS_CODE_TYPE       Type,
+  IN EFI_STATUS_CODE_VALUE      Value,
+  IN EFI_UINT32                 Instance,
+  IN EFI_GUID                   * CallerId OPTIONAL,
+  IN EFI_STATUS_CODE_DATA       * Data OPTIONAL
+  ) __attribute__((regparm(0)));
+
+#endif
+//
+// EFI Runtime Services Table
+//
+#define EFI_RUNTIME_SERVICES_SIGNATURE  0x56524553544e5552ULL
+#define EFI_RUNTIME_SERVICES_REVISION   ((EFI_SPECIFICATION_MAJOR_REVISION << 16) | (EFI_SPECIFICATION_MINOR_REVISION))
+
+typedef struct {
+  EFI_TABLE_HEADER              Hdr;
+
+  //
+  // Time services
+  //
+  EFI_GET_TIME                  GetTime;
+  EFI_SET_TIME                  SetTime;
+  EFI_GET_WAKEUP_TIME           GetWakeupTime;
+  EFI_SET_WAKEUP_TIME           SetWakeupTime;
+
+  //
+  // Virtual memory services
+  //
+  EFI_SET_VIRTUAL_ADDRESS_MAP   SetVirtualAddressMap;
+  EFI_CONVERT_POINTER           ConvertPointer;
+
+  //
+  // Variable services
+  //
+  EFI_GET_VARIABLE              GetVariable;
+  EFI_GET_NEXT_VARIABLE_NAME    GetNextVariableName;
+  EFI_SET_VARIABLE              SetVariable;
+
+  //
+  // Misc
+  //
+  EFI_GET_NEXT_HIGH_MONO_COUNT  GetNextHighMonotonicCount;
+  EFI_RESET_SYSTEM              ResetSystem;
+
+#ifdef TIANO_EXTENSION_FLAG
+  //
+  // ////////////////////////////////////////////////////
+  // Extended EFI Services
+    //////////////////////////////////////////////////////
+  //
+  EFI_REPORT_STATUS_CODE  ReportStatusCode;
+#endif
+
+} __attribute__((aligned(8))) EFI_RUNTIME_SERVICES;
+
+typedef struct {
+  EFI_TABLE_HEADER              Hdr;
+
+  //
+  // Time services
+  //
+  EFI_PTR64                  GetTime;
+  EFI_PTR64                  SetTime;
+  EFI_PTR64           GetWakeupTime;
+  EFI_PTR64           SetWakeupTime;
+
+  //
+  // Virtual memory services
+  //
+  EFI_PTR64   SetVirtualAddressMap;
+  EFI_PTR64           ConvertPointer;
+
+  //
+  // Variable services
+  //
+  EFI_PTR64             GetVariable;
+  EFI_PTR64    GetNextVariableName;
+  EFI_PTR64              SetVariable;
+
+  //
+  // Misc
+  //
+  EFI_PTR64  GetNextHighMonotonicCount;
+  EFI_PTR64              ResetSystem;
+
+#ifdef TIANO_EXTENSION_FLAG
+  //
+  // ////////////////////////////////////////////////////
+  // Extended EFI Services
+    //////////////////////////////////////////////////////
+  //
+  EFI_PTR64 ReportStatusCode;
+#endif
+
+} __attribute__((aligned(8))) EFI_RUNTIME_SERVICES_64;
+
+//
+// EFI Configuration Table
+//
+typedef struct {
+  EFI_GUID  VendorGuid;
+  VOID      *VendorTable;
+} EFI_CONFIGURATION_TABLE;
+
+//
+// EFI System Table
+//
+#define EFI_SYSTEM_TABLE_SIGNATURE      0x5453595320494249ULL
+#define EFI_SYSTEM_TABLE_REVISION       ((EFI_SPECIFICATION_MAJOR_REVISION << 16) | (EFI_SPECIFICATION_MINOR_REVISION))
+#define EFI_2_00_SYSTEM_TABLE_REVISION  ((2 << 16) | 00)
+#define EFI_1_02_SYSTEM_TABLE_REVISION  ((1 << 16) | 02)
+#define EFI_1_10_SYSTEM_TABLE_REVISION  ((1 << 16) | 10)
+
+typedef struct EFI_SYSTEM_TABLE {
+  EFI_TABLE_HEADER              Hdr;
+
+  EFI_CHAR16                    *FirmwareVendor;
+  EFI_UINT32                    FirmwareRevision;
+
+  EFI_HANDLE                    ConsoleInHandle;
+  VOID				*ConIn;
+
+  EFI_HANDLE                    ConsoleOutHandle;
+  VOID				*ConOut;
+
+  EFI_HANDLE                    StandardErrorHandle;
+  VOID				*StdErr;
+
+  EFI_RUNTIME_SERVICES          *RuntimeServices;
+  VOID				*BootServices;
+
+  EFI_UINTN                     NumberOfTableEntries;
+  EFI_CONFIGURATION_TABLE       *ConfigurationTable;
+
+} __attribute__((aligned(8))) EFI_SYSTEM_TABLE;
+
+typedef struct EFI_SYSTEM_TABLE_64 {
+  EFI_TABLE_HEADER              Hdr;
+
+  EFI_PTR64                     FirmwareVendor;
+  EFI_UINT32                    FirmwareRevision;
+
+  EFI_UINT32                    __pad;
+
+  EFI_HANDLE64                  ConsoleInHandle;
+  EFI_PTR64			ConIn;
+
+  EFI_HANDLE64                  ConsoleOutHandle;
+  EFI_PTR64			ConOut;
+
+  EFI_HANDLE64                  StandardErrorHandle;
+  EFI_PTR64			StdErr;
+
+  EFI_PTR64                     RuntimeServices;
+  EFI_PTR64			BootServices;
+
+  EFI_UINT64                    NumberOfTableEntries;
+  EFI_PTR64                     ConfigurationTable;
+
+} __attribute__((aligned(8))) EFI_SYSTEM_TABLE_64;
+
+#endif /* _PEXPERT_I386_EFI_H */
diff --git a/pexpert/pexpert/ppc/protos.h b/pexpert/pexpert/ppc/protos.h
index 4b81f1cf6..a5b6a0d8b 100644
--- a/pexpert/pexpert/ppc/protos.h
+++ b/pexpert/pexpert/ppc/protos.h
@@ -149,4 +149,9 @@ extern void GratefulDebInit(void);
 extern void GratefulDebDisp(unsigned int coord, unsigned int data);
 extern void checkNMI(void);
 
+#ifndef VM_WIMG_IO
+#define VM_WIMG_IO		(VM_MEM_COHERENT | 	\
+				VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED)
+#endif
+
 #endif /* _PEXPERT_PPC_PROTOS_H_ */
diff --git a/pexpert/ppc/pe_identify_machine.c b/pexpert/ppc/pe_identify_machine.c
index d4336b384..1417dbaf4 100644
--- a/pexpert/ppc/pe_identify_machine.c
+++ b/pexpert/ppc/pe_identify_machine.c
@@ -29,6 +29,7 @@
 #include <pexpert/pexpert.h>
 #include <pexpert/ppc/powermac.h>
 #include <pexpert/device_tree.h>
+#include <vm/pmap.h>
 
 /* pe_identify_machine:
  *
@@ -181,7 +182,7 @@ unsigned int PE_init_taproot(vm_offset_t *taddr)
 
 	tappdata[1] = (tappdata[1] + 4095 ) & -4096;	/* Make sure this is a whole page */
 
-	*taddr = io_map_spec(tappdata[0], tappdata[1]);		/* Map it in and return the address */
+	*taddr = io_map_spec(tappdata[0], tappdata[1], VM_WIMG_IO);	/* Map it in and return the address */
 	tappdata[0] = *taddr;					/* Also change property */
 	return tappdata[1];						/* And the size */
 }
diff --git a/pexpert/ppc/pe_init.c b/pexpert/ppc/pe_init.c
index a1816b94f..9958e8cc6 100644
--- a/pexpert/ppc/pe_init.c
+++ b/pexpert/ppc/pe_init.c
@@ -37,6 +37,7 @@
 #include <pexpert/pe_images.h>
 #include <kern/debug.h>
 #include <kern/sched_prim.h>
+#include <vm/pmap.h>
 
 
 /* extern references */
diff --git a/pexpert/ppc/pe_kprintf.c b/pexpert/ppc/pe_kprintf.c
index a8118fc5a..922291fa0 100644
--- a/pexpert/ppc/pe_kprintf.c
+++ b/pexpert/ppc/pe_kprintf.c
@@ -37,6 +37,7 @@
 #include <pexpert/device_tree.h>
 #include <kern/debug.h>
 #include <kern/simple_lock.h>
+#include <vm/pmap.h>
 
 /* extern references */
 extern void init_display_putc(unsigned char*, int, int);
@@ -99,7 +100,7 @@ void PE_init_kprintf(boolean_t vm_initialized)
 		if (serial_baud != -1) gPESerialBaud = serial_baud; 
 
 	if( (scc = PE_find_scc())) {				/* See if we can find the serial port */
-		scc = io_map_spec(scc, 0x1000);				 /* Map it in */
+		scc = io_map_spec(scc, 0x1000, VM_WIMG_IO);	/* Map it in */
 		initialize_serial((void *)scc, gPESerialBaud); /* Start up the serial driver */
 		PE_kputc = serial_putc;
 
-- 
2.50.0

